{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 550436 characters]",
      "text/plain": [
       "<Figure size 1400x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "train_all_path = '../../data_preprocess/dataset/1/train_all.csv'\n",
    "test_path = '../../translation/0.result/1/test_p.csv'\n",
    "\n",
    "train_df = pd.read_csv(train_all_path)\n",
    "test_df = pd.read_csv(test_path)\n",
    "\n",
    "data = pd.concat([train_df, test_df], ignore_index=True)\n",
    "data = data[data['MDM'] == True]\n",
    "\n",
    "vectorizer = TfidfVectorizer(token_pattern=r'\\S+')\n",
    "tfidf_matrix = vectorizer.fit_transform(data['tag_description'])\n",
    "\n",
    "grouped_data = data.groupby(['thing', 'property']).agg(\n",
    "    tp_count=('thing', 'size'), pattern=('pattern', 'first')).reset_index()\n",
    "\n",
    "result = grouped_data.loc[grouped_data.groupby('pattern')['tp_count'].idxmax()].reset_index(drop=True)\n",
    "\n",
    "top_n = 20\n",
    "top_n_patterns = result.nlargest(top_n, 'tp_count')\n",
    "\n",
    "filtered_data = data[data['pattern'].isin(top_n_patterns['pattern'])]\n",
    "filtered_data.to_csv('filtered_data_plot.csv', index=False)\n",
    "filtered_tfidf_matrix = vectorizer.transform(filtered_data['tag_description'])\n",
    "\n",
    "tsne = TSNE(n_components=2, random_state=42)\n",
    "tsne_results = tsne.fit_transform(filtered_tfidf_matrix.toarray())\n",
    "\n",
    "unique_patterns = filtered_data['pattern'].unique()\n",
    "color_map = plt.get_cmap('tab20', len(unique_patterns))\n",
    "pattern_to_color = {pattern: idx for idx, pattern in enumerate(unique_patterns)}\n",
    "\n",
    "plt.figure(figsize=(14, 7))\n",
    "\n",
    "for pattern, color_idx in pattern_to_color.items():\n",
    "    pattern_indices = filtered_data['pattern'] == pattern\n",
    "    plt.scatter(tsne_results[pattern_indices, 0], tsne_results[pattern_indices, 1], \n",
    "                color=color_map(color_idx), marker='o', s=100, alpha=0.6, edgecolor='k', linewidth=1.2)\n",
    "\n",
    "plt.xticks(fontsize=24)\n",
    "plt.yticks(fontsize=24)\n",
    "plt.grid(True, which='both', linestyle='--', linewidth=0.5, alpha=0.6)\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 566012 characters]",
      "text/plain": [
       "<Figure size 1400x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# 'filtered_data_plot.csv' 파일을 읽기\n",
    "filtered_data = pd.read_csv('filtered_data_plot.csv')\n",
    "\n",
    "# BoW로 벡터화 (token_pattern=r'\\S+'로 설정하여 모든 non-whitespace를 토큰으로 사용)\n",
    "vectorizer = CountVectorizer(token_pattern=r'\\S+')\n",
    "bow_matrix = vectorizer.fit_transform(filtered_data['tag_description'])\n",
    "\n",
    "# t-SNE 차원 축소\n",
    "tsne = TSNE(n_components=2, random_state=42)\n",
    "tsne_results = tsne.fit_transform(bow_matrix.toarray())\n",
    "\n",
    "# 시각화를 위한 준비\n",
    "unique_patterns = filtered_data['pattern'].unique()\n",
    "color_map = plt.get_cmap('tab20', len(unique_patterns))\n",
    "pattern_to_color = {pattern: idx for idx, pattern in enumerate(unique_patterns)}\n",
    "\n",
    "plt.figure(figsize=(14, 7))\n",
    "\n",
    "# 각 패턴별로 시각화\n",
    "for pattern, color_idx in pattern_to_color.items():\n",
    "    pattern_indices = filtered_data['pattern'] == pattern\n",
    "    plt.scatter(tsne_results[pattern_indices, 0], tsne_results[pattern_indices, 1], \n",
    "                color=color_map(color_idx), marker='o', s=100, alpha=0.6, edgecolor='k', linewidth=1.2)\n",
    "\n",
    "# 그래프 설정\n",
    "plt.xticks(fontsize=24)\n",
    "plt.yticks(fontsize=24)\n",
    "plt.grid(True, which='both', linestyle='--', linewidth=0.5, alpha=0.6)\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 489196 characters]",
      "text/plain": [
       "<Figure size 1400x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from gensim.models import Word2Vec\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "def compute_sentence_vector(sentence, model, vector_size):\n",
    "    words = sentence.split()\n",
    "    word_vectors = [model.wv[word] for word in words if word in model.wv]\n",
    "    return np.mean(word_vectors, axis=0) if word_vectors else np.zeros(vector_size)\n",
    "\n",
    "filtered_data = pd.read_csv('filtered_data_plot.csv')\n",
    "\n",
    "sentences = [desc.split() for desc in filtered_data['tag_description'].tolist()]\n",
    "\n",
    "vector_size = 200\n",
    "window_size = 3\n",
    "model = Word2Vec(sentences, vector_size=vector_size, window=window_size, min_count=1, workers=4)\n",
    "\n",
    "sentence_vectors = np.array([compute_sentence_vector(desc, model, vector_size) for desc in filtered_data['tag_description']])\n",
    "\n",
    "tsne = TSNE(n_components=2, random_state=42)\n",
    "tsne_results = tsne.fit_transform(sentence_vectors)\n",
    "\n",
    "unique_patterns = filtered_data['pattern'].unique()\n",
    "color_map = plt.get_cmap('tab20', len(unique_patterns))\n",
    "pattern_to_color = {pattern: idx for idx, pattern in enumerate(unique_patterns)}\n",
    "\n",
    "plt.figure(figsize=(14, 7))\n",
    "\n",
    "for pattern, color_idx in pattern_to_color.items():\n",
    "    pattern_indices = filtered_data['pattern'] == pattern\n",
    "    plt.scatter(tsne_results[pattern_indices, 0], tsne_results[pattern_indices, 1], \n",
    "                color=color_map(color_idx), marker='o', s=100, alpha=0.6, edgecolor='k', linewidth=1.2)\n",
    "\n",
    "plt.xticks(fontsize=24)\n",
    "plt.yticks(fontsize=24)\n",
    "plt.grid(True, which='both', linestyle='--', linewidth=0.5, alpha=0.6)\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
