342 lines
19 KiB
Plaintext
342 lines
19 KiB
Plaintext
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 35,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"Some weights of AlbertForSequenceClassification were not initialized from the model checkpoint at albert-base-v2 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
|
||
|
|
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
|
||
|
|
"/home/hwang/anaconda3/envs/torch/lib/python3.10/site-packages/transformers/optimization.py:521: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
|
||
|
|
" warnings.warn(\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"Epoch 1 completed. Loss: 5.564172267913818\n",
|
||
|
|
"Epoch 2 completed. Loss: 4.88321590423584\n",
|
||
|
|
"Epoch 3 completed. Loss: 3.5059947967529297\n",
|
||
|
|
"Epoch 4 completed. Loss: 3.18548583984375\n",
|
||
|
|
"Epoch 5 completed. Loss: 2.8037068843841553\n",
|
||
|
|
"Epoch 6 completed. Loss: 2.2223541736602783\n",
|
||
|
|
"Epoch 7 completed. Loss: 1.8634291887283325\n",
|
||
|
|
"Epoch 8 completed. Loss: 1.3251842260360718\n",
|
||
|
|
"Epoch 9 completed. Loss: 0.6083177328109741\n",
|
||
|
|
"Epoch 10 completed. Loss: 0.9423710703849792\n",
|
||
|
|
"Epoch 11 completed. Loss: 0.5799884796142578\n",
|
||
|
|
"Epoch 12 completed. Loss: 0.6948736310005188\n",
|
||
|
|
"Epoch 13 completed. Loss: 0.5177479386329651\n",
|
||
|
|
"Epoch 14 completed. Loss: 0.47343072295188904\n",
|
||
|
|
"Epoch 15 completed. Loss: 0.26853761076927185\n",
|
||
|
|
"Epoch 16 completed. Loss: 0.19693760573863983\n",
|
||
|
|
"Epoch 17 completed. Loss: 0.3199688494205475\n",
|
||
|
|
"Epoch 18 completed. Loss: 0.23672448098659515\n",
|
||
|
|
"Epoch 19 completed. Loss: 0.40235987305641174\n",
|
||
|
|
"Epoch 20 completed. Loss: 0.28102293610572815\n",
|
||
|
|
"Epoch 21 completed. Loss: 0.17575399577617645\n",
|
||
|
|
"Epoch 22 completed. Loss: 0.24652625620365143\n",
|
||
|
|
"Epoch 23 completed. Loss: 0.109055295586586\n",
|
||
|
|
"Epoch 24 completed. Loss: 0.19015412032604218\n",
|
||
|
|
"Epoch 25 completed. Loss: 0.10130400210618973\n",
|
||
|
|
"Epoch 26 completed. Loss: 0.14203257858753204\n",
|
||
|
|
"Epoch 27 completed. Loss: 0.1248723715543747\n",
|
||
|
|
"Epoch 28 completed. Loss: 0.05851107835769653\n",
|
||
|
|
"Epoch 29 completed. Loss: 0.041425254195928574\n",
|
||
|
|
"Epoch 30 completed. Loss: 0.0353962741792202\n",
|
||
|
|
"Epoch 31 completed. Loss: 0.04445452615618706\n",
|
||
|
|
"Epoch 32 completed. Loss: 0.026403019204735756\n",
|
||
|
|
"Epoch 33 completed. Loss: 0.028079884126782417\n",
|
||
|
|
"Epoch 34 completed. Loss: 0.059587348252534866\n",
|
||
|
|
"Epoch 35 completed. Loss: 0.02851276472210884\n",
|
||
|
|
"Epoch 36 completed. Loss: 0.09271513670682907\n",
|
||
|
|
"Epoch 37 completed. Loss: 0.06418397277593613\n",
|
||
|
|
"Epoch 38 completed. Loss: 0.03638231381773949\n",
|
||
|
|
"Epoch 39 completed. Loss: 0.022959664463996887\n",
|
||
|
|
"Epoch 40 completed. Loss: 0.044602662324905396\n",
|
||
|
|
"Epoch 41 completed. Loss: 0.03491249307990074\n",
|
||
|
|
"Epoch 42 completed. Loss: 0.039797600358724594\n",
|
||
|
|
"Epoch 43 completed. Loss: 0.04217083007097244\n",
|
||
|
|
"Epoch 44 completed. Loss: 0.4122176170349121\n",
|
||
|
|
"Epoch 45 completed. Loss: 0.1664775162935257\n",
|
||
|
|
"Epoch 46 completed. Loss: 0.04505300521850586\n",
|
||
|
|
"Epoch 47 completed. Loss: 0.14913827180862427\n",
|
||
|
|
"Epoch 48 completed. Loss: 0.016096509993076324\n",
|
||
|
|
"Epoch 49 completed. Loss: 0.05338064581155777\n",
|
||
|
|
"Epoch 50 completed. Loss: 0.10259533673524857\n",
|
||
|
|
"Epoch 51 completed. Loss: 0.008849691599607468\n",
|
||
|
|
"Epoch 52 completed. Loss: 0.028069255873560905\n",
|
||
|
|
"Epoch 53 completed. Loss: 0.008924427442252636\n",
|
||
|
|
"Epoch 54 completed. Loss: 0.015527592971920967\n",
|
||
|
|
"Epoch 55 completed. Loss: 0.009189464151859283\n",
|
||
|
|
"Epoch 56 completed. Loss: 0.007252057082951069\n",
|
||
|
|
"Epoch 57 completed. Loss: 0.01684846170246601\n",
|
||
|
|
"Epoch 58 completed. Loss: 0.010840333066880703\n",
|
||
|
|
"Epoch 59 completed. Loss: 0.05179211124777794\n",
|
||
|
|
"Epoch 60 completed. Loss: 0.007003726437687874\n",
|
||
|
|
"Epoch 61 completed. Loss: 0.00555015355348587\n",
|
||
|
|
"Epoch 62 completed. Loss: 0.0065276664681732655\n",
|
||
|
|
"Epoch 63 completed. Loss: 0.007942711934447289\n",
|
||
|
|
"Epoch 64 completed. Loss: 0.00675524678081274\n",
|
||
|
|
"Epoch 65 completed. Loss: 0.010359193198382854\n",
|
||
|
|
"Epoch 66 completed. Loss: 0.00662408908829093\n",
|
||
|
|
"Epoch 67 completed. Loss: 0.007672889623790979\n",
|
||
|
|
"Epoch 68 completed. Loss: 0.004661311395466328\n",
|
||
|
|
"Epoch 69 completed. Loss: 0.014480670914053917\n",
|
||
|
|
"Epoch 70 completed. Loss: 0.05042335391044617\n",
|
||
|
|
"Epoch 71 completed. Loss: 0.035947512835264206\n",
|
||
|
|
"Epoch 72 completed. Loss: 0.01213429868221283\n",
|
||
|
|
"Epoch 73 completed. Loss: 0.033572785556316376\n",
|
||
|
|
"Epoch 74 completed. Loss: 0.009208262898027897\n",
|
||
|
|
"Epoch 75 completed. Loss: 0.08961852639913559\n",
|
||
|
|
"Epoch 76 completed. Loss: 4.632999897003174\n",
|
||
|
|
"Epoch 77 completed. Loss: 5.957398891448975\n",
|
||
|
|
"Epoch 78 completed. Loss: 5.970841407775879\n",
|
||
|
|
"Epoch 79 completed. Loss: 5.905709266662598\n",
|
||
|
|
"Epoch 80 completed. Loss: 5.864459037780762\n",
|
||
|
|
"Validation Accuracy: 0.14%\n",
|
||
|
|
"Accuracy (MDM=True) for Group 4: 0.48%\n",
|
||
|
|
"Results saved to 0.class_document/albert/4/test_p_c.csv\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"import pandas as pd\n",
|
||
|
|
"from transformers import AlbertTokenizer, AlbertForSequenceClassification, AdamW\n",
|
||
|
|
"from sklearn.preprocessing import LabelEncoder\n",
|
||
|
|
"import torch\n",
|
||
|
|
"from torch.utils.data import Dataset, DataLoader\n",
|
||
|
|
"import numpy as np\n",
|
||
|
|
"import torch.nn.functional as F\n",
|
||
|
|
"import os \n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"group_number = 4\n",
|
||
|
|
"train_path = f'../../data_preprocess/dataset/{group_number}/train.csv'\n",
|
||
|
|
"valid_path = f'../../data_preprocess/dataset/{group_number}/valid.csv'\n",
|
||
|
|
"test_path = f'../../translation/0.result/{group_number}/test_p.csv'\n",
|
||
|
|
"output_path = f'0.class_document/albert/{group_number}/test_p_c.csv'\n",
|
||
|
|
"\n",
|
||
|
|
"train_data = pd.read_csv(train_path)\n",
|
||
|
|
"valid_data = pd.read_csv(valid_path)\n",
|
||
|
|
"test_data = pd.read_csv(test_path)\n",
|
||
|
|
"\n",
|
||
|
|
"train_data['thing_property'] = train_data['thing'] + '_' + train_data['property']\n",
|
||
|
|
"valid_data['thing_property'] = valid_data['thing'] + '_' + valid_data['property']\n",
|
||
|
|
"test_data['thing_property'] = test_data['thing'] + '_' + test_data['property']\n",
|
||
|
|
"\n",
|
||
|
|
"tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')\n",
|
||
|
|
"label_encoder = LabelEncoder()\n",
|
||
|
|
"label_encoder.fit(train_data['thing_property'])\n",
|
||
|
|
"\n",
|
||
|
|
"valid_data['thing_property'] = valid_data['thing_property'].apply(\n",
|
||
|
|
" lambda x: x if x in label_encoder.classes_ else 'unknown_label')\n",
|
||
|
|
"test_data['thing_property'] = test_data['thing_property'].apply(\n",
|
||
|
|
" lambda x: x if x in label_encoder.classes_ else 'unknown_label')\n",
|
||
|
|
"\n",
|
||
|
|
"label_encoder.classes_ = np.append(label_encoder.classes_, 'unknown_label')\n",
|
||
|
|
"\n",
|
||
|
|
"train_data['label'] = label_encoder.transform(train_data['thing_property'])\n",
|
||
|
|
"valid_data['label'] = label_encoder.transform(valid_data['thing_property'])\n",
|
||
|
|
"test_data['label'] = label_encoder.transform(test_data['thing_property'])\n",
|
||
|
|
"\n",
|
||
|
|
"train_texts, train_labels = train_data['tag_description'], train_data['label']\n",
|
||
|
|
"valid_texts, valid_labels = valid_data['tag_description'], valid_data['label']\n",
|
||
|
|
"\n",
|
||
|
|
"train_encodings = tokenizer(list(train_texts), truncation=True, padding=True, return_tensors='pt')\n",
|
||
|
|
"valid_encodings = tokenizer(list(valid_texts), truncation=True, padding=True, return_tensors='pt')\n",
|
||
|
|
"\n",
|
||
|
|
"train_labels = torch.tensor(train_labels.values)\n",
|
||
|
|
"valid_labels = torch.tensor(valid_labels.values)\n",
|
||
|
|
"\n",
|
||
|
|
"class CustomDataset(Dataset):\n",
|
||
|
|
" def __init__(self, encodings, labels):\n",
|
||
|
|
" self.encodings = encodings\n",
|
||
|
|
" self.labels = labels\n",
|
||
|
|
"\n",
|
||
|
|
" def __getitem__(self, idx):\n",
|
||
|
|
" item = {key: val[idx] for key, val in self.encodings.items()}\n",
|
||
|
|
" item['labels'] = self.labels[idx]\n",
|
||
|
|
" return item\n",
|
||
|
|
"\n",
|
||
|
|
" def __len__(self):\n",
|
||
|
|
" return len(self.labels)\n",
|
||
|
|
"\n",
|
||
|
|
"train_dataset = CustomDataset(train_encodings, train_labels)\n",
|
||
|
|
"valid_dataset = CustomDataset(valid_encodings, valid_labels)\n",
|
||
|
|
"\n",
|
||
|
|
"train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n",
|
||
|
|
"valid_loader = DataLoader(valid_dataset, batch_size=32, shuffle=False)\n",
|
||
|
|
"\n",
|
||
|
|
"model = AlbertForSequenceClassification.from_pretrained('albert-base-v2', num_labels=len(train_data['thing_property'].unique()))\n",
|
||
|
|
"optimizer = AdamW(model.parameters(), lr=5e-5)\n",
|
||
|
|
"\n",
|
||
|
|
"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
|
||
|
|
"model.to(device)\n",
|
||
|
|
"\n",
|
||
|
|
"epochs = 80\n",
|
||
|
|
"for epoch in range(epochs):\n",
|
||
|
|
" model.train()\n",
|
||
|
|
" for batch in train_loader:\n",
|
||
|
|
" optimizer.zero_grad()\n",
|
||
|
|
" input_ids = batch['input_ids'].to(device)\n",
|
||
|
|
" attention_mask = batch['attention_mask'].to(device)\n",
|
||
|
|
" labels = batch['labels'].to(device)\n",
|
||
|
|
" outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
|
||
|
|
" loss = outputs.loss\n",
|
||
|
|
" loss.backward()\n",
|
||
|
|
" optimizer.step()\n",
|
||
|
|
" print(f\"Epoch {epoch + 1} completed. Loss: {loss.item()}\")\n",
|
||
|
|
"\n",
|
||
|
|
"model.eval()\n",
|
||
|
|
"correct, total = 0, 0\n",
|
||
|
|
"\n",
|
||
|
|
"with torch.no_grad():\n",
|
||
|
|
" for batch in valid_loader:\n",
|
||
|
|
" input_ids, attention_mask, labels = batch['input_ids'].to(device), batch['attention_mask'].to(device), batch['labels'].to(device)\n",
|
||
|
|
" outputs = model(input_ids, attention_mask=attention_mask)\n",
|
||
|
|
" predictions = torch.argmax(outputs.logits, dim=-1)\n",
|
||
|
|
" correct += (predictions == labels).sum().item()\n",
|
||
|
|
" total += labels.size(0)\n",
|
||
|
|
"\n",
|
||
|
|
"valid_accuracy = correct / total\n",
|
||
|
|
"print(f'Validation Accuracy: {valid_accuracy * 100:.2f}%')\n",
|
||
|
|
"\n",
|
||
|
|
"# Test 데이터 예측 및 c_thing, c_property 추가\n",
|
||
|
|
"test_encodings = tokenizer(list(test_data['tag_description']), truncation=True, padding=True, return_tensors='pt')\n",
|
||
|
|
"test_dataset = CustomDataset(test_encodings, torch.zeros(len(test_data))) # 레이블은 사용되지 않으므로 임시로 0을 사용\n",
|
||
|
|
"\n",
|
||
|
|
"test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)\n",
|
||
|
|
"\n",
|
||
|
|
"model.eval()\n",
|
||
|
|
"predicted_thing_properties = []\n",
|
||
|
|
"predicted_scores = []\n",
|
||
|
|
"\n",
|
||
|
|
"with torch.no_grad():\n",
|
||
|
|
" for batch in test_loader:\n",
|
||
|
|
" input_ids, attention_mask = batch['input_ids'].to(device), batch['attention_mask'].to(device)\n",
|
||
|
|
" outputs = model(input_ids, attention_mask=attention_mask)\n",
|
||
|
|
" softmax_scores = F.softmax(outputs.logits, dim=-1)\n",
|
||
|
|
" predictions = torch.argmax(softmax_scores, dim=-1)\n",
|
||
|
|
" predicted_thing_properties.extend(predictions.cpu().numpy())\n",
|
||
|
|
" predicted_scores.extend(softmax_scores[range(len(predictions)), predictions].cpu().numpy())\n",
|
||
|
|
"\n",
|
||
|
|
"# 예측된 thing_property를 레이블 인코더로 디코딩\n",
|
||
|
|
"predicted_thing_property_labels = label_encoder.inverse_transform(predicted_thing_properties)\n",
|
||
|
|
"\n",
|
||
|
|
"# thing_property를 thing과 property로 나눔\n",
|
||
|
|
"test_data['c_thing'] = [x.split('_')[0] for x in predicted_thing_property_labels]\n",
|
||
|
|
"test_data['c_property'] = [x.split('_')[1] for x in predicted_thing_property_labels]\n",
|
||
|
|
"test_data['c_score'] = predicted_scores\n",
|
||
|
|
"\n",
|
||
|
|
"test_data['cthing_correct'] = test_data['thing'] == test_data['c_thing']\n",
|
||
|
|
"test_data['cproperty_correct'] = test_data['property'] == test_data['c_property']\n",
|
||
|
|
"test_data['ctp_correct'] = test_data['cthing_correct'] & test_data['cproperty_correct']\n",
|
||
|
|
"\n",
|
||
|
|
"mdm_true_count = len(test_data[test_data['MDM'] == True])\n",
|
||
|
|
"accuracy = (test_data['ctp_correct'].sum() / mdm_true_count) * 100\n",
|
||
|
|
"\n",
|
||
|
|
"print(f\"Accuracy (MDM=True) for Group {group_number}: {accuracy:.2f}%\")\n",
|
||
|
|
"\n",
|
||
|
|
"os.makedirs(os.path.dirname(output_path), exist_ok=True)\n",
|
||
|
|
"\n",
|
||
|
|
"test_data.to_csv(output_path, index=False)\n",
|
||
|
|
"print(f'Results saved to {output_path}')\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 29,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"ename": "AttributeError",
|
||
|
|
"evalue": "'AlbertForSequenceClassification' object has no attribute 'bert'",
|
||
|
|
"output_type": "error",
|
||
|
|
"traceback": [
|
||
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||
|
|
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||
|
|
"Cell \u001b[0;32mIn[29], line 20\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m outputs\u001b[38;5;241m.\u001b[39mlast_hidden_state\u001b[38;5;241m.\u001b[39mmean(dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mcpu()\u001b[38;5;241m.\u001b[39mnumpy() \u001b[38;5;66;03m# 각 문장의 평균 임베딩 추출\u001b[39;00m\n\u001b[1;32m 19\u001b[0m \u001b[38;5;66;03m# BERT 모델로 임베딩 계산\u001b[39;00m\n\u001b[0;32m---> 20\u001b[0m bert_embeddings \u001b[38;5;241m=\u001b[39m \u001b[43mget_bert_embeddings\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfiltered_encodings\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 22\u001b[0m \u001b[38;5;66;03m# t-SNE 차원 축소\u001b[39;00m\n\u001b[1;32m 23\u001b[0m tsne \u001b[38;5;241m=\u001b[39m TSNE(n_components\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m, random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m42\u001b[39m)\n",
|
||
|
|
"Cell \u001b[0;32mIn[29], line 16\u001b[0m, in \u001b[0;36mget_bert_embeddings\u001b[0;34m(model, encodings, device)\u001b[0m\n\u001b[1;32m 14\u001b[0m input_ids \u001b[38;5;241m=\u001b[39m encodings[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m 15\u001b[0m attention_mask \u001b[38;5;241m=\u001b[39m encodings[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mattention_mask\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m---> 16\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbert\u001b[49m(input_ids\u001b[38;5;241m=\u001b[39minput_ids, attention_mask\u001b[38;5;241m=\u001b[39mattention_mask)\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m outputs\u001b[38;5;241m.\u001b[39mlast_hidden_state\u001b[38;5;241m.\u001b[39mmean(dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mcpu()\u001b[38;5;241m.\u001b[39mnumpy()\n",
|
||
|
|
"File \u001b[0;32m~/anaconda3/envs/torch/lib/python3.10/site-packages/torch/nn/modules/module.py:1709\u001b[0m, in \u001b[0;36mModule.__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 1707\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m modules:\n\u001b[1;32m 1708\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m modules[name]\n\u001b[0;32m-> 1709\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m object has no attribute \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
|
||
|
|
"\u001b[0;31mAttributeError\u001b[0m: 'AlbertForSequenceClassification' object has no attribute 'bert'"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"from sklearn.manifold import TSNE\n",
|
||
|
|
"import matplotlib.pyplot as plt\n",
|
||
|
|
"\n",
|
||
|
|
"# 'filtered_data_plot.csv' 읽기\n",
|
||
|
|
"filtered_data = pd.read_csv('filtered_data_plot.csv')\n",
|
||
|
|
"\n",
|
||
|
|
"# 데이터 토큰화\n",
|
||
|
|
"filtered_encodings = tokenizer(list(filtered_data['tag_description']), truncation=True, padding=True, return_tensors='pt')\n",
|
||
|
|
"\n",
|
||
|
|
"# BERT 임베딩 계산 함수\n",
|
||
|
|
"def get_bert_embeddings(model, encodings, device):\n",
|
||
|
|
" model.eval()\n",
|
||
|
|
" with torch.no_grad():\n",
|
||
|
|
" input_ids = encodings['input_ids'].to(device)\n",
|
||
|
|
" attention_mask = encodings['attention_mask'].to(device)\n",
|
||
|
|
" outputs = model.bert(input_ids=input_ids, attention_mask=attention_mask)\n",
|
||
|
|
" return outputs.last_hidden_state.mean(dim=1).cpu().numpy() # 각 문장의 평균 임베딩 추출\n",
|
||
|
|
"\n",
|
||
|
|
"# BERT 모델로 임베딩 계산\n",
|
||
|
|
"bert_embeddings = get_bert_embeddings(model, filtered_encodings, device)\n",
|
||
|
|
"\n",
|
||
|
|
"# t-SNE 차원 축소\n",
|
||
|
|
"tsne = TSNE(n_components=2, random_state=42)\n",
|
||
|
|
"tsne_results = tsne.fit_transform(bert_embeddings)\n",
|
||
|
|
"\n",
|
||
|
|
"# 시각화를 위한 준비\n",
|
||
|
|
"unique_patterns = filtered_data['pattern'].unique()\n",
|
||
|
|
"color_map = plt.get_cmap('tab20', len(unique_patterns))\n",
|
||
|
|
"pattern_to_color = {pattern: idx for idx, pattern in enumerate(unique_patterns)}\n",
|
||
|
|
"\n",
|
||
|
|
"plt.figure(figsize=(14, 7))\n",
|
||
|
|
"\n",
|
||
|
|
"# 각 패턴별로 시각화\n",
|
||
|
|
"for pattern, color_idx in pattern_to_color.items():\n",
|
||
|
|
" pattern_indices = filtered_data['pattern'] == pattern\n",
|
||
|
|
" plt.scatter(tsne_results[pattern_indices, 0], tsne_results[pattern_indices, 1], \n",
|
||
|
|
" color=color_map(color_idx), marker='o', s=100, alpha=0.6, edgecolor='k', linewidth=1.2)\n",
|
||
|
|
"\n",
|
||
|
|
"# 그래프 설정\n",
|
||
|
|
"plt.xticks(fontsize=24)\n",
|
||
|
|
"plt.yticks(fontsize=24)\n",
|
||
|
|
"plt.grid(True, which='both', linestyle='--', linewidth=0.5, alpha=0.6)\n",
|
||
|
|
"plt.tight_layout()\n",
|
||
|
|
"plt.show()\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "torch",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 3
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython3",
|
||
|
|
"version": "3.10.14"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 2
|
||
|
|
}
|