117 lines
4.4 KiB
Plaintext
117 lines
4.4 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Accuracy (MDM=True) for Group 1: 73.50%\n",
|
|
"Accuracy (MDM=True) for Group 2: 78.04%\n",
|
|
"Accuracy (MDM=True) for Group 3: 81.73%\n",
|
|
"Accuracy (MDM=True) for Group 4: 79.83%\n",
|
|
"Accuracy (MDM=True) for Group 5: 81.31%\n",
|
|
"Average Accuracy (MDM=True) across all groups: 78.88%\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
|
"from sklearn.metrics import pairwise_distances\n",
|
|
"from tqdm import tqdm\n",
|
|
"import os\n",
|
|
"\n",
|
|
"accuracies = []\n",
|
|
"\n",
|
|
"for group_number in range(1, 6):\n",
|
|
" \n",
|
|
" sdl_class_rdoc_path = f'0.class_document/{group_number}/sdl_class_rdoc.csv'\n",
|
|
" test_path = f'../../translation/0.result/{group_number}/test_p.csv'\n",
|
|
" \n",
|
|
" if not os.path.exists(test_path):\n",
|
|
" print(f\"test file for Group {group_number} does not exist. Skipping...\")\n",
|
|
" continue\n",
|
|
" \n",
|
|
" sdl_class_rdoc_csv = pd.read_csv(sdl_class_rdoc_path, low_memory=False)\n",
|
|
" test_csv = pd.read_csv(test_path, low_memory=False)\n",
|
|
" \n",
|
|
" sdl_class_rdoc_csv['tag_description'] = sdl_class_rdoc_csv['tag_description'].fillna('')\n",
|
|
" test_csv['tag_description'] = test_csv['tag_description'].fillna('')\n",
|
|
" \n",
|
|
" test_csv['c_thing'] = ''\n",
|
|
" test_csv['c_property'] = ''\n",
|
|
" test_csv['c_score'] = ''\n",
|
|
" test_csv['c_duplicate'] = 0\n",
|
|
" \n",
|
|
" combined_tag_descriptions = sdl_class_rdoc_csv['tag_description'].tolist() + test_csv['tag_description'].tolist()\n",
|
|
" \n",
|
|
" vectorizer = TfidfVectorizer(\n",
|
|
" use_idf=True, \n",
|
|
" token_pattern=r'\\S+',\n",
|
|
" ngram_range=(1, 1),\n",
|
|
" )\n",
|
|
" \n",
|
|
" vectorizer.fit(combined_tag_descriptions)\n",
|
|
" \n",
|
|
" sdl_class_rdoc_tfidf_matrix = vectorizer.transform(sdl_class_rdoc_csv['tag_description'])\n",
|
|
" test_tfidf_matrix = vectorizer.transform(test_csv['tag_description'])\n",
|
|
" \n",
|
|
" distance_matrix = pairwise_distances(test_tfidf_matrix, sdl_class_rdoc_tfidf_matrix, metric='cosine')\n",
|
|
" \n",
|
|
" most_similar_indices = distance_matrix.argmin(axis=1)\n",
|
|
" most_similar_scores = 1 - distance_matrix.min(axis=1)\n",
|
|
" \n",
|
|
" test_csv['c_thing'] = sdl_class_rdoc_csv.iloc[most_similar_indices]['thing'].values\n",
|
|
" test_csv['c_property'] = sdl_class_rdoc_csv.iloc[most_similar_indices]['property'].values\n",
|
|
" test_csv['c_score'] = most_similar_scores\n",
|
|
" \n",
|
|
" test_csv['cthing_correct'] = test_csv['thing'] == test_csv['c_thing']\n",
|
|
" test_csv['cproperty_correct'] = test_csv['property'] == test_csv['c_property']\n",
|
|
" test_csv['ctp_correct'] = test_csv['cthing_correct'] & test_csv['cproperty_correct']\n",
|
|
" \n",
|
|
" mdm_true_count = len(test_csv[test_csv['MDM'] == True])\n",
|
|
" accuracy = (test_csv['ctp_correct'].sum() / mdm_true_count) * 100\n",
|
|
" accuracies.append(accuracy)\n",
|
|
" \n",
|
|
" print(f\"Accuracy (MDM=True) for Group {group_number}: {accuracy:.2f}%\")\n",
|
|
" \n",
|
|
" output_path = f'0.class_document/{group_number}/test_p_c.csv'\n",
|
|
" test_csv.to_csv(output_path, index=False, encoding='utf-8-sig')\n",
|
|
" \n",
|
|
" false_positive_rows = test_csv[(test_csv['MDM'] == True) & (test_csv['ctp_correct'] == False)]\n",
|
|
" \n",
|
|
" fp_output_path = f'0.class_document/{group_number}/fp_class.csv'\n",
|
|
" false_positive_rows.to_csv(fp_output_path, index=False, encoding='utf-8-sig')\n",
|
|
"\n",
|
|
"average_accuracy = sum(accuracies) / len(accuracies)\n",
|
|
"print(f\"Average Accuracy (MDM=True) across all groups: {average_accuracy:.2f}%\")\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "torch",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.14"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|