hipom_data_mapping/translation/t5/3.produce_test_predictions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Goal: end to end inference and evaluation\n",
    "\n",
    "given a csv, make predictions and evaluate predictions, then return results in a csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12938it [00:07, 1674.63it/s]                  \n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import json\n",
    "from transformers.pipelines.pt_utils import KeyDataset\n",
    "from transformers import pipeline\n",
    "from tqdm import tqdm\n",
    "from transformers import AutoTokenizer\n",
    "from datasets import Dataset\n",
    "\n",
    "with open(\"mode.json\", \"r\") as json_file:\n",
    "    mode_dict = json.load(json_file)\n",
    "\n",
    "mode = mode_dict.get(\"mode\", \"none\")\n",
    "model_name = mode_dict.get(\"model\", \"none\")\n",
    "train_epochs = mode_dict.get(\"train_epochs\", \"none\")\n",
    "fold_group = mode_dict.get(\"fold_group\", \"none\")\n",
    "\n",
    "base_dir = f\"train_{fold_group}_{model_name}_{mode}_{train_epochs}\"\n",
    "checkpoints = [d for d in os.listdir(base_dir) if d.startswith(\"checkpoint-\")]\n",
    "\n",
    "model_checkpoint = os.path.join(base_dir, checkpoints[0]) if checkpoints else None\n",
    "\n",
    "data_path = f\"../../data_preprocess/dataset/{fold_group}/test.csv\"\n",
    "\n",
    "try:\n",
    "    df = pd.read_csv(data_path)\n",
    "except UnicodeDecodeError:\n",
    "    df = pd.read_csv(data_path, encoding='ISO-8859-1')\n",
    "\n",
    "df = df.dropna(subset=['tag_description']).reset_index(drop=True)\n",
    "df_org = df.copy()\n",
    "df[['thing', 'property', 'tag_description', 'min', 'max', 'MDM', 'pattern']] = df[['thing', 'property', 'tag_description', 'min', 'max', 'MDM', 'pattern']].astype(\"string\")\n",
    "\n",
    "def process_df(df, mode='only_td'):\n",
    "    output_list = []\n",
    "    for _, row in df.iterrows():\n",
    "        try:\n",
    "            if mode == 'only_td':\n",
    "                input_str = f\"<TD_START>{str(row['tag_description'])}<TD_END>\"\n",
    "            elif mode == 'tn_td':\n",
    "                input_str = f\"<TN_START>{str(row['tag_name'])}<TN_END><TD_START>{str(row['tag_description'])}<TD_END>\"\n",
    "            elif mode == 'tn_td_min_max':\n",
    "                input_str = f\"<TN_START>{str(row['tag_name'])}<TN_END><TD_START>{str(row['tag_description'])}<TD_END><MIN_START>{row['min']}<MIN_END><MAX_START>{row['max']}<MAX_END>\"\n",
    "            elif mode == 'td_min_max':\n",
    "                input_str = f\"<TD_START>{str(row['tag_description'])}<TD_END><MIN_START>{row['min']}<MIN_END><MAX_START>{row['max']}<MAX_END>\"\n",
    "            elif mode == 'td_unit':\n",
    "                input_str = f\"<TD_START>{str(row['tag_description'])}<TD_END><UNIT_START>{str(row['unit'])}<UNIT_END>\"\n",
    "            elif mode == 'tn_td_unit':\n",
    "                input_str = f\"<TN_START>{str(row['tag_name'])}<TN_END><TD_START>{str(row['tag_description'])}<TD_END><UNIT_START>{str(row['unit'])}<UNIT_END>\"\n",
    "            elif mode == 'td_min_max_unit':\n",
    "                input_str = f\"<TD_START>{str(row['tag_description'])}<TD_END><MIN_START>{row['min']}<MIN_END><MAX_START>{row['max']}<MAX_END><UNIT_START>{str(row['unit'])}<UNIT_END>\"\n",
    "            else:\n",
    "                raise ValueError(\"Invalid mode specified\")\n",
    "\n",
    "            output_list.append({\n",
    "                'translation': {\n",
    "                    'ships_idx': row['ships_idx'],\n",
    "                    'input': input_str,\n",
    "                    'thing_property': f\"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>\",\n",
    "                    'answer_thing': row['thing'],\n",
    "                    'answer_property': row['property'],\n",
    "                    'MDM': row['MDM'],\n",
    "                }\n",
    "            })\n",
    "        except Exception as e:\n",
    "            print(f\"Error processing row: {e}\")\n",
    "    return output_list\n",
    "\n",
    "processed_data = process_df(df, mode=mode)\n",
    "test_dataset = Dataset.from_list(processed_data)\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name, return_tensors=\"pt\")\n",
    "additional_special_tokens = [\"<THING_START>\", \"<THING_END>\", \"<PROPERTY_START>\", \"<PROPERTY_END>\", \"<TN_START>\", \"<TN_END>\", \"<TD_START>\", \"<TD_END>\", \"<MIN_START>\", \"<MIN_END>\", \"<MAX_START>\", \"<MAX_END>\", \"<UNIT_START>\", \"<UNIT_END>\"]\n",
    "tokenizer.add_special_tokens({\"additional_special_tokens\": additional_special_tokens})\n",
    "\n",
    "pipe = pipeline(\"translation_XX_to_YY\", model=model_checkpoint, tokenizer=tokenizer, return_tensors=True, max_length=128, device=0)\n",
    "\n",
    "thing_start_id = tokenizer.convert_tokens_to_ids(\"<THING_START>\")\n",
    "thing_end_id = tokenizer.convert_tokens_to_ids(\"<THING_END>\")\n",
    "property_start_id = tokenizer.convert_tokens_to_ids(\"<PROPERTY_START>\")\n",
    "property_end_id = tokenizer.convert_tokens_to_ids(\"<PROPERTY_END>\")\n",
    "\n",
    "def extract_seq(tokens, start_value, end_value):\n",
    "    if start_value in tokens and end_value in tokens:\n",
    "        return tokens[tokens.index(start_value)+1:tokens.index(end_value)]\n",
    "    return None\n",
    "\n",
    "def extract_seq_from_output(output):\n",
    "    tokens = output[0][\"translation_token_ids\"].tolist()\n",
    "    p_thing = tokenizer.decode(extract_seq(tokens, thing_start_id, thing_end_id)) if thing_start_id in tokens and thing_end_id in tokens else None\n",
    "    p_property = tokenizer.decode(extract_seq(tokens, property_start_id, property_end_id)) if property_start_id in tokens and property_end_id in tokens else None\n",
    "    return p_thing, p_property\n",
    "\n",
    "p_thing_list = []\n",
    "p_property_list = []\n",
    "\n",
    "for out in tqdm(pipe(KeyDataset(test_dataset[\"translation\"], \"input\"), batch_size=256)):\n",
    "    p_thing, p_property = extract_seq_from_output(out)\n",
    "    p_thing_list.append(p_thing)\n",
    "    p_property_list.append(p_property)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thing prediction accuracy: 0.9793861658268438\n",
      "Correct thing predictions: 2138, Incorrect thing predictions: 10800\n",
      "Property prediction accuracy: 0.9651855245075585\n",
      "Correct property predictions: 2107, Incorrect property predictions: 10831\n",
      "Total accuracy: 0.9496106275767293\n",
      "Correct total predictions: 2073, Incorrect total predictions: 110\n"
     ]
    }
   ],
   "source": [
    "answer_thing = [item['answer_thing'] for item in test_dataset[\"translation\"]]\n",
    "answer_property = [item['answer_property'] for item in test_dataset[\"translation\"]]\n",
    "mdm_list = [item['MDM'] for item in test_dataset[\"translation\"]]\n",
    "\n",
    "mdm_count = sum([1 for mdm in mdm_list if mdm == \"True\"])\n",
    "\n",
    "def correctness_test(input, reference, mdm_list):\n",
    "    assert len(input) == len(reference)\n",
    "    return [input[i] == reference[i] if mdm_list[i] == \"True\" else False for i in range(len(input))]\n",
    "\n",
    "thing_correctness = correctness_test(p_thing_list, answer_thing, mdm_list)\n",
    "property_correctness = correctness_test(p_property_list, answer_property, mdm_list)\n",
    "\n",
    "correctness_mdm = [thing_correctness[i] & property_correctness[i] for i in range(len(mdm_list))]\n",
    "\n",
    "thing_accuracy = sum(thing_correctness) / mdm_count\n",
    "property_accuracy = sum(property_correctness) / mdm_count\n",
    "total_accuracy = sum(correctness_mdm) / mdm_count\n",
    "\n",
    "thing_true_count = thing_correctness.count(True)\n",
    "thing_false_count = thing_correctness.count(False)\n",
    "\n",
    "property_true_count = property_correctness.count(True)\n",
    "property_false_count = property_correctness.count(False)\n",
    "\n",
    "total_true_count = correctness_mdm.count(True)\n",
    "total_false_count = mdm_count - total_true_count\n",
    "\n",
    "print(\"Thing prediction accuracy:\", thing_accuracy)\n",
    "print(f\"Correct thing predictions: {thing_true_count}, Incorrect thing predictions: {thing_false_count}\")\n",
    "print(\"Property prediction accuracy:\", property_accuracy)\n",
    "print(f\"Correct property predictions: {property_true_count}, Incorrect property predictions: {property_false_count}\")\n",
    "print(\"Total accuracy:\", total_accuracy)\n",
    "print(f\"Correct total predictions: {total_true_count}, Incorrect total predictions: {total_false_count}\")\n",
    "\n",
    "df_pred = pd.DataFrame({\n",
    "    'p_thing': p_thing_list,\n",
    "    'p_property': p_property_list,\n",
    "    'p_thing_correct': thing_correctness,\n",
    "    'p_property_correct': property_correctness\n",
    "})\n",
    "\n",
    "with open(\"mode.json\", \"r\") as json_file:\n",
    "    mode_dict = json.load(json_file)\n",
    "\n",
    "mode_dict[\"model\"] = model_name\n",
    "mode_dict[\"train_epochs\"] = train_epochs\n",
    "\n",
    "with open(\"mode.json\", \"w\") as json_file:\n",
    "    json.dump(mode_dict, json_file)\n",
    "\n",
    "if os.path.exists(\"results.json\") and os.path.getsize(\"results.json\") > 0:\n",
    "    with open(\"results.json\", \"r\") as json_file:\n",
    "        try:\n",
    "            results_dict = json.load(json_file)\n",
    "        except json.JSONDecodeError:\n",
    "            results_dict = {}\n",
    "else:\n",
    "    results_dict = {}\n",
    "\n",
    "model_key = model_checkpoint\n",
    "\n",
    "results_dict[model_key] = {\n",
    "    \"thing_accuracy\": thing_accuracy,\n",
    "    \"thing_true\": thing_true_count,\n",
    "    \"thing_false\": thing_false_count,\n",
    "    \"property_accuracy\": property_accuracy,\n",
    "    \"property_true\": property_true_count,\n",
    "    \"property_false\": property_false_count,\n",
    "    \"total_accuracy\": total_accuracy,\n",
    "    \"total_true\": total_true_count,\n",
    "    \"total_false\": total_false_count\n",
    "}\n",
    "\n",
    "with open(\"results.json\", \"w\") as json_file:\n",
    "    json.dump(results_dict, json_file, indent=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updated data saved to ../0.result/5/test_p.csv\n",
      "Updated data saved to 0.dresult/td_unit/google/t5-efficient-tiny/5/test_p.csv\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "df_pred = pd.DataFrame({\n",
    "    'p_thing': p_thing_list,\n",
    "    'p_property': p_property_list,\n",
    "    'p_thing_correct': thing_correctness,\n",
    "    'p_property_correct': property_correctness,\n",
    "})\n",
    "\n",
    "df_org['p_thing'] = df_pred['p_thing']\n",
    "df_org['p_property'] = df_pred['p_property']\n",
    "df_org['p_thing_correct'] = df_pred['p_thing_correct']\n",
    "df_org['p_property_correct'] = df_pred['p_property_correct']\n",
    "df_org['p_correct'] = df_pred['p_thing_correct'] & df_org['p_property_correct']\n",
    "\n",
    "df_master = pd.read_csv('../../data_import/data_model_master_export.csv')\n",
    "\n",
    "df_org['pattern'] = df_org['thing'].str.replace(r'\\d', '#', regex=True) + \" \" + df_org['property'].str.replace(r'\\d', '#', regex=True)\n",
    "df_org['p_pattern'] = df_org['p_thing'].str.replace(r'\\d', '#', regex=True) + \" \" + df_org['p_property'].str.replace(r'\\d', '#', regex=True)\n",
    "df_master['master_pattern'] = df_master['thing'] + \" \" + df_master['property']\n",
    "\n",
    "master_patterns = set(df_master['master_pattern'])\n",
    "df_org['p_MDM'] =  df_org['p_pattern'].apply(lambda x: x in master_patterns)\n",
    "\n",
    "output_path = f\"../0.result/{fold_group}/test_p.csv\"\n",
    "debug_output_path = f\"0.dresult/{mode}/{model_name}/{fold_group}/test_p.csv\"\n",
    "\n",
    "os.makedirs(os.path.dirname(output_path), exist_ok=True)\n",
    "df_org.to_csv(output_path, index=False, encoding='utf-8-sig')\n",
    "\n",
    "os.makedirs(os.path.dirname(debug_output_path), exist_ok=True)\n",
    "df_org.to_csv(debug_output_path, index=False, encoding='utf-8-sig')\n",
    "\n",
    "print(f\"Updated data saved to {output_path}\")\n",
    "print(f\"Updated data saved to {debug_output_path}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}