262 lines
8.3 KiB
Python
262 lines
8.3 KiB
Python
|
# %%
|
||
|
|
import pandas as pd
|
||
|
|
import os
|
||
|
|
import glob
|
||
|
|
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix
|
||
|
|
import numpy as np
|
||
|
|
from utils import BertEmbedder, cosine_similarity_chunked
|
||
|
|
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# directory for checkpoints
|
||
|
|
checkpoint_directory = '../../train/mapping_pattern'
|
||
|
|
|
||
|
|
fold = 5
|
||
|
|
# import test data
|
||
|
|
data_path = f"../../train/mapping_pattern/mapping_prediction/exports/result_group_{fold}.csv"
|
||
|
|
df = pd.read_csv(data_path, skipinitialspace=True)
|
||
|
|
|
||
|
|
# get target data
|
||
|
|
data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
|
||
|
|
train_df = pd.read_csv(data_path, skipinitialspace=True)
|
||
|
|
# processing to help with selection later
|
||
|
|
|
||
|
|
# %%
|
||
|
|
df['p_pattern'] = df['p_thing'] + " " + df['p_property']
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# obtain the full mdm_list
|
||
|
|
data_path = '../../data_import/exports/data_mapping_mdm.csv'
|
||
|
|
full_df = pd.read_csv(data_path, skipinitialspace=True)
|
||
|
|
full_mdm_pattern_list = sorted(list((set(full_df['pattern']))))
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# we have to split into per-ship analysis
|
||
|
|
ships_list = sorted(list(set(df['ships_idx'])))
|
||
|
|
# %%
|
||
|
|
# for ship_idx in ships_list:
|
||
|
|
ship_idx = 1009 # choose an example ship
|
||
|
|
ship_df = df[df['ships_idx'] == ship_idx].reset_index(drop=True)
|
||
|
|
|
||
|
|
class Embedder():
|
||
|
|
input_df: pd.DataFrame
|
||
|
|
fold: int
|
||
|
|
|
||
|
|
def __init__(self, input_df):
|
||
|
|
self.input_df = input_df
|
||
|
|
|
||
|
|
|
||
|
|
def make_embedding(self, checkpoint_path):
|
||
|
|
|
||
|
|
def generate_input_list(df):
|
||
|
|
input_list = []
|
||
|
|
for _, row in df.iterrows():
|
||
|
|
desc = f"{row['tag_description']}"
|
||
|
|
element = f"{desc}"
|
||
|
|
input_list.append(element)
|
||
|
|
return input_list
|
||
|
|
|
||
|
|
# prepare reference embed
|
||
|
|
train_data = list(generate_input_list(self.input_df))
|
||
|
|
# Define the directory and the pattern
|
||
|
|
embedder = BertEmbedder(train_data, checkpoint_path)
|
||
|
|
embedder.make_embedding(batch_size=64)
|
||
|
|
return embedder.embeddings.to('cpu')
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
# %%
|
||
|
|
data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
|
||
|
|
train_df = pd.read_csv(data_path, skipinitialspace=True)
|
||
|
|
|
||
|
|
checkpoint_directory = "../../train/classification_bert"
|
||
|
|
directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}')
|
||
|
|
# Use glob to find matching paths
|
||
|
|
# path is usually checkpoint_fold_1/checkpoint-<step number>
|
||
|
|
# we are guaranteed to save only 1 checkpoint from training
|
||
|
|
pattern = 'checkpoint-*'
|
||
|
|
checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
|
||
|
|
|
||
|
|
train_embedder = Embedder(input_df=train_df)
|
||
|
|
train_embeds = train_embedder.make_embedding(checkpoint_path)
|
||
|
|
|
||
|
|
test_embedder = Embedder(input_df=ship_df)
|
||
|
|
test_embeds = test_embedder.make_embedding(checkpoint_path)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# test embeds are inputs since we are looking back at train data
|
||
|
|
cos_sim_matrix = cosine_similarity_chunked(test_embeds, train_embeds, chunk_size=8).cpu().numpy()
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
# The general idea:
|
||
|
|
# step 1: keep only pattern generations that belong to mdm list
|
||
|
|
# -> this removes totally wrong datasets that mapped to totally wrong things
|
||
|
|
# step 2: loop through the mdm list and isolate data in both train and test that
|
||
|
|
# belong to the same pattern class
|
||
|
|
# -> this is more tricky, because we have non-mdm mapping to correct classes
|
||
|
|
# -> so we have to find which candidate is most similar to the training data
|
||
|
|
|
||
|
|
# it is very tricky to keep track of classification across multiple stages so we
|
||
|
|
# will use a boolean answer list
|
||
|
|
|
||
|
|
# %%
|
||
|
|
answer_list = np.ones(len(ship_df), dtype=bool)
|
||
|
|
|
||
|
|
##########################################
|
||
|
|
# %%
|
||
|
|
# STEP 1
|
||
|
|
# we want to loop through the the ship_df and find which ones match our full_mdm_list
|
||
|
|
pattern_match_mask = ship_df['p_pattern'].apply(lambda x: x in full_mdm_pattern_list).to_numpy()
|
||
|
|
# we assign only those that are False to our answer list
|
||
|
|
# right now the 2 arrays are basically equal
|
||
|
|
answer_list[~pattern_match_mask] = False
|
||
|
|
|
||
|
|
# %% TEMP
|
||
|
|
print('proportion belonging to mdm classes', sum(pattern_match_mask)/len(pattern_match_mask))
|
||
|
|
|
||
|
|
# %% TEMP
|
||
|
|
y_true = ship_df['MDM'].to_list()
|
||
|
|
y_pred = pattern_match_mask
|
||
|
|
|
||
|
|
# Compute metrics
|
||
|
|
accuracy = accuracy_score(y_true, y_pred)
|
||
|
|
print(f'Accuracy: {accuracy:.5f}')
|
||
|
|
|
||
|
|
# we can see that the accuracy is not good
|
||
|
|
# %%
|
||
|
|
#########################################
|
||
|
|
# STEP 2
|
||
|
|
# we want to go through each mdm class label
|
||
|
|
# but we do not want to make subsets of dataframes
|
||
|
|
# we will make heavy use of boolean masks
|
||
|
|
|
||
|
|
# we want to identify per-ship mdm classes
|
||
|
|
ship_mdm_classes = sorted(set(ship_df['p_pattern'][pattern_match_mask].to_list()))
|
||
|
|
|
||
|
|
# %%
|
||
|
|
len(ship_mdm_classes)
|
||
|
|
|
||
|
|
# %%
|
||
|
|
for idx,select_class in enumerate(ship_mdm_classes):
|
||
|
|
print(idx, len(ship_df[ship_df['p_pattern'] == select_class]))
|
||
|
|
|
||
|
|
# %%
|
||
|
|
select_class = ship_mdm_classes[22]
|
||
|
|
sample_df = ship_df[ship_df['p_pattern'] == select_class]
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# we need to set all idx of chosen entries as False in answer_list
|
||
|
|
selected_idx_list = sample_df.index.to_list()
|
||
|
|
answer_list[selected_idx_list] = False
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# because we have variants of a tag_description, we cannot choose 1 from the
|
||
|
|
# given candidates we have to first group the candidates, and then choose which
|
||
|
|
# group is most similar
|
||
|
|
|
||
|
|
# %%
|
||
|
|
from fuzzywuzzy import fuzz
|
||
|
|
|
||
|
|
# the purpose of this function is to group the strings that are similar to each other
|
||
|
|
# we need to form related groups of inputs
|
||
|
|
def group_similar_strings(obj_list, threshold=80):
|
||
|
|
groups = []
|
||
|
|
processed_strings = set() # To keep track of already grouped strings
|
||
|
|
|
||
|
|
for obj in obj_list:
|
||
|
|
# tuple is (idx, string)
|
||
|
|
if obj in processed_strings:
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Find all strings similar to the current string above the threshold
|
||
|
|
similar_strings = [s for s in obj_list if s[1] != obj[1] and fuzz.ratio(obj[1], s[1]) >= threshold]
|
||
|
|
|
||
|
|
# Add the original string to the similar group
|
||
|
|
similar_group = [obj] + similar_strings
|
||
|
|
|
||
|
|
# Mark all similar strings as processed
|
||
|
|
processed_strings.update(similar_group)
|
||
|
|
|
||
|
|
# Add the group to the list of groups
|
||
|
|
groups.append(similar_group)
|
||
|
|
|
||
|
|
return groups
|
||
|
|
|
||
|
|
# Example usage
|
||
|
|
string_list = sample_df['tag_description'].to_list()
|
||
|
|
index_list = sample_df.index.to_list()
|
||
|
|
obj_list = list(zip(index_list, string_list))
|
||
|
|
groups = group_similar_strings(obj_list, threshold=90)
|
||
|
|
print(groups)
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# this function takes in groups of related terms and create candidate entries
|
||
|
|
def make_candidates(groups):
|
||
|
|
candidates = []
|
||
|
|
for group in groups:
|
||
|
|
first_tuple = group[0]
|
||
|
|
# string_of_tuple = first_tuple[1]
|
||
|
|
id_of_tuple = first_tuple[0]
|
||
|
|
candidates.append(id_of_tuple)
|
||
|
|
return candidates
|
||
|
|
|
||
|
|
# %%
|
||
|
|
test_candidates = make_candidates(groups)
|
||
|
|
test_candidates_mask = np.zeros(len(ship_df), dtype=bool)
|
||
|
|
test_candidates_mask[test_candidates] = True
|
||
|
|
|
||
|
|
# %%
|
||
|
|
train_candidates_mask = (train_df['pattern'] == select_class).to_numpy()
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# we need to make the cos_sim_matrix
|
||
|
|
# for that, we need to generate the embeddings of the ship_df (test embedding)
|
||
|
|
# and the train_df (train embeddin)
|
||
|
|
|
||
|
|
# we then use the selection function using the given mask to choose the most
|
||
|
|
# appropriate candidate
|
||
|
|
|
||
|
|
# the selection function takes in the full cos_sim_matrix then subsets the
|
||
|
|
# matrix according to the test_candidates_mask and train_candidates_mask that we
|
||
|
|
# give it
|
||
|
|
# it returns the most likely source candidate index and score among the source
|
||
|
|
# candidate list - aka it returns a local idx
|
||
|
|
def selection(cos_sim_matrix, source_mask, target_mask):
|
||
|
|
# subset_matrix = cos_sim_matrix[condition_source]
|
||
|
|
# except we are subsetting 2D matrix (row, column)
|
||
|
|
subset_matrix = cos_sim_matrix[np.ix_(source_mask, target_mask)]
|
||
|
|
# we select top-k here
|
||
|
|
# Get the indices of the top-k maximum values along axis 1
|
||
|
|
top_k = 1
|
||
|
|
top_k_indices = np.argsort(subset_matrix, axis=1)[:, -top_k:] # Get indices of top k values
|
||
|
|
|
||
|
|
# Get the values of the top 5 maximum scores
|
||
|
|
top_k_values = np.take_along_axis(subset_matrix, top_k_indices, axis=1)
|
||
|
|
|
||
|
|
# Calculate the average of the top-k scores along axis 1
|
||
|
|
y_scores = np.mean(top_k_values, axis=1)
|
||
|
|
max_idx = np.argmax(y_scores)
|
||
|
|
max_score = y_scores[max_idx]
|
||
|
|
|
||
|
|
return max_idx, max_score
|
||
|
|
|
||
|
|
|
||
|
|
# %%
|
||
|
|
max_idx, max_score = selection(cos_sim_matrix, test_candidates_mask, train_candidates_mask)
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# after obtaining best group, we set all candidates of the group as True
|
||
|
|
chosen_group = groups[max_idx]
|
||
|
|
chosen_idx = [tuple[0] for tuple in chosen_group]
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# before doing this, we have to use the max_score and evaluate if its close enough
|
||
|
|
THRESHOLD = 0.8
|
||
|
|
if max_score > THRESHOLD:
|
||
|
|
answer_list[chosen_idx] = True
|
||
|
|
|
||
|
|
# %%
|