# %% import pandas as pd import os import glob from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix import numpy as np from utils import BertEmbedder, cosine_similarity_chunked from fuzzywuzzy import fuzz ################## # global parameters DIAGNOSTIC = True THRESHOLD = 0.90 FUZZY_SIM_THRESHOLD=90 checkpoint_directory = "../../train/classification_bert" ################### # %% # helper functions class Embedder(): input_df: pd.DataFrame fold: int def __init__(self, input_df): self.input_df = input_df def make_embedding(self, checkpoint_path): def generate_input_list(df): input_list = [] for _, row in df.iterrows(): desc = f"{row['tag_description']}" element = f"{desc}" input_list.append(element) return input_list # prepare reference embed train_data = list(generate_input_list(self.input_df)) # Define the directory and the pattern embedder = BertEmbedder(train_data, checkpoint_path) embedder.make_embedding(batch_size=64) return embedder.embeddings.to('cpu') # the purpose of this function is to group the strings that are similar to each other # we need to form related groups of inputs def group_similar_strings(obj_list, threshold=80): groups = [] processed_strings = set() # To keep track of already grouped strings for obj in obj_list: # tuple is (idx, string) if obj in processed_strings: continue # Find all strings similar to the current string above the threshold similar_strings = [s for s in obj_list if s[1] != obj[1] and fuzz.ratio(obj[1], s[1]) >= threshold] # Add the original string to the similar group similar_group = [obj] + similar_strings # Mark all similar strings as processed processed_strings.update(similar_group) # Add the group to the list of groups groups.append(similar_group) return groups # this function takes in groups of related terms and create candidate entries def make_candidates(groups): candidates = [] for group in groups: first_tuple = group[0] # string_of_tuple = first_tuple[1] id_of_tuple = first_tuple[0] candidates.append(id_of_tuple) return candidates # the selection function takes in the full cos_sim_matrix then subsets the # matrix according to the test_candidates_mask and train_candidates_mask that we # give it # it returns the most likely source candidate index and score among the source # candidate list - aka it returns a local idx def selection(cos_sim_matrix, source_mask, target_mask, file_object=None): # subset_matrix = cos_sim_matrix[condition_source] # except we are subsetting 2D matrix (row, column) subset_matrix = cos_sim_matrix[np.ix_(source_mask, target_mask)] # we select top-k here # Get the indices of the top-k maximum values along axis 1 top_k = 1 top_k_indices = np.argsort(subset_matrix, axis=1)[:, -top_k:] # Get indices of top k values # Get the values of the top 5 maximum scores top_k_values = np.take_along_axis(subset_matrix, top_k_indices, axis=1) # Calculate the average of the top-k scores along axis 1 y_scores = np.mean(top_k_values, axis=1) max_idx = np.argmax(y_scores) max_score = y_scores[max_idx] if DIAGNOSTIC and (file_object is not None): print('all scores:', file=file_object) print(y_scores, file=file_object) return max_idx, max_score #################### # global level # %% # obtain the full mdm_list data_path = '../../data_import/exports/data_mapping_mdm.csv' full_df = pd.read_csv(data_path, skipinitialspace=True) full_mdm_pattern_list = sorted(list((set(full_df['pattern'])))) ##################### # fold level def run_selection(fold): file_object = open(f'exports/output_{fold}.txt', 'w') # set the fold # import test data data_path = f"../../train/mapping_pattern/mapping_prediction/exports/result_group_{fold}.csv" df = pd.read_csv(data_path, skipinitialspace=True) df['p_pattern'] = df['p_thing'] + " " + df['p_property'] # get target data data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv" train_df = pd.read_csv(data_path, skipinitialspace=True) # generate your embeddings # checkpoint_directory defined at global level directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}') # Use glob to find matching paths # path is usually checkpoint_fold_1/checkpoint- # we are guaranteed to save only 1 checkpoint from training pattern = 'checkpoint-*' checkpoint_path = glob.glob(os.path.join(directory, pattern))[0] # we can generate the train embeddings once and re-use for every ship train_embedder = Embedder(input_df=train_df) train_embeds = train_embedder.make_embedding(checkpoint_path) # create global_answer array # the purpose of this array is to track the classification state at the global # level global_answer = np.zeros(len(df), dtype=bool) global_sim = np.zeros(len(df)) ############################# # ship level # we have to split into per-ship analysis ships_list = sorted(list(set(df['ships_idx']))) print(ships_list) for ship_idx in ships_list: # ship_idx = 1001 # choose an example ship print(ship_idx, file=file_object) # print selected ship ship_df = df[df['ships_idx'] == ship_idx] # required to map local ship_answer array to global_answer array map_local_index_to_global_index = ship_df.index.to_numpy() ship_df = df[df['ships_idx'] == ship_idx].reset_index(drop=True) # generate new embeddings for each ship test_embedder = Embedder(input_df=ship_df) test_embeds = test_embedder.make_embedding(checkpoint_path) # generate the cosine sim matrix cos_sim_matrix = cosine_similarity_chunked(test_embeds, train_embeds, chunk_size=8).cpu().numpy() ############################## # selection level # The general idea: # step 1: keep only pattern generations that belong to mdm list # -> this removes totally wrong datasets that mapped to totally wrong things # step 2: loop through the mdm list and isolate data in both train and test that # belong to the same pattern class # -> this is more tricky, because we have non-mdm mapping to correct classes # -> so we have to find which candidate is most similar to the training data # it is very tricky to keep track of classification across multiple stages so we # will use a boolean answer list # initialize the local answer list ship_answer_list = np.ones(len(ship_df), dtype=bool) ship_answer_sim = np.ones(len(ship_df)) ########### # STEP 1 # we want to loop through the generated class labels and find which ones match # our pattern list pattern_match_mask = ship_df['p_pattern'].apply(lambda x: x in full_mdm_pattern_list).to_numpy() # we assign only those that are False to our answer list # right now the 2 arrays are basically equal ship_answer_list[~pattern_match_mask] = False ########### # STEP 2 # we now go through each class found in our generated set # we want to identify per-ship mdm classes ship_predicted_classes = sorted(set(ship_df['p_pattern'][pattern_match_mask].to_list())) # this function performs the selection given a class # it takes in the cos_sim_matrix # it returns the selection by mutating the answer_list # it sets all relevant idxs to False initially, then sets the selected values to True def selection_for_class(select_class, cos_sim_matrix, answer_list, score_list): # separate the global variable from function variable answer_list = answer_list.copy() score_list = score_list.copy() sample_df = ship_df[ship_df['p_pattern'] == select_class] # we need to set all idx of chosen entries as False in answer_list selected_idx_list = sample_df.index.to_list() answer_list[selected_idx_list] = False # basic assumption check # group related inputs by description similarity string_list = sample_df['tag_description'].to_list() index_list = sample_df.index.to_list() obj_list = list(zip(index_list, string_list)) # groups is a list of list, where each list is composed of a # (idx, string) tuple groups = group_similar_strings(obj_list, threshold=FUZZY_SIM_THRESHOLD) if DIAGNOSTIC: print('*' * 10, file=file_object) print(select_class, file=file_object) print('candidate groups', file=file_object) print(groups, file=file_object) # generate the masking arrays for both test and train embeddings # we select a tuple from each group, and use that as a candidate for selection test_candidates = make_candidates(groups) test_candidates_mask = np.zeros(len(ship_df), dtype=bool) test_candidates_mask[test_candidates] = True # we make candidates to compare against in the data sharing the same class train_candidates_mask = (train_df['pattern'] == select_class).to_numpy() # perform selection # it returns the group index that is most likely max_idx, max_score = selection(cos_sim_matrix, test_candidates_mask, train_candidates_mask, file_object) # consolidate all idx's in the same group chosen_group = groups[max_idx] chosen_idx_list = [tuple[0] for tuple in chosen_group] if DIAGNOSTIC: print('chosen group', file=file_object) print(chosen_group, file=file_object) # before doing this, we have to use the max_score and evaluate if its close enough if max_score > THRESHOLD: answer_list[chosen_idx_list] = True if DIAGNOSTIC: print('max score', file=file_object) print(max_score, file=file_object) print('accepted', file=file_object) else: if DIAGNOSTIC: print('max score', file=file_object) print(max_score, file=file_object) print('rejected', file=file_object) # for analysis test_candidates_mask = np.ones(len(ship_df), dtype=bool) _, every_score = selection(cos_sim_matrix, test_candidates_mask, train_candidates_mask, None) # write the score for every idx of this class score_list[selected_idx_list] = every_score return answer_list, score_list # we choose one mdm class for select_class in ship_predicted_classes: ship_answer_list, ship_answer_sim = selection_for_class(select_class, cos_sim_matrix, ship_answer_list, ship_answer_sim) # we want to write back to global_answer # first we convert local indices to global indices local_indices = np.where(ship_answer_list)[0] global_indices = map_local_index_to_global_index[local_indices] global_answer[global_indices] = True # similarity score global_sim[map_local_index_to_global_index] = ship_answer_sim if DIAGNOSTIC: # evaluation at per-ship level y_true = ship_df['MDM'].to_list() y_pred = ship_answer_list # Compute metrics accuracy = accuracy_score(y_true, y_pred) f1 = f1_score(y_true, y_pred, average='macro') precision = precision_score(y_true, y_pred, average='macro') recall = recall_score(y_true, y_pred, average='macro') # Print the results print(f'Accuracy: {accuracy:.5f}', file=file_object) print(f'F1 Score: {f1:.5f}', file=file_object) print(f'Precision: {precision:.5f}', file=file_object) print(f'Recall: {recall:.5f}', file=file_object) y_true = df['MDM'].to_list() y_pred = global_answer # Compute metrics accuracy = accuracy_score(y_true, y_pred) f1 = f1_score(y_true, y_pred, average='macro') precision = precision_score(y_true, y_pred, average='macro') recall = recall_score(y_true, y_pred, average='macro') # Print the results print(f'Accuracy: {accuracy:.5f}') print(f'F1 Score: {f1:.5f}') print(f'Precision: {precision:.5f}') print(f'Recall: {recall:.5f}') file_object.close() return global_answer, global_sim # %% for fold in [1]: print(f'Perform selection for fold {fold}') global_answer, global_sim = run_selection(fold) # %% data_path = f"../../train/mapping_pattern/mapping_prediction/exports/result_group_{fold}.csv" df = pd.read_csv(data_path, skipinitialspace=True) df['p_pattern'] = df['p_thing'] + " " + df['p_property'] df['score'] = global_sim # %% # %% def print_summary(similarity_scores): # Convert list to numpy array for additional stats np_array = np.array(similarity_scores) # Get stats mean_value = np.mean(np_array) percentiles = np.percentile(np_array, [25, 50, 75]) # 25th, 50th, and 75th percentiles # Display numpy results print("Mean:", mean_value) print("25th, 50th, 75th Percentiles:", percentiles) # %% # analysis of non-mdm in predicted df_selected = df[global_answer] df_selected[~df_selected['MDM']] in_scores = df_selected[df_selected['MDM']]['score'].to_numpy() print_summary(in_scores) # %% # analysis of mdm in non-predicted df_selected = df[~global_answer] # df_selected df_selected[df_selected['MDM']] # %% out_scores = df_selected[df_selected['MDM']]['score'].to_numpy() print_summary(out_scores) # %% import matplotlib.pyplot as plt # Sample data list1 = in_scores list2 = out_scores # Plot histograms bins = 20 plt.hist(list1, bins=bins, alpha=0.5, label='List 1', density=False) plt.hist(list2, bins=bins, alpha=0.5, label='List 2', density=False) # Labels and legend plt.xlabel('Value') plt.ylabel('Frequency') plt.legend(loc='upper right') plt.title('Histograms of Three Lists') # Show plot plt.show() # %%