# %% import pandas as pd from utils import Retriever, cosine_similarity_chunked import os import glob import numpy as np def analysis_for_fold(fold): file_object = open(f'exports/output_{fold}.txt', 'w') # %% data_path = f'../../train/mapping_pattern/mapping_prediction/exports/result_group_{fold}.csv' df = pd.read_csv(data_path, skipinitialspace=True) # %% # subset to mdm df = df[df['MDM']] thing_condition = df['p_thing'] == df['thing_pattern'] error_thing_df = df[~thing_condition][['tag_description', 'thing_pattern','p_thing']] property_condition = df['p_property'] == df['property_pattern'] error_property_df = df[~property_condition][['tag_description', 'property_pattern','p_property']] correct_df = df[thing_condition & property_condition][['tag_description', 'property_pattern', 'p_property']] test_df = df # %% print("Number of errors related to 'thing'", len(error_thing_df), file=file_object) print("Number of errors related to 'property'", len(error_property_df), file=file_object) # %% # thing_df.to_html('thing_errors.html') # property_df.to_html('property_errors.html') ########################################## # what we need now is understand why the model is making these mispredictions # import train data and test data # %% class Embedder(): input_df: pd.DataFrame fold: int def __init__(self, input_df): self.input_df = input_df def make_embedding(self, checkpoint_path): def generate_input_list(df): input_list = [] for _, row in df.iterrows(): desc = f"{row['tag_description']}" unit = f"{row['unit']}" element = f"{desc}{unit}" input_list.append(element) return input_list # prepare reference embed train_data = list(generate_input_list(self.input_df)) # Define the directory and the pattern retriever_train = Retriever(train_data, checkpoint_path) retriever_train.make_embedding(batch_size=64) return retriever_train.embeddings.to('cpu') # %% data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv" train_df = pd.read_csv(data_path, skipinitialspace=True) checkpoint_directory = "../../train/classification_bert" directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}') # Use glob to find matching paths # path is usually checkpoint_fold_1/checkpoint- # we are guaranteed to save only 1 checkpoint from training pattern = 'checkpoint-*' checkpoint_path = glob.glob(os.path.join(directory, pattern))[0] train_embedder = Embedder(input_df=train_df) train_embeds = train_embedder.make_embedding(checkpoint_path) test_embedder = Embedder(input_df=test_df) test_embeds = test_embedder.make_embedding(checkpoint_path) # %% # test embeds are inputs since we are looking back at train data cos_sim_matrix = cosine_similarity_chunked(test_embeds, train_embeds, chunk_size=8).cpu().numpy() # %% # the following function takes in a full cos_sim_matrix # condition_source: boolean selectors of the source embedding # condition_target: boolean selectors of the target embedding def find_closest(cos_sim_matrix, condition_source, condition_target): # subset_matrix = cos_sim_matrix[condition_source] # except we are subsetting 2D matrix (row, column) subset_matrix = cos_sim_matrix[np.ix_(condition_source, condition_target)] # we select top k here # Get the indices of the top 5 maximum values along axis 1 top_k = 3 top_k_indices = np.argsort(subset_matrix, axis=1)[:, -top_k:] # Get indices of top k values # note that top_k_indices is a nested list because of the 2d nature of the matrix # the result is flipped top_k_indices[0] = top_k_indices[0][::-1] # Get the values of the top 5 maximum scores top_k_values = np.take_along_axis(subset_matrix, top_k_indices, axis=1) return top_k_indices, top_k_values #################################################### # special find-back code # %% def find_back_element_with_print(select_idx): condition_source = test_df['tag_description'] == test_df[test_df.index == select_idx]['tag_description'].tolist()[0] condition_target = np.ones(train_embeds.shape[0], dtype=bool) top_k_indices, top_k_values = find_closest( cos_sim_matrix=cos_sim_matrix, condition_source=condition_source, condition_target=condition_target) training_data_pattern_list = train_df.iloc[top_k_indices[0]]['pattern'].to_list() training_desc_list = train_df.iloc[top_k_indices[0]]['tag_description'].to_list() test_data_pattern_list = test_df[test_df.index == select_idx]['pattern'].to_list() test_desc_list = test_df[test_df.index == select_idx]['tag_description'].to_list() test_ship_id = test_df[test_df.index == select_idx]['ships_idx'].to_list()[0] predicted_test_data = test_df[test_df.index == select_idx]['p_thing'] + ' ' + test_df[test_df.index == select_idx]['p_property'] predicted_test_data = predicted_test_data.to_list()[0] print("*" * 20, file=file_object) print("idx:", select_idx, file=file_object) print("train desc", training_desc_list, file=file_object) print("train thing+property", training_data_pattern_list, file=file_object) print("test desc", test_desc_list, file=file_object) print("test thing+property", test_data_pattern_list, file=file_object) print("predicted thing+property", predicted_test_data, file=file_object) print("ships idx", test_ship_id, file=file_object) print("score:", top_k_values[0], file=file_object) test_pattern = test_data_pattern_list[0] find_back_list = [ test_pattern in pattern for pattern in training_data_pattern_list ] if sum(find_back_list) > 0: return True else: return False # %% # for error thing print('\n', file=file_object) print('*' * 80, file=file_object) print('Error analysis for thing errors', file=file_object) pattern_in_train = [] for select_idx in error_thing_df.index: result = find_back_element_with_print(select_idx) print("status:", result, file=file_object) pattern_in_train.append(result) proportion_in_train = sum(pattern_in_train)/len(pattern_in_train) print('\n', file=file_object) print('*' * 80, file=file_object) print("Proportion of entries found in training data", proportion_in_train, file=file_object) # for error property # %% print('\n', file=file_object) print('*' * 80, file=file_object) print('Error analysis for property errors', file=file_object) pattern_in_train = [] for select_idx in error_property_df.index: result = find_back_element_with_print(select_idx) print("status:", result, file=file_object) pattern_in_train.append(result) proportion_in_train = sum(pattern_in_train)/len(pattern_in_train) print('\n', file=file_object) print('*' * 80, file=file_object) print("Proportion of entries found in training data", proportion_in_train, file=file_object) #################################################### # %% # make function to compute similarity of closest retrieved result def compute_similarity(select_idx): condition_source = test_df['tag_description'] == test_df[test_df.index == select_idx]['tag_description'].tolist()[0] condition_target = np.ones(train_embeds.shape[0], dtype=bool) top_k_indices, top_k_values = find_closest( cos_sim_matrix=cos_sim_matrix, condition_source=condition_source, condition_target=condition_target) return np.mean(top_k_values[0]) # %% def print_summary(similarity_scores): # Convert list to numpy array for additional stats np_array = np.array(similarity_scores) # Get stats mean_value = np.mean(np_array) percentiles = np.percentile(np_array, [25, 50, 75]) # 25th, 50th, and 75th percentiles # Display numpy results print("Mean:", mean_value, file=file_object) print("25th, 50th, 75th Percentiles:", percentiles, file=file_object) # %% ########################################## # Analyze the degree of similarity differences between correct and incorrect results print('\n', file=file_object) print("*" * 80, file=file_object) print("This section analyzes the similarity statistics for the error and correct groups", file=file_object) # %% # compute similarity scores for all values in error_thing_df similarity_thing_scores = [] for idx in error_thing_df.index: similarity_thing_scores.append(compute_similarity(idx)) print_summary(similarity_thing_scores) # %% similarity_property_scores = [] for idx in error_property_df.index: similarity_property_scores.append(compute_similarity(idx)) print_summary(similarity_property_scores) # %% similarity_correct_scores = [] for idx in correct_df.index: similarity_correct_scores.append(compute_similarity(idx)) print_summary(similarity_correct_scores) file_object.close() for fold in [1,2,3,4,5]: print(f"running for fold {fold}") analysis_for_fold(fold)