hipom_data_mapping/analysis/bert/error_analysis.py

# %%
import pandas as pd
from utils import Retriever, cosine_similarity_chunked
import os
import glob
import numpy as np

def analysis_for_fold(fold):
    file_object = open(f'exports/output_{fold}.txt', 'w')

    # %%
    data_path = f'../../train/mapping_pattern/mapping_prediction/exports/result_group_{fold}.csv'
    df = pd.read_csv(data_path, skipinitialspace=True)

    # %%
    # subset to mdm
    df = df[df['MDM']]

    thing_condition = df['p_thing'] == df['thing_pattern']
    error_thing_df = df[~thing_condition][['tag_description', 'thing_pattern','p_thing']]

    property_condition = df['p_property'] == df['property_pattern']
    error_property_df = df[~property_condition][['tag_description', 'property_pattern','p_property']]

    correct_df = df[thing_condition & property_condition][['tag_description', 'property_pattern', 'p_property']]

    test_df = df

    # %%
    print("Number of errors related to 'thing'", len(error_thing_df), file=file_object)
    print("Number of errors related to 'property'", len(error_property_df), file=file_object)

    # %%
    # thing_df.to_html('thing_errors.html')
    # property_df.to_html('property_errors.html')

    ##########################################
    # what we need now is understand why the model is making these mispredictions
    # import train data and test data
    # %%
    class Embedder():
        input_df: pd.DataFrame
        fold: int

        def __init__(self, input_df):
            self.input_df = input_df


        def make_embedding(self, checkpoint_path):

            def generate_input_list(df):
                input_list = []
                for _, row in df.iterrows():
                    desc = f"<DESC>{row['tag_description']}<DESC>"
                    unit = f"<UNIT>{row['unit']}<UNIT>"
                    element = f"{desc}{unit}"
                    input_list.append(element)
                return input_list

            # prepare reference embed
            train_data = list(generate_input_list(self.input_df))
            # Define the directory and the pattern
            retriever_train = Retriever(train_data, checkpoint_path)
            retriever_train.make_embedding(batch_size=64)
            return retriever_train.embeddings.to('cpu')



    # %%
    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
    train_df = pd.read_csv(data_path, skipinitialspace=True)

    checkpoint_directory = "../../train/classification_bert"
    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}')
    # Use glob to find matching paths
    # path is usually checkpoint_fold_1/checkpoint-<step number>
    # we are guaranteed to save only 1 checkpoint from training
    pattern = 'checkpoint-*'
    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]

    train_embedder = Embedder(input_df=train_df)
    train_embeds = train_embedder.make_embedding(checkpoint_path)

    test_embedder = Embedder(input_df=test_df)
    test_embeds = test_embedder.make_embedding(checkpoint_path)



    # %%
    # test embeds are inputs since we are looking back at train data
    cos_sim_matrix = cosine_similarity_chunked(test_embeds, train_embeds, chunk_size=8).cpu().numpy()

    # %%
    # the following function takes in a full cos_sim_matrix
    # condition_source: boolean selectors of the source embedding
    # condition_target: boolean selectors of the target embedding
    def find_closest(cos_sim_matrix, condition_source, condition_target):
        # subset_matrix = cos_sim_matrix[condition_source]
        # except we are subsetting 2D matrix (row, column)
        subset_matrix = cos_sim_matrix[np.ix_(condition_source, condition_target)]
        # we select top k here
        # Get the indices of the top 5 maximum values along axis 1
        top_k = 3
        top_k_indices = np.argsort(subset_matrix, axis=1)[:, -top_k:]  # Get indices of top k values
        # note that top_k_indices is a nested list because of the 2d nature of the matrix
        # the result is flipped
        top_k_indices[0] = top_k_indices[0][::-1]
        
        # Get the values of the top 5 maximum scores
        top_k_values = np.take_along_axis(subset_matrix, top_k_indices, axis=1)
        

        return top_k_indices, top_k_values


    ####################################################
    # special find-back code
    # %%
    def find_back_element_with_print(select_idx):
        condition_source = test_df['tag_description'] == test_df[test_df.index == select_idx]['tag_description'].tolist()[0]
        condition_target = np.ones(train_embeds.shape[0], dtype=bool)

        top_k_indices, top_k_values = find_closest(
            cos_sim_matrix=cos_sim_matrix,
            condition_source=condition_source,
            condition_target=condition_target)

        training_data_pattern_list = train_df.iloc[top_k_indices[0]]['pattern'].to_list()
        training_desc_list = train_df.iloc[top_k_indices[0]]['tag_description'].to_list()

        test_data_pattern_list = test_df[test_df.index == select_idx]['pattern'].to_list()
        test_desc_list = test_df[test_df.index == select_idx]['tag_description'].to_list()
        test_ship_id = test_df[test_df.index == select_idx]['ships_idx'].to_list()[0]
        predicted_test_data = test_df[test_df.index == select_idx]['p_thing'] + ' ' + test_df[test_df.index == select_idx]['p_property']
        predicted_test_data = predicted_test_data.to_list()[0]

        print("*" * 20, file=file_object)
        print("idx:", select_idx, file=file_object)
        print("train desc", training_desc_list, file=file_object)
        print("train thing+property", training_data_pattern_list, file=file_object)
        print("test desc", test_desc_list, file=file_object)
        print("test thing+property", test_data_pattern_list, file=file_object)
        print("predicted thing+property", predicted_test_data, file=file_object)
        print("ships idx", test_ship_id, file=file_object)
        print("score:", top_k_values[0], file=file_object)

        test_pattern = test_data_pattern_list[0]

        find_back_list = [ test_pattern in pattern for pattern in training_data_pattern_list ]

        if sum(find_back_list) > 0:
            return True
        else:
            return False



    # %%
    # for error thing
    print('\n', file=file_object)
    print('*' * 80, file=file_object)
    print('Error analysis for thing errors', file=file_object)
    pattern_in_train = []
    for select_idx in error_thing_df.index:
        result = find_back_element_with_print(select_idx)
        print("status:", result, file=file_object)
        pattern_in_train.append(result)


    proportion_in_train = sum(pattern_in_train)/len(pattern_in_train)

    print('\n', file=file_object)
    print('*' * 80, file=file_object)
    print("Proportion of entries found in training data", proportion_in_train, file=file_object)

    # for error property
    # %%
    print('\n', file=file_object)
    print('*' * 80, file=file_object)
    print('Error analysis for property errors', file=file_object)
    pattern_in_train = []
    for select_idx in error_property_df.index:
        result = find_back_element_with_print(select_idx)
        print("status:", result, file=file_object)
        pattern_in_train.append(result)

    proportion_in_train = sum(pattern_in_train)/len(pattern_in_train)

    print('\n', file=file_object)
    print('*' * 80, file=file_object)
    print("Proportion of entries found in training data", proportion_in_train, file=file_object)

    ####################################################

    # %%
    # make function to compute similarity of closest retrieved result
    def compute_similarity(select_idx):
        condition_source = test_df['tag_description'] == test_df[test_df.index == select_idx]['tag_description'].tolist()[0]
        condition_target = np.ones(train_embeds.shape[0], dtype=bool)
        top_k_indices, top_k_values = find_closest(
            cos_sim_matrix=cos_sim_matrix,
            condition_source=condition_source,
            condition_target=condition_target)

        return np.mean(top_k_values[0])

    # %%
    def print_summary(similarity_scores):
        # Convert list to numpy array for additional stats
        np_array = np.array(similarity_scores)

        # Get stats
        mean_value = np.mean(np_array)
        percentiles = np.percentile(np_array, [25, 50, 75])  # 25th, 50th, and 75th percentiles

        # Display numpy results
        print("Mean:", mean_value, file=file_object)
        print("25th, 50th, 75th Percentiles:", percentiles, file=file_object)


    # %%
    ##########################################
    # Analyze the degree of similarity differences between correct and incorrect results

    print('\n', file=file_object)
    print("*" * 80, file=file_object)
    print("This section analyzes the similarity statistics for the error and correct groups", file=file_object)
    # %%
    # compute similarity scores for all values in error_thing_df
    similarity_thing_scores = []
    for idx in error_thing_df.index:
        similarity_thing_scores.append(compute_similarity(idx))
    print_summary(similarity_thing_scores)


    # %%
    similarity_property_scores = []
    for idx in error_property_df.index:
        similarity_property_scores.append(compute_similarity(idx))
    print_summary(similarity_property_scores)

    # %%
    similarity_correct_scores = []
    for idx in correct_df.index:
        similarity_correct_scores.append(compute_similarity(idx))
    print_summary(similarity_correct_scores)

    file_object.close()

for fold in [1,2,3,4,5]:
    print(f"running for fold {fold}")
    analysis_for_fold(fold)