Feat: include basic ood similarity analysis using bert
This commit is contained in:
parent
2b5994cb52
commit
bb3ddfaa2f
|
@ -49,23 +49,34 @@ class Retriever:
|
|||
|
||||
self.embeddings = all_embeddings
|
||||
|
||||
def cosine_similarity_chunked(batch1, batch2, chunk_size=16):
|
||||
def cosine_similarity_chunked(batch1, batch2, chunk_size=1024):
|
||||
device = 'cuda'
|
||||
batch1_size = batch1.size(0)
|
||||
batch2_size = batch2.size(0)
|
||||
batch2.to(device)
|
||||
|
||||
# Prepare an empty tensor to store results
|
||||
cos_sim = torch.empty(batch1_size, batch2_size, device=batch1.device)
|
||||
cos_sim = torch.empty(batch1_size, batch2_size, device=device)
|
||||
|
||||
# Process batch1 in chunks
|
||||
for i in range(0, batch1_size, chunk_size):
|
||||
batch1_chunk = batch1[i:i + chunk_size] # Get chunk of batch1
|
||||
|
||||
batch1_chunk.to(device)
|
||||
# Expand batch1 chunk and entire batch2 for comparison
|
||||
batch1_chunk_exp = batch1_chunk.unsqueeze(1) # Shape: (chunk_size, 1, seq_len)
|
||||
batch2_exp = batch2.unsqueeze(0) # Shape: (1, batch2_size, seq_len)
|
||||
# batch1_chunk_exp = batch1_chunk.unsqueeze(1) # Shape: (chunk_size, 1, seq_len)
|
||||
# batch2_exp = batch2.unsqueeze(0) # Shape: (1, batch2_size, seq_len)
|
||||
batch2_norms = batch2.norm(dim=1, keepdim=True)
|
||||
|
||||
|
||||
# Compute cosine similarity for the chunk and store it in the final tensor
|
||||
cos_sim[i:i + chunk_size] = F.cosine_similarity(batch1_chunk_exp, batch2_exp, dim=-1)
|
||||
# cos_sim[i:i + chunk_size] = F.cosine_similarity(batch1_chunk_exp, batch2_exp, dim=-1)
|
||||
|
||||
# Compute cosine similarity by matrix multiplication and normalizing
|
||||
sim_chunk = torch.mm(batch1_chunk, batch2.T) / (batch1_chunk.norm(dim=1, keepdim=True) * batch2_norms.T + 1e-8)
|
||||
|
||||
# Store the results in the appropriate part of the final tensor
|
||||
cos_sim[i:i + chunk_size] = sim_chunk
|
||||
|
||||
return cos_sim
|
||||
|
||||
|
|
|
@ -0,0 +1,12 @@
|
|||
# %%
|
||||
# we need to create the mdm_list
|
||||
# import the full mdm-only file
|
||||
import pandas as pd
|
||||
data_path = '../../data_import/exports/data_mapping_mdm.csv'
|
||||
full_df = pd.read_csv(data_path, skipinitialspace=True)
|
||||
mdm_list = sorted(list((set(full_df['pattern']))))
|
||||
|
||||
|
||||
# %%
|
||||
mdm_list
|
||||
# %%
|
|
@ -53,23 +53,34 @@ class Retriever:
|
|||
|
||||
self.embeddings = all_embeddings
|
||||
|
||||
def cosine_similarity_chunked(batch1, batch2, chunk_size=16):
|
||||
def cosine_similarity_chunked(batch1, batch2, chunk_size=1024):
|
||||
device = 'cuda'
|
||||
batch1_size = batch1.size(0)
|
||||
batch2_size = batch2.size(0)
|
||||
batch2.to(device)
|
||||
|
||||
# Prepare an empty tensor to store results
|
||||
cos_sim = torch.empty(batch1_size, batch2_size, device=batch1.device)
|
||||
cos_sim = torch.empty(batch1_size, batch2_size, device=device)
|
||||
|
||||
# Process batch1 in chunks
|
||||
for i in range(0, batch1_size, chunk_size):
|
||||
batch1_chunk = batch1[i:i + chunk_size] # Get chunk of batch1
|
||||
|
||||
batch1_chunk.to(device)
|
||||
# Expand batch1 chunk and entire batch2 for comparison
|
||||
batch1_chunk_exp = batch1_chunk.unsqueeze(1) # Shape: (chunk_size, 1, seq_len)
|
||||
batch2_exp = batch2.unsqueeze(0) # Shape: (1, batch2_size, seq_len)
|
||||
# batch1_chunk_exp = batch1_chunk.unsqueeze(1) # Shape: (chunk_size, 1, seq_len)
|
||||
# batch2_exp = batch2.unsqueeze(0) # Shape: (1, batch2_size, seq_len)
|
||||
batch2_norms = batch2.norm(dim=1, keepdim=True)
|
||||
|
||||
|
||||
# Compute cosine similarity for the chunk and store it in the final tensor
|
||||
cos_sim[i:i + chunk_size] = F.cosine_similarity(batch1_chunk_exp, batch2_exp, dim=-1)
|
||||
# cos_sim[i:i + chunk_size] = F.cosine_similarity(batch1_chunk_exp, batch2_exp, dim=-1)
|
||||
|
||||
# Compute cosine similarity by matrix multiplication and normalizing
|
||||
sim_chunk = torch.mm(batch1_chunk, batch2.T) / (batch1_chunk.norm(dim=1, keepdim=True) * batch2_norms.T + 1e-8)
|
||||
|
||||
# Store the results in the appropriate part of the final tensor
|
||||
cos_sim[i:i + chunk_size] = sim_chunk
|
||||
|
||||
return cos_sim
|
||||
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
__pycache__
|
|
@ -0,0 +1,288 @@
|
|||
# %%
|
||||
import pandas as pd
|
||||
from utils import Retriever, cosine_similarity_chunked
|
||||
import os
|
||||
import glob
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
|
||||
# %%
|
||||
fold = 1
|
||||
data_path = f'../../train/mapping_pattern/mapping_prediction/exports/result_group_{fold}.csv'
|
||||
test_df = pd.read_csv(data_path, skipinitialspace=True)
|
||||
|
||||
# %%
|
||||
class Embedder():
|
||||
input_df: pd.DataFrame
|
||||
fold: int
|
||||
|
||||
def __init__(self, input_df):
|
||||
self.input_df = input_df
|
||||
|
||||
|
||||
def make_embedding(self, checkpoint_path):
|
||||
|
||||
def generate_input_list(df):
|
||||
input_list = []
|
||||
for _, row in df.iterrows():
|
||||
desc = f"<DESC>{row['tag_description']}<DESC>"
|
||||
unit = f"<UNIT>{row['unit']}<UNIT>"
|
||||
element = f"{desc}{unit}"
|
||||
input_list.append(element)
|
||||
return input_list
|
||||
|
||||
# prepare reference embed
|
||||
train_data = list(generate_input_list(self.input_df))
|
||||
# Define the directory and the pattern
|
||||
retriever_train = Retriever(train_data, checkpoint_path)
|
||||
retriever_train.make_embedding(batch_size=64)
|
||||
return retriever_train.embeddings.to('cpu')
|
||||
|
||||
|
||||
|
||||
# %%
|
||||
data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
|
||||
train_df = pd.read_csv(data_path, skipinitialspace=True)
|
||||
|
||||
checkpoint_directory = "../../train/classification_bert"
|
||||
directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}')
|
||||
# Use glob to find matching paths
|
||||
# path is usually checkpoint_fold_1/checkpoint-<step number>
|
||||
# we are guaranteed to save only 1 checkpoint from training
|
||||
pattern = 'checkpoint-*'
|
||||
checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
|
||||
|
||||
train_embedder = Embedder(input_df=train_df)
|
||||
train_embeds = train_embedder.make_embedding(checkpoint_path)
|
||||
|
||||
test_embedder = Embedder(input_df=test_df)
|
||||
test_embeds = test_embedder.make_embedding(checkpoint_path)
|
||||
|
||||
|
||||
|
||||
# %%
|
||||
# test embeds are inputs since we are looking back at train data
|
||||
cos_sim_matrix = cosine_similarity_chunked(test_embeds, train_embeds, chunk_size=1024).cpu().numpy()
|
||||
|
||||
|
||||
# %%
|
||||
# the following function takes in a full cos_sim_matrix
|
||||
# condition_source: boolean selectors of the source embedding
|
||||
# condition_target: boolean selectors of the target embedding
|
||||
def find_closest(cos_sim_matrix, condition_source, condition_target):
|
||||
# subset_matrix = cos_sim_matrix[condition_source]
|
||||
# except we are subsetting 2D matrix (row, column)
|
||||
subset_matrix = cos_sim_matrix[np.ix_(condition_source, condition_target)]
|
||||
# we select top k here
|
||||
# Get the indices of the top 5 maximum values along axis 1
|
||||
top_k = 3
|
||||
top_k_indices = np.argsort(subset_matrix, axis=1)[:, -top_k:] # Get indices of top k values
|
||||
# note that top_k_indices is a nested list because of the 2d nature of the matrix
|
||||
# the result is flipped
|
||||
top_k_indices[0] = top_k_indices[0][::-1]
|
||||
|
||||
# Get the values of the top 5 maximum scores
|
||||
top_k_values = np.take_along_axis(subset_matrix, top_k_indices, axis=1)
|
||||
|
||||
|
||||
return top_k_indices, top_k_values
|
||||
|
||||
|
||||
####################################################
|
||||
# special find-back code
|
||||
# %%
|
||||
def find_back_element_with_print(select_idx):
|
||||
condition_source = test_df['tag_description'] == test_df[test_df.index == select_idx]['tag_description'].tolist()[0]
|
||||
condition_target = np.ones(train_embeds.shape[0], dtype=bool)
|
||||
|
||||
top_k_indices, top_k_values = find_closest(
|
||||
cos_sim_matrix=cos_sim_matrix,
|
||||
condition_source=condition_source,
|
||||
condition_target=condition_target)
|
||||
|
||||
training_data_pattern_list = train_df.iloc[top_k_indices[0]]['pattern'].to_list()
|
||||
training_desc_list = train_df.iloc[top_k_indices[0]]['tag_description'].to_list()
|
||||
|
||||
test_data_pattern_list = test_df[test_df.index == select_idx]['pattern'].to_list()
|
||||
test_desc_list = test_df[test_df.index == select_idx]['tag_description'].to_list()
|
||||
test_ship_id = test_df[test_df.index == select_idx]['ships_idx'].to_list()[0]
|
||||
predicted_test_data = test_df[test_df.index == select_idx]['p_thing'] + ' ' + test_df[test_df.index == select_idx]['p_property']
|
||||
predicted_test_data = predicted_test_data.to_list()[0]
|
||||
|
||||
print("*" * 80)
|
||||
print("idx:", select_idx)
|
||||
print("train desc", training_desc_list)
|
||||
print("train thing+property", training_data_pattern_list)
|
||||
print("test desc", test_desc_list)
|
||||
print("test thing+property", test_data_pattern_list)
|
||||
print("predicted thing+property", predicted_test_data)
|
||||
print("ships idx", test_ship_id)
|
||||
print("score:", top_k_values[0])
|
||||
|
||||
test_pattern = test_data_pattern_list[0]
|
||||
|
||||
find_back_list = [ test_pattern in pattern for pattern in training_data_pattern_list ]
|
||||
|
||||
if sum(find_back_list) > 0:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
# %%
|
||||
def find_back_element(select_idx):
|
||||
in_train_flag = False
|
||||
condition_source = test_df['tag_description'] == test_df[test_df.index == select_idx]['tag_description'].tolist()[0]
|
||||
condition_target = np.ones(train_embeds.shape[0], dtype=bool)
|
||||
|
||||
top_k_indices, top_k_values = find_closest(
|
||||
cos_sim_matrix=cos_sim_matrix,
|
||||
condition_source=condition_source,
|
||||
condition_target=condition_target)
|
||||
|
||||
training_data_pattern_list = train_df.iloc[top_k_indices[0]]['pattern'].to_list()
|
||||
|
||||
test_data_pattern_list = test_df[test_df.index == select_idx]['pattern'].to_list()
|
||||
# just to convert the series format to string
|
||||
test_pattern = test_data_pattern_list[0]
|
||||
# print(training_data_pattern_list)
|
||||
# print(test_data_pattern_list)
|
||||
|
||||
|
||||
find_back_list = [ test_pattern in pattern for pattern in training_data_pattern_list ]
|
||||
|
||||
if sum(find_back_list) > 0:
|
||||
in_train_flag = True
|
||||
else:
|
||||
in_train_flag = False
|
||||
|
||||
return in_train_flag, top_k_values[0][0]
|
||||
|
||||
# %%
|
||||
in_train_list = []
|
||||
sim_list = []
|
||||
for select_idx in tqdm(test_df.index):
|
||||
in_train_flag, top_sim_value = find_back_element(select_idx)
|
||||
in_train_list.append(in_train_flag)
|
||||
sim_list.append(top_sim_value)
|
||||
|
||||
# analysis 1: using threshold to perform find-back prediction success
|
||||
# %%
|
||||
threshold = 0.9
|
||||
predict_list = [ elem > threshold for elem in sim_list ]
|
||||
|
||||
# %%
|
||||
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix
|
||||
y_true = in_train_list
|
||||
y_pred = predict_list
|
||||
|
||||
# Compute metrics
|
||||
accuracy = accuracy_score(y_true, y_pred)
|
||||
f1 = f1_score(y_true, y_pred, average='macro')
|
||||
precision = precision_score(y_true, y_pred, average='macro')
|
||||
recall = recall_score(y_true, y_pred, average='macro')
|
||||
|
||||
# Print the results
|
||||
print(f'Accuracy: {accuracy:.5f}')
|
||||
print(f'F1 Score: {f1:.5f}')
|
||||
print(f'Precision: {precision:.5f}')
|
||||
print(f'Recall: {recall:.5f}')
|
||||
|
||||
# analysis 2: using find-back class to check distribution of similarities
|
||||
# %%
|
||||
sim_list_true = []
|
||||
sim_list_false = []
|
||||
for idx, elem in enumerate(in_train_list):
|
||||
# true condition
|
||||
if elem:
|
||||
sim_list_true.append(sim_list[idx])
|
||||
else:
|
||||
sim_list_false.append(sim_list[idx])
|
||||
|
||||
# %%
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Sample data
|
||||
list1 = sim_list_true
|
||||
list2 = sim_list_false
|
||||
|
||||
# Plot histograms
|
||||
bins = 50
|
||||
plt.hist(list1, bins=bins, alpha=0.5, label='List 1', density=False)
|
||||
plt.hist(list2, bins=bins, alpha=0.5, label='List 2', density=False)
|
||||
|
||||
# Labels and legend
|
||||
plt.xlabel('Value')
|
||||
plt.ylabel('Frequency')
|
||||
plt.legend(loc='upper right')
|
||||
plt.title('Histograms of in-dist and out-dist similarities')
|
||||
|
||||
# Show plot
|
||||
plt.show()
|
||||
|
||||
# analysis 3
|
||||
# MDM result
|
||||
# MDM is not an accurate measure due to inconsistencies in training and test
|
||||
# distributions
|
||||
# e.g. training is a subset of MDM data, but test could contain MDM data not
|
||||
# found in train, therefore we cannot possibly achieve perfect prediction of
|
||||
# 'MDM' data
|
||||
|
||||
# it is more accurate to use the result obtained from the find-back search
|
||||
# %%
|
||||
# there are 2183 actual datasets
|
||||
sum(test_df['MDM'])
|
||||
# %%
|
||||
# we find 3079 to be similar to the training distribution
|
||||
sum(predict_list)
|
||||
# %%
|
||||
# in actuality only 2051 are similar to the training distribution enough to find
|
||||
# answers during find-back
|
||||
sum(in_train_list)
|
||||
# %%
|
||||
# out of predicted, 1947 are mdm
|
||||
# by setting a threshold, we are able to get 95% of 2051
|
||||
sum(test_df[predict_list]['MDM'])
|
||||
|
||||
# %%
|
||||
# out of find-back labels, 2051 are mdm
|
||||
# this represents the limit of the data distributional differences
|
||||
sum(test_df[in_train_list]['MDM'])
|
||||
|
||||
# analysis 4
|
||||
# check if similarity is different between mdm and non-mdm
|
||||
# this also checks the validity of the selection approach
|
||||
# %%
|
||||
sim_list_true = []
|
||||
sim_list_false = []
|
||||
in_mdm_list = test_df['MDM'].to_list()
|
||||
for idx, elem in enumerate(in_mdm_list):
|
||||
# true condition
|
||||
if elem:
|
||||
sim_list_true.append(sim_list[idx])
|
||||
else:
|
||||
sim_list_false.append(sim_list[idx])
|
||||
|
||||
# %%
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Sample data
|
||||
list1 = sim_list_true
|
||||
list2 = sim_list_false
|
||||
|
||||
# Plot histograms
|
||||
bins = 50
|
||||
plt.hist(list1, bins=bins, alpha=0.5, label='List 1', density=False)
|
||||
plt.hist(list2, bins=bins, alpha=0.5, label='List 2', density=False)
|
||||
|
||||
# Labels and legend
|
||||
plt.xlabel('Value')
|
||||
plt.ylabel('Frequency')
|
||||
plt.legend(loc='upper right')
|
||||
plt.title('Histograms of in-dist and out-dist similarities')
|
||||
|
||||
# Show plot
|
||||
plt.show()
|
||||
|
||||
|
||||
# %%
|
|
@ -0,0 +1,81 @@
|
|||
import torch
|
||||
from transformers import (
|
||||
AutoTokenizer,
|
||||
AutoModelForSequenceClassification,
|
||||
DataCollatorWithPadding,
|
||||
)
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
|
||||
class Retriever:
|
||||
def __init__(self, input_texts, model_checkpoint):
|
||||
# we need to generate the embedding from list of input strings
|
||||
self.embeddings = []
|
||||
self.inputs = input_texts
|
||||
model_checkpoint = model_checkpoint
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
|
||||
|
||||
model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint)
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
# device = "cpu"
|
||||
model.to(self.device)
|
||||
self.model = model.eval()
|
||||
|
||||
|
||||
def make_embedding(self, batch_size=64):
|
||||
all_embeddings = self.embeddings
|
||||
input_texts = self.inputs
|
||||
|
||||
for i in range(0, len(input_texts), batch_size):
|
||||
batch_texts = input_texts[i:i+batch_size]
|
||||
# Tokenize the input text
|
||||
inputs = self.tokenizer(batch_texts, return_tensors="pt", padding=True, truncation=True, max_length=64)
|
||||
input_ids = inputs.input_ids.to(self.device)
|
||||
attention_mask = inputs.attention_mask.to(self.device)
|
||||
|
||||
|
||||
# Pass the input through the encoder and retrieve the embeddings
|
||||
with torch.no_grad():
|
||||
encoder_outputs = self.model(input_ids, attention_mask=attention_mask, output_hidden_states=True)
|
||||
# get last layer
|
||||
embeddings = encoder_outputs.hidden_states[-1]
|
||||
# get cls token embedding
|
||||
cls_embeddings = embeddings[:, 0, :] # Shape: (batch_size, hidden_size)
|
||||
all_embeddings.append(cls_embeddings)
|
||||
|
||||
# remove the batch list and makes a single large tensor, dim=0 increases row-wise
|
||||
all_embeddings = torch.cat(all_embeddings, dim=0)
|
||||
|
||||
self.embeddings = all_embeddings
|
||||
|
||||
def cosine_similarity_chunked(batch1, batch2, chunk_size=1024):
|
||||
device = 'cuda'
|
||||
batch1_size = batch1.size(0)
|
||||
batch2_size = batch2.size(0)
|
||||
batch2.to(device)
|
||||
|
||||
# Prepare an empty tensor to store results
|
||||
cos_sim = torch.empty(batch1_size, batch2_size, device=device)
|
||||
|
||||
# Process batch1 in chunks
|
||||
for i in range(0, batch1_size, chunk_size):
|
||||
batch1_chunk = batch1[i:i + chunk_size] # Get chunk of batch1
|
||||
|
||||
batch1_chunk.to(device)
|
||||
# Expand batch1 chunk and entire batch2 for comparison
|
||||
# batch1_chunk_exp = batch1_chunk.unsqueeze(1) # Shape: (chunk_size, 1, seq_len)
|
||||
# batch2_exp = batch2.unsqueeze(0) # Shape: (1, batch2_size, seq_len)
|
||||
batch2_norms = batch2.norm(dim=1, keepdim=True)
|
||||
|
||||
|
||||
# Compute cosine similarity for the chunk and store it in the final tensor
|
||||
# cos_sim[i:i + chunk_size] = F.cosine_similarity(batch1_chunk_exp, batch2_exp, dim=-1)
|
||||
|
||||
# Compute cosine similarity by matrix multiplication and normalizing
|
||||
sim_chunk = torch.mm(batch1_chunk, batch2.T) / (batch1_chunk.norm(dim=1, keepdim=True) * batch2_norms.T + 1e-8)
|
||||
|
||||
# Store the results in the appropriate part of the final tensor
|
||||
cos_sim[i:i + chunk_size] = sim_chunk
|
||||
|
||||
return cos_sim
|
|
@ -54,23 +54,34 @@ class Retriever:
|
|||
|
||||
self.embeddings = all_embeddings
|
||||
|
||||
def cosine_similarity_chunked(batch1, batch2, chunk_size=16):
|
||||
def cosine_similarity_chunked(batch1, batch2, chunk_size=1024):
|
||||
device = 'cuda'
|
||||
batch1_size = batch1.size(0)
|
||||
batch2_size = batch2.size(0)
|
||||
batch2.to(device)
|
||||
|
||||
# Prepare an empty tensor to store results
|
||||
cos_sim = torch.empty(batch1_size, batch2_size, device=batch1.device)
|
||||
cos_sim = torch.empty(batch1_size, batch2_size, device=device)
|
||||
|
||||
# Process batch1 in chunks
|
||||
for i in range(0, batch1_size, chunk_size):
|
||||
batch1_chunk = batch1[i:i + chunk_size] # Get chunk of batch1
|
||||
|
||||
batch1_chunk.to(device)
|
||||
# Expand batch1 chunk and entire batch2 for comparison
|
||||
batch1_chunk_exp = batch1_chunk.unsqueeze(1) # Shape: (chunk_size, 1, seq_len)
|
||||
batch2_exp = batch2.unsqueeze(0) # Shape: (1, batch2_size, seq_len)
|
||||
# batch1_chunk_exp = batch1_chunk.unsqueeze(1) # Shape: (chunk_size, 1, seq_len)
|
||||
# batch2_exp = batch2.unsqueeze(0) # Shape: (1, batch2_size, seq_len)
|
||||
batch2_norms = batch2.norm(dim=1, keepdim=True)
|
||||
|
||||
|
||||
# Compute cosine similarity for the chunk and store it in the final tensor
|
||||
cos_sim[i:i + chunk_size] = F.cosine_similarity(batch1_chunk_exp, batch2_exp, dim=-1)
|
||||
# cos_sim[i:i + chunk_size] = F.cosine_similarity(batch1_chunk_exp, batch2_exp, dim=-1)
|
||||
|
||||
# Compute cosine similarity by matrix multiplication and normalizing
|
||||
sim_chunk = torch.mm(batch1_chunk, batch2.T) / (batch1_chunk.norm(dim=1, keepdim=True) * batch2_norms.T + 1e-8)
|
||||
|
||||
# Store the results in the appropriate part of the final tensor
|
||||
cos_sim[i:i + chunk_size] = sim_chunk
|
||||
|
||||
return cos_sim
|
||||
|
||||
|
|
Loading…
Reference in New Issue