analysis/categories/label_print.py

@@ -10,12 +10,6 @@ mdm_list = sorted(list((set(full_df['pattern']))))
 # %%
 full_df
 
-# %%
-mdm_list
-
-# %%
-mask = full_df['pattern'] == 'GE#Flow FGMassFlow'
-full_df[mask]
 # %%
 mask1 = full_df['thing'] == 'ME1TurboCharger1'
 mask2 = full_df['property'] == 'LOInletPress'

overall/README.md

@@ -1,2 +0,0 @@
-This section is to evaluate the combined (relevant-class prediction) and
-(mapping prediction) to evaluate the final correct mapping accuracy.

overall/combined_mapping_and_classification_analysis.py

@@ -1,34 +0,0 @@
-# %%
-import pandas as pd
-
-# following code computes final mapping + classification accuracy
-# %%
-def run(fold):
-    data_path = f'../relevant_class/binary_classifier_desc_unit/classification_prediction/exports/result_group_{fold}.csv'
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    p_mdm = df['p_mdm']
-
-    # data_path = f'../train/mapping_t5_complete_desc_unit_name/mapping_prediction/exports/result_group_{fold}.csv'
-    data_path = f'../train/modified_t5_decoder_4_layers/mapping_prediction/exports/result_group_{fold}.csv'
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    actual_mdm = df['MDM']
-
-    thing_correctness = df['thing'] == df['p_thing']
-    property_correctness = df['property'] == df['p_property']
-    answer = thing_correctness & property_correctness
-
-    # if is non-MDM -> then should be unmapped
-    # if is MDM -> then should be mapped correctly
-
-    # out of correctly predicted relevant data, how many are mapped correctly?
-    correct_positive_mdm_and_map = sum(p_mdm & actual_mdm & answer)
-
-    # number of correctly predicted non-relevant data
-    correct_negative_mdm = sum(~(p_mdm) & ~(actual_mdm))
-
-    overall_correct = (correct_positive_mdm_and_map + correct_negative_mdm)/len(actual_mdm)
-    print(overall_correct)
-# %%
-for fold in [1,2,3,4,5]:
-    run(fold)
-

relevant_class/binary_classifier_desc/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

relevant_class/binary_classifier_desc/classification_prediction/.gitignore

@@ -1,2 +0,0 @@
-exports
-output.txt

relevant_class/binary_classifier_desc/classification_prediction/predict.py

@@ -1,235 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-from torch.utils.data import DataLoader
-
-from transformers import (
-    AutoTokenizer,
-    AutoModelForSequenceClassification,
-    DataCollatorWithPadding,
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-from tqdm import tqdm
-
-torch.set_float32_matmul_precision('high')
-
-
-BATCH_SIZE = 256
-
-# %%
-
-# %%
-
-# outputs a list of dictionaries
-# processes dataframe into lists of dictionaries
-# each element maps input to output
-# input: tag_description
-# output: class label
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        in_mdm_label = int(row['MDM'])
-        element = {
-            'text' : f"{desc}{unit}",
-            'label': in_mdm_label,
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_dataset(fold):
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    test_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    test_dataset = Dataset.from_list(process_df_to_dict(test_df))
-
-    return test_dataset
-
-
-# %%
-
-# function to perform training for a given fold
-def test(fold):
-
-    test_dataset = create_dataset(fold)
-
-    # prepare tokenizer
-
-    checkpoint_directory = f'../checkpoint_fold_{fold}'
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    model_checkpoint = glob.glob(os.path.join(checkpoint_directory, pattern))[0]
-
-    tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    # %%
-    # compute max token length
-    max_length = 0
-    for sample in test_dataset['text']:
-        # Tokenize the sample and get the length
-        input_ids = tokenizer(sample, truncation=False, add_special_tokens=True)["input_ids"]
-        length = len(input_ids)
-        
-        # Update max_length if this sample is longer
-        if length > max_length:
-            max_length = length
-
-    print(max_length)
-
-    # %%
-
-    max_length = 128
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['text']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            max_length=max_length,
-            # truncation=True,
-            padding='max_length'
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    datasets = test_dataset.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns="text",
-    )
-
-
-    datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])
-
-    # %% temp
-    # tokenized_datasets['train'].rename_columns()
-
-    # %%
-    # create data collator
-
-    # data_collator = DataCollatorWithPadding(tokenizer=tokenizer, padding="max_length")
-
-    # %%
-    # compute metrics
-    # metric = evaluate.load("accuracy")
-    # 
-    # 
-    # def compute_metrics(eval_preds):
-    #     preds, labels = eval_preds
-    #     preds = np.argmax(preds, axis=1)
-    #     return metric.compute(predictions=preds, references=labels)
-
-    model = AutoModelForSequenceClassification.from_pretrained(
-        model_checkpoint,
-        num_labels=2)
-    # important! after extending tokens vocab
-    model.resize_token_embeddings(len(tokenizer))
-
-    model = model.eval()
-
-    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-    model.to(device)
-
-    pred_labels = []
-    actual_labels = []
-
-
-    dataloader = DataLoader(datasets, batch_size=BATCH_SIZE, shuffle=False)
-    for batch in tqdm(dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            actual_labels.extend(batch['label'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                logits = model(
-                    input_ids,
-                    attention_mask).logits
-                predicted_class_ids = logits.argmax(dim=1).to("cpu")
-                pred_labels.extend(predicted_class_ids)
-
-    pred_labels = [tensor.item() for tensor in pred_labels]
-    pred_labels = np.array(pred_labels, dtype=bool)
-
-    # append the mdm prediction to the test_df for analysis later
-    df_out = pd.DataFrame({
-        'p_mdm': pred_labels, 
-    })
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    test_df = pd.read_csv(data_path, skipinitialspace=True)
-    df_export = pd.concat([test_df, df_out], axis=1)
-    df_export.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-
-
-    # %%
-    from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix
-    y_true = actual_labels
-    y_pred = pred_labels
-
-    # Compute metrics
-    accuracy = accuracy_score(y_true, y_pred)
-    f1 = f1_score(y_true, y_pred)
-    precision = precision_score(y_true, y_pred)
-    recall = recall_score(y_true, y_pred)
-
-    cm = confusion_matrix(y_true, y_pred)
-    tn, fp, fn, tp = cm.ravel()
-
-    with open("output.txt", "a") as f:
-
-
-        print('*' * 80, file=f)
-        print(f'Fold: {fold}', file=f)
-        # Print the results
-        print(f"tp: {tp}", file=f)
-        print(f"tn: {tn}", file=f)
-        print(f"fp: {fp}", file=f)
-        print(f"fn: {fn}", file=f)
-        print(f'Accuracy: {accuracy:.5f}', file=f)
-        print(f'F1 Score: {f1:.5f}', file=f)
-        print(f'Precision: {precision:.5f}', file=f)
-        print(f'Recall: {recall:.5f}', file=f)
-
-
-# %%
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    test(fold)

relevant_class/binary_classifier_desc/train.py

@@ -1,218 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-from transformers import (
-    AutoTokenizer,
-    AutoModelForSequenceClassification,
-    DataCollatorWithPadding,
-    Trainer,
-    EarlyStoppingCallback,
-    TrainingArguments
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-# %%
-
-# we need to create the mdm_list
-# import the full mdm-only file
-# data_path = '../../data_import/exports/data_mapping_mdm.csv'
-# full_df = pd.read_csv(data_path, skipinitialspace=True)
-# rather than use pattern, we use the real thing and property
-
-
-# %%
-id2label = {0: False, 1: True}
-label2id = {False: 0, True: 1}
-
-# %%
-
-# outputs a list of dictionaries
-# processes dataframe into lists of dictionaries
-# each element maps input to output
-# input: tag_description
-# output: class label
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        in_mdm_label = int(row['MDM'])
-        element = {
-            'text' : f"{desc}",
-            'label': in_mdm_label,
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    # data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-
-    # reconstruct full training data with non-mdm data
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    test_df = pd.read_csv(data_path, skipinitialspace=True)
-    ships_list = list(set(test_df['ships_idx']))
-    data_path = '../../data_preprocess/exports/preprocessed_data.csv'
-    full_df = pd.read_csv(data_path, skipinitialspace=True)
-    train_df = full_df[~full_df['ships_idx'].isin(ships_list)]
-
-    train_ships_list = sorted(list(set(train_df['ships_idx'])))
-
-    train_ships_set = set(train_ships_list)
-    test_ships_set = set(ships_list)
-
-    # assertion for non data leakage
-    assert not set(train_ships_set).intersection(test_ships_set)
-
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# %%
-
-# function to perform training for a given fold
-def train(fold):
-
-    save_path = f'checkpoint_fold_{fold}'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-
-    model_checkpoint = "distilbert/distilbert-base-cased"
-    # model_checkpoint = 'google-bert/bert-base-cased'
-    tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['text']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            max_length=max_length,
-            truncation=True,
-            padding=True
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns="text",
-    )
-
-    # %% temp
-    # tokenized_datasets['train'].rename_columns()
-
-    # %%
-    # create data collator
-
-    data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
-
-    # %%
-    # compute metrics
-    metric = evaluate.load("accuracy")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        preds = np.argmax(preds, axis=1)
-        return metric.compute(predictions=preds, references=labels)
-
-    # %%
-    # create id2label and label2id
-
-
-    # %%
-    model = AutoModelForSequenceClassification.from_pretrained(
-        model_checkpoint,
-        num_labels=2)
-    # important! after extending tokens vocab
-    model.resize_token_embeddings(len(tokenizer))
-
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # %%
-    # Trainer
-
-    training_args = TrainingArguments(
-        output_dir=f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-5,
-        per_device_train_batch_size=128,
-        per_device_eval_batch_size=128,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=80,
-        bf16=True,
-        push_to_hub=False,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Trainer(
-        model,
-        training_args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        tokenizer=tokenizer,
-        data_collator=data_collator,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-
-
-# %%

relevant_class/binary_classifier_desc_unit/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

relevant_class/binary_classifier_desc_unit/classification_prediction/.gitignore

@@ -1,2 +0,0 @@
-exports
-output.txt

relevant_class/binary_classifier_desc_unit/classification_prediction/predict.py

@@ -1,235 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-from torch.utils.data import DataLoader
-
-from transformers import (
-    AutoTokenizer,
-    AutoModelForSequenceClassification,
-    DataCollatorWithPadding,
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-from tqdm import tqdm
-
-torch.set_float32_matmul_precision('high')
-
-
-BATCH_SIZE = 256
-
-# %%
-
-# %%
-
-# outputs a list of dictionaries
-# processes dataframe into lists of dictionaries
-# each element maps input to output
-# input: tag_description
-# output: class label
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        in_mdm_label = int(row['MDM'])
-        element = {
-            'text' : f"{desc}{unit}",
-            'label': in_mdm_label,
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_dataset(fold):
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    test_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    test_dataset = Dataset.from_list(process_df_to_dict(test_df))
-
-    return test_dataset
-
-
-# %%
-
-# function to perform training for a given fold
-def test(fold):
-
-    test_dataset = create_dataset(fold)
-
-    # prepare tokenizer
-
-    checkpoint_directory = f'../checkpoint_fold_{fold}'
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    model_checkpoint = glob.glob(os.path.join(checkpoint_directory, pattern))[0]
-
-    tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    # %%
-    # compute max token length
-    max_length = 0
-    for sample in test_dataset['text']:
-        # Tokenize the sample and get the length
-        input_ids = tokenizer(sample, truncation=False, add_special_tokens=True)["input_ids"]
-        length = len(input_ids)
-        
-        # Update max_length if this sample is longer
-        if length > max_length:
-            max_length = length
-
-    print(max_length)
-
-    # %%
-
-    max_length = 128
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['text']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            max_length=max_length,
-            # truncation=True,
-            padding='max_length'
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    datasets = test_dataset.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns="text",
-    )
-
-
-    datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])
-
-    # %% temp
-    # tokenized_datasets['train'].rename_columns()
-
-    # %%
-    # create data collator
-
-    # data_collator = DataCollatorWithPadding(tokenizer=tokenizer, padding="max_length")
-
-    # %%
-    # compute metrics
-    # metric = evaluate.load("accuracy")
-    # 
-    # 
-    # def compute_metrics(eval_preds):
-    #     preds, labels = eval_preds
-    #     preds = np.argmax(preds, axis=1)
-    #     return metric.compute(predictions=preds, references=labels)
-
-    model = AutoModelForSequenceClassification.from_pretrained(
-        model_checkpoint,
-        num_labels=2)
-    # important! after extending tokens vocab
-    model.resize_token_embeddings(len(tokenizer))
-
-    model = model.eval()
-
-    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-    model.to(device)
-
-    pred_labels = []
-    actual_labels = []
-
-
-    dataloader = DataLoader(datasets, batch_size=BATCH_SIZE, shuffle=False)
-    for batch in tqdm(dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            actual_labels.extend(batch['label'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                logits = model(
-                    input_ids,
-                    attention_mask).logits
-                predicted_class_ids = logits.argmax(dim=1).to("cpu")
-                pred_labels.extend(predicted_class_ids)
-
-    pred_labels = [tensor.item() for tensor in pred_labels]
-    pred_labels = np.array(pred_labels, dtype=bool)
-
-    # append the mdm prediction to the test_df for analysis later
-    df_out = pd.DataFrame({
-        'p_mdm': pred_labels, 
-    })
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    test_df = pd.read_csv(data_path, skipinitialspace=True)
-    df_export = pd.concat([test_df, df_out], axis=1)
-    df_export.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-
-
-    # %%
-    from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix
-    y_true = actual_labels
-    y_pred = pred_labels
-
-    # Compute metrics
-    accuracy = accuracy_score(y_true, y_pred)
-    f1 = f1_score(y_true, y_pred)
-    precision = precision_score(y_true, y_pred)
-    recall = recall_score(y_true, y_pred)
-
-    cm = confusion_matrix(y_true, y_pred)
-    tn, fp, fn, tp = cm.ravel()
-
-    with open("output.txt", "a") as f:
-
-
-        print('*' * 80, file=f)
-        print(f'Fold: {fold}', file=f)
-        # Print the results
-        print(f"tp: {tp}", file=f)
-        print(f"tn: {tn}", file=f)
-        print(f"fp: {fp}", file=f)
-        print(f"fn: {fn}", file=f)
-        print(f'Accuracy: {accuracy:.5f}', file=f)
-        print(f'F1 Score: {f1:.5f}', file=f)
-        print(f'Precision: {precision:.5f}', file=f)
-        print(f'Recall: {recall:.5f}', file=f)
-
-
-# %%
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    test(fold)

relevant_class/binary_classifier_desc_unit/train.py

@@ -1,219 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-from transformers import (
-    AutoTokenizer,
-    AutoModelForSequenceClassification,
-    DataCollatorWithPadding,
-    Trainer,
-    EarlyStoppingCallback,
-    TrainingArguments
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-# %%
-
-# we need to create the mdm_list
-# import the full mdm-only file
-# data_path = '../../data_import/exports/data_mapping_mdm.csv'
-# full_df = pd.read_csv(data_path, skipinitialspace=True)
-# rather than use pattern, we use the real thing and property
-
-
-# %%
-id2label = {0: False, 1: True}
-label2id = {False: 0, True: 1}
-
-# %%
-
-# outputs a list of dictionaries
-# processes dataframe into lists of dictionaries
-# each element maps input to output
-# input: tag_description
-# output: class label
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        in_mdm_label = int(row['MDM'])
-        element = {
-            'text' : f"{desc}{unit}",
-            'label': in_mdm_label,
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    # data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-
-    # reconstruct full training data with non-mdm data
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    test_df = pd.read_csv(data_path, skipinitialspace=True)
-    ships_list = list(set(test_df['ships_idx']))
-    data_path = '../../data_preprocess/exports/preprocessed_data.csv'
-    full_df = pd.read_csv(data_path, skipinitialspace=True)
-    train_df = full_df[~full_df['ships_idx'].isin(ships_list)]
-
-    train_ships_list = sorted(list(set(train_df['ships_idx'])))
-
-    train_ships_set = set(train_ships_list)
-    test_ships_set = set(ships_list)
-
-    # assertion for non data leakage
-    assert not set(train_ships_set).intersection(test_ships_set)
-
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# %%
-
-# function to perform training for a given fold
-def train(fold):
-
-    save_path = f'checkpoint_fold_{fold}'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-
-    model_checkpoint = "distilbert/distilbert-base-cased"
-    # model_checkpoint = 'google-bert/bert-base-cased'
-    tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['text']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            max_length=max_length,
-            truncation=True,
-            padding=True
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns="text",
-    )
-
-    # %% temp
-    # tokenized_datasets['train'].rename_columns()
-
-    # %%
-    # create data collator
-
-    data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
-
-    # %%
-    # compute metrics
-    metric = evaluate.load("accuracy")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        preds = np.argmax(preds, axis=1)
-        return metric.compute(predictions=preds, references=labels)
-
-    # %%
-    # create id2label and label2id
-
-
-    # %%
-    model = AutoModelForSequenceClassification.from_pretrained(
-        model_checkpoint,
-        num_labels=2)
-    # important! after extending tokens vocab
-    model.resize_token_embeddings(len(tokenizer))
-
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # %%
-    # Trainer
-
-    training_args = TrainingArguments(
-        output_dir=f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-5,
-        per_device_train_batch_size=128,
-        per_device_eval_batch_size=128,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=80,
-        bf16=True,
-        push_to_hub=False,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Trainer(
-        model,
-        training_args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        tokenizer=tokenizer,
-        data_collator=data_collator,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-
-
-# %%

relevant_class/similarity_classifier_desc/.gitignore

@@ -1,3 +0,0 @@
-__pycache__
-exports
-output.txt

relevant_class/similarity_classifier_desc/README.md

@@ -1,4 +0,0 @@
-# one-class classification by similarity
-
-Purpose: using only Ship Domain attributes, we want to find if the data belongs
-to MDM

relevant_class/similarity_classifier_desc/run.py

@@ -1,175 +0,0 @@
-# %%
-import pandas as pd
-from utils import Retriever, cosine_similarity_chunked
-import os
-import glob
-import numpy as np
-from tqdm import tqdm
-from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix
-
-##################################################
-# helper functions
-
-
-# the following function takes in a full cos_sim_matrix
-# condition_source: boolean selectors of the source embedding
-# condition_target: boolean selectors of the target embedding
-def find_closest(cos_sim_matrix, condition_source, condition_target):
-    # subset_matrix = cos_sim_matrix[condition_source]
-    # except we are subsetting 2D matrix (row, column)
-    subset_matrix = cos_sim_matrix[np.ix_(condition_source, condition_target)]
-    # we select top k here
-    # Get the indices of the top k maximum values along axis 1
-    top_k = 3
-    top_k_indices = np.argsort(subset_matrix, axis=1)[:, -top_k:]  # Get indices of top k values
-    # note that top_k_indices is a nested list because of the 2d nature of the matrix
-    # the result is flipped
-    top_k_indices[0] = top_k_indices[0][::-1]
-    
-    # Get the values of the top 5 maximum scores
-    top_k_values = np.take_along_axis(subset_matrix, top_k_indices, axis=1)
-    
-
-    return top_k_indices, top_k_values
-
-
-
-
-class Embedder():
-    input_df: pd.DataFrame
-    fold: int
-
-    def __init__(self, input_df):
-        self.input_df = input_df
-
-
-    def make_embedding(self, checkpoint_path):
-
-        def generate_input_list(df):
-            input_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                element = f"{desc}"
-                input_list.append(element)
-            return input_list
-
-        # prepare reference embed
-        train_data = list(generate_input_list(self.input_df))
-        # Define the directory and the pattern
-        retriever_train = Retriever(train_data, checkpoint_path)
-        retriever_train.make_embedding(batch_size=64)
-        return retriever_train.embeddings.to('cpu')
-
-
-
-def run_similarity_classifier(fold):
-    data_path = f'../../train/mapping_t5_complete_desc_unit_name/mapping_prediction/exports/result_group_{fold}.csv'
-    test_df = pd.read_csv(data_path, skipinitialspace=True)
-
-
-
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    checkpoint_directory = "../../train/classification_bert_complete_desc"
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-    train_embedder = Embedder(input_df=train_df)
-    train_embeds = train_embedder.make_embedding(checkpoint_path)
-
-    test_embedder = Embedder(input_df=test_df)
-    test_embeds = test_embedder.make_embedding(checkpoint_path)
-
-    def compute_top_k(select_idx):
-        condition_source = test_df['tag_description'] == test_df[test_df.index == select_idx]['tag_description'].tolist()[0]
-        condition_target = np.ones(train_embeds.shape[0], dtype=bool)
-
-        _, top_k_values = find_closest(
-            cos_sim_matrix=cos_sim_matrix,
-            condition_source=condition_source,
-            condition_target=condition_target)
-
-        return top_k_values[0][0]
-
-
-
-    # test embeds are inputs since we are looking back at train data
-    cos_sim_matrix = cosine_similarity_chunked(test_embeds, train_embeds, chunk_size=1024).cpu().numpy()
-
-
-    sim_list = []
-    for select_idx in tqdm(test_df.index):
-        top_sim_value = compute_top_k(select_idx)
-        sim_list.append(top_sim_value)
-
-    # analysis 1: using threshold to perform find-back prediction success
-    threshold_values = np.linspace(0.85, 1.00, 21) # test 20 values, 21 to get nice round numbers
-    best_threshold = 0
-    best_f1 = 0
-    for threshold in threshold_values:
-        predict_list = [ elem > threshold for elem in sim_list ]
-
-        y_true = test_df['MDM'].to_list()
-        y_pred = predict_list
-
-        # Compute metrics
-        accuracy = accuracy_score(y_true, y_pred)
-        f1 = f1_score(y_true, y_pred)
-        precision = precision_score(y_true, y_pred)
-        recall = recall_score(y_true, y_pred)
-
-        if f1 > best_f1:
-            best_threshold = threshold
-            best_f1 = f1
-    
-    # OR just manually set best_threshold
-    # best_threshold = 0.90
-
-    # compute metrics again with best threshold
-    predict_list = [ elem > best_threshold for elem in sim_list ]
-
-    # save
-    pred_labels = np.array(predict_list, dtype=bool)
-
-    # append the mdm prediction to the test_df for analysis later
-    df_out = pd.DataFrame({
-        'p_mdm': pred_labels, 
-    })
-    df_out.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-
-    y_true = test_df['MDM'].to_list()
-    y_pred = predict_list
-    # Compute metrics
-    accuracy = accuracy_score(y_true, y_pred)
-    f1 = f1_score(y_true, y_pred)
-    precision = precision_score(y_true, y_pred)
-    recall = recall_score(y_true, y_pred)
-
-
-
-    with open("output.txt", "a") as f:
-
-        print(f'Fold: {fold}', file=f)
-        print(f'Best threshold: {best_threshold}', file=f)
-        # Print the results
-        print(f'Accuracy: {accuracy:.5f}', file=f)
-        print(f'F1 Score: {f1:.5f}', file=f)
-        print(f'Precision: {precision:.5f}', file=f)
-        print(f'Recall: {recall:.5f}', file=f)
-
-
-
-# %%
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    print(fold)
-    run_similarity_classifier(fold)

relevant_class/similarity_classifier_desc/utils.py

@@ -1,81 +0,0 @@
-import torch
-from transformers import (
-    AutoTokenizer,
-    AutoModelForSequenceClassification,
-    DataCollatorWithPadding,
-)
-import torch.nn.functional as F
-
-
-
-class Retriever:
-    def __init__(self, input_texts, model_checkpoint):
-        # we need to generate the embedding from list of input strings
-        self.embeddings = []
-        self.inputs = input_texts
-        model_checkpoint = model_checkpoint 
-        self.tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-
-        model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint)
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        # device = "cpu"
-        model.to(self.device)
-        self.model = model.eval()
-
-
-    def make_embedding(self, batch_size=64):
-        all_embeddings = self.embeddings
-        input_texts = self.inputs
-
-        for i in range(0, len(input_texts), batch_size):
-            batch_texts = input_texts[i:i+batch_size]
-            # Tokenize the input text
-            inputs = self.tokenizer(batch_texts, return_tensors="pt", padding=True, truncation=True, max_length=64)
-            input_ids = inputs.input_ids.to(self.device)
-            attention_mask = inputs.attention_mask.to(self.device)
-
-
-            # Pass the input through the encoder and retrieve the embeddings
-            with torch.no_grad():
-                encoder_outputs = self.model(input_ids, attention_mask=attention_mask, output_hidden_states=True)
-                # get last layer
-                embeddings = encoder_outputs.hidden_states[-1]
-                # get cls token embedding
-                cls_embeddings = embeddings[:, 0, :]  # Shape: (batch_size, hidden_size)
-                all_embeddings.append(cls_embeddings)
-        
-        # remove the batch list and makes a single large tensor, dim=0 increases row-wise
-        all_embeddings = torch.cat(all_embeddings, dim=0)
-
-        self.embeddings = all_embeddings
-
-def cosine_similarity_chunked(batch1, batch2, chunk_size=1024):
-    device = 'cuda'
-    batch1_size = batch1.size(0)
-    batch2_size = batch2.size(0)
-    batch2.to(device)
-    
-    # Prepare an empty tensor to store results
-    cos_sim = torch.empty(batch1_size, batch2_size, device=device)
-
-    # Process batch1 in chunks
-    for i in range(0, batch1_size, chunk_size):
-        batch1_chunk = batch1[i:i + chunk_size]  # Get chunk of batch1
-        
-        batch1_chunk.to(device)
-        # Expand batch1 chunk and entire batch2 for comparison
-        # batch1_chunk_exp = batch1_chunk.unsqueeze(1)  # Shape: (chunk_size, 1, seq_len)
-        # batch2_exp = batch2.unsqueeze(0)  # Shape: (1, batch2_size, seq_len)
-        batch2_norms = batch2.norm(dim=1, keepdim=True)
-
-        
-        # Compute cosine similarity for the chunk and store it in the final tensor
-        # cos_sim[i:i + chunk_size] = F.cosine_similarity(batch1_chunk_exp, batch2_exp, dim=-1)
-
-        # Compute cosine similarity by matrix multiplication and normalizing
-        sim_chunk = torch.mm(batch1_chunk, batch2.T) / (batch1_chunk.norm(dim=1, keepdim=True) * batch2_norms.T + 1e-8)
-        
-        # Store the results in the appropriate part of the final tensor
-        cos_sim[i:i + chunk_size] = sim_chunk
-    
-    return cos_sim

relevant_class/similarity_classifier_desc_unit/.gitignore

@@ -1,3 +0,0 @@
-__pycache__
-exports
-output.txt

relevant_class/similarity_classifier_desc_unit/README.md

@@ -1,4 +0,0 @@
-# one-class classification by similarity
-
-Purpose: using only Ship Domain attributes, we want to find if the data belongs
-to MDM

relevant_class/similarity_classifier_desc_unit/run.py

@@ -1,176 +0,0 @@
-# %%
-import pandas as pd
-from utils import Retriever, cosine_similarity_chunked
-import os
-import glob
-import numpy as np
-from tqdm import tqdm
-from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix
-
-##################################################
-# helper functions
-
-
-# the following function takes in a full cos_sim_matrix
-# condition_source: boolean selectors of the source embedding
-# condition_target: boolean selectors of the target embedding
-def find_closest(cos_sim_matrix, condition_source, condition_target):
-    # subset_matrix = cos_sim_matrix[condition_source]
-    # except we are subsetting 2D matrix (row, column)
-    subset_matrix = cos_sim_matrix[np.ix_(condition_source, condition_target)]
-    # we select top k here
-    # Get the indices of the top k maximum values along axis 1
-    top_k = 3
-    top_k_indices = np.argsort(subset_matrix, axis=1)[:, -top_k:]  # Get indices of top k values
-    # note that top_k_indices is a nested list because of the 2d nature of the matrix
-    # the result is flipped
-    top_k_indices[0] = top_k_indices[0][::-1]
-    
-    # Get the values of the top 5 maximum scores
-    top_k_values = np.take_along_axis(subset_matrix, top_k_indices, axis=1)
-    
-
-    return top_k_indices, top_k_values
-
-
-
-
-class Embedder():
-    input_df: pd.DataFrame
-    fold: int
-
-    def __init__(self, input_df):
-        self.input_df = input_df
-
-
-    def make_embedding(self, checkpoint_path):
-
-        def generate_input_list(df):
-            input_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = f"{desc}{unit}"
-                input_list.append(element)
-            return input_list
-
-        # prepare reference embed
-        train_data = list(generate_input_list(self.input_df))
-        # Define the directory and the pattern
-        retriever_train = Retriever(train_data, checkpoint_path)
-        retriever_train.make_embedding(batch_size=64)
-        return retriever_train.embeddings.to('cpu')
-
-
-
-def run_similarity_classifier(fold):
-    data_path = f'../../train/mapping_t5_complete_desc_unit_name/mapping_prediction/exports/result_group_{fold}.csv'
-    test_df = pd.read_csv(data_path, skipinitialspace=True)
-
-
-
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    checkpoint_directory = "../../train/classification_bert_complete_desc_unit"
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-    train_embedder = Embedder(input_df=train_df)
-    train_embeds = train_embedder.make_embedding(checkpoint_path)
-
-    test_embedder = Embedder(input_df=test_df)
-    test_embeds = test_embedder.make_embedding(checkpoint_path)
-
-    def compute_top_k(select_idx):
-        condition_source = test_df['tag_description'] == test_df[test_df.index == select_idx]['tag_description'].tolist()[0]
-        condition_target = np.ones(train_embeds.shape[0], dtype=bool)
-
-        _, top_k_values = find_closest(
-            cos_sim_matrix=cos_sim_matrix,
-            condition_source=condition_source,
-            condition_target=condition_target)
-
-        return top_k_values[0][0]
-
-
-
-    # test embeds are inputs since we are looking back at train data
-    cos_sim_matrix = cosine_similarity_chunked(test_embeds, train_embeds, chunk_size=1024).cpu().numpy()
-
-
-    sim_list = []
-    for select_idx in tqdm(test_df.index):
-        top_sim_value = compute_top_k(select_idx)
-        sim_list.append(top_sim_value)
-
-    # analysis 1: using threshold to perform find-back prediction success
-    threshold_values = np.linspace(0.85, 1.00, 21) # test 20 values, 21 to get nice round numbers
-    best_threshold = 0
-    best_f1 = 0
-    for threshold in threshold_values:
-        predict_list = [ elem > threshold for elem in sim_list ]
-
-        y_true = test_df['MDM'].to_list()
-        y_pred = predict_list
-
-        # Compute metrics
-        accuracy = accuracy_score(y_true, y_pred)
-        f1 = f1_score(y_true, y_pred)
-        precision = precision_score(y_true, y_pred)
-        recall = recall_score(y_true, y_pred)
-
-        if f1 > best_f1:
-            best_threshold = threshold
-            best_f1 = f1
-    
-    # just manually set best_threshold
-    # best_threshold = 0.90
-
-    # compute metrics again with best threshold
-    predict_list = [ elem > best_threshold for elem in sim_list ]
-
-    # save
-    pred_labels = np.array(predict_list, dtype=bool)
-
-    # append the mdm prediction to the test_df for analysis later
-    df_out = pd.DataFrame({
-        'p_mdm': pred_labels, 
-    })
-    df_out.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-
-    y_true = test_df['MDM'].to_list()
-    y_pred = predict_list
-    # Compute metrics
-    accuracy = accuracy_score(y_true, y_pred)
-    f1 = f1_score(y_true, y_pred)
-    precision = precision_score(y_true, y_pred)
-    recall = recall_score(y_true, y_pred)
-
-
-
-    with open("output.txt", "a") as f:
-
-        print(f'Fold: {fold}', file=f)
-        print(f'Best threshold: {best_threshold}', file=f)
-        # Print the results
-        print(f'Accuracy: {accuracy:.5f}', file=f)
-        print(f'F1 Score: {f1:.5f}', file=f)
-        print(f'Precision: {precision:.5f}', file=f)
-        print(f'Recall: {recall:.5f}', file=f)
-
-
-
-# %%
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    print(fold)
-    run_similarity_classifier(fold)

relevant_class/similarity_classifier_desc_unit/utils.py

@@ -1,81 +0,0 @@
-import torch
-from transformers import (
-    AutoTokenizer,
-    AutoModelForSequenceClassification,
-    DataCollatorWithPadding,
-)
-import torch.nn.functional as F
-
-
-
-class Retriever:
-    def __init__(self, input_texts, model_checkpoint):
-        # we need to generate the embedding from list of input strings
-        self.embeddings = []
-        self.inputs = input_texts
-        model_checkpoint = model_checkpoint 
-        self.tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-
-        model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint)
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        # device = "cpu"
-        model.to(self.device)
-        self.model = model.eval()
-
-
-    def make_embedding(self, batch_size=64):
-        all_embeddings = self.embeddings
-        input_texts = self.inputs
-
-        for i in range(0, len(input_texts), batch_size):
-            batch_texts = input_texts[i:i+batch_size]
-            # Tokenize the input text
-            inputs = self.tokenizer(batch_texts, return_tensors="pt", padding=True, truncation=True, max_length=64)
-            input_ids = inputs.input_ids.to(self.device)
-            attention_mask = inputs.attention_mask.to(self.device)
-
-
-            # Pass the input through the encoder and retrieve the embeddings
-            with torch.no_grad():
-                encoder_outputs = self.model(input_ids, attention_mask=attention_mask, output_hidden_states=True)
-                # get last layer
-                embeddings = encoder_outputs.hidden_states[-1]
-                # get cls token embedding
-                cls_embeddings = embeddings[:, 0, :]  # Shape: (batch_size, hidden_size)
-                all_embeddings.append(cls_embeddings)
-        
-        # remove the batch list and makes a single large tensor, dim=0 increases row-wise
-        all_embeddings = torch.cat(all_embeddings, dim=0)
-
-        self.embeddings = all_embeddings
-
-def cosine_similarity_chunked(batch1, batch2, chunk_size=1024):
-    device = 'cuda'
-    batch1_size = batch1.size(0)
-    batch2_size = batch2.size(0)
-    batch2.to(device)
-    
-    # Prepare an empty tensor to store results
-    cos_sim = torch.empty(batch1_size, batch2_size, device=device)
-
-    # Process batch1 in chunks
-    for i in range(0, batch1_size, chunk_size):
-        batch1_chunk = batch1[i:i + chunk_size]  # Get chunk of batch1
-        
-        batch1_chunk.to(device)
-        # Expand batch1 chunk and entire batch2 for comparison
-        # batch1_chunk_exp = batch1_chunk.unsqueeze(1)  # Shape: (chunk_size, 1, seq_len)
-        # batch2_exp = batch2.unsqueeze(0)  # Shape: (1, batch2_size, seq_len)
-        batch2_norms = batch2.norm(dim=1, keepdim=True)
-
-        
-        # Compute cosine similarity for the chunk and store it in the final tensor
-        # cos_sim[i:i + chunk_size] = F.cosine_similarity(batch1_chunk_exp, batch2_exp, dim=-1)
-
-        # Compute cosine similarity by matrix multiplication and normalizing
-        sim_chunk = torch.mm(batch1_chunk, batch2.T) / (batch1_chunk.norm(dim=1, keepdim=True) * batch2_norms.T + 1e-8)
-        
-        # Store the results in the appropriate part of the final tensor
-        cos_sim[i:i + chunk_size] = sim_chunk
-    
-    return cos_sim

train/classification_bert_complete_desc/train.py

@@ -176,9 +176,9 @@ def train(fold):
         logging_strategy="epoch",
         # save_strategy="epoch",
         load_best_model_at_end=False,
-        learning_rate=1e-3,
+        learning_rate=1e-5,
-        per_device_train_batch_size=64,
+        per_device_train_batch_size=128,
-        per_device_eval_batch_size=64,
+        per_device_eval_batch_size=128,
         auto_find_batch_size=False,
         ddp_find_unused_parameters=False,
         weight_decay=0.01,

train/classification_bert_complete_desc_unit/train.py

@@ -178,8 +178,8 @@ def train(fold):
         # save_strategy="epoch",
         load_best_model_at_end=False,
         learning_rate=1e-5,
-        per_device_train_batch_size=64,
+        per_device_train_batch_size=128,
-        per_device_eval_batch_size=64,
+        per_device_eval_batch_size=128,
         auto_find_batch_size=False,
         ddp_find_unused_parameters=False,
         weight_decay=0.01,

train/frozen_t5_decoder/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

train/frozen_t5_decoder/custom_t5/.gitignore

@@ -1 +0,0 @@
-__pycache__

train/frozen_t5_decoder/custom_t5/modeling_t5.py

@@ -1,125 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Optional, Tuple, Union
-
-import torch
-import torch.utils.checkpoint
-from torch import nn
-from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
-
-from transformers import (
-    T5PreTrainedModel,
-    T5Model
-    
-)
-
-from transformers.modeling_outputs import (
-    SequenceClassifierOutput,
-)
-
-def mean_pooling(encoder_outputs, attention_mask):
-    """
-    Perform mean pooling over encoder outputs, considering the attention mask.
-    """
-    hidden_states = encoder_outputs.last_hidden_state  # Shape: (batch_size, seq_length, hidden_size)
-    mask = attention_mask.unsqueeze(-1)  # Shape: (batch_size, seq_length, 1)
-    masked_hidden_states = hidden_states * mask  # Zero out padding tokens
-    sum_hidden_states = masked_hidden_states.sum(dim=1)  # Sum over sequence length
-    sum_mask = mask.sum(dim=1)  # Sum the mask (number of non-padding tokens)
-    return sum_hidden_states / sum_mask  # Mean pooling
-
-
-class T5EncoderForSequenceClassification(T5PreTrainedModel):
-
-    def __init__(self, checkpoint, tokenizer, config, num_labels):
-        super().__init__(config)
-        self.num_labels = num_labels
-        self.config = config
-
-        # we force the loading of a pre-trained model here
-        self.t5 = T5Model.from_pretrained(checkpoint)
-        self.t5.resize_token_embeddings(len(tokenizer))
-        classifier_dropout = (
-            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
-        )
-        self.dropout = nn.Dropout(classifier_dropout)
-        self.classifier = nn.Linear(config.hidden_size, self.num_labels)
-
-
-    def forward(
-        self,
-        input_ids: Optional[torch.Tensor] = None,
-        attention_mask: Optional[torch.Tensor] = None,
-        token_type_ids: Optional[torch.Tensor] = None,
-        position_ids: Optional[torch.Tensor] = None,
-        head_mask: Optional[torch.Tensor] = None,
-        inputs_embeds: Optional[torch.Tensor] = None,
-        labels: Optional[torch.Tensor] = None,
-        output_attentions: Optional[bool] = None,
-        output_hidden_states: Optional[bool] = None,
-        return_dict: Optional[bool] = None,
-    ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]:
-        r"""
-        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
-            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
-            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
-            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
-        """
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
-
-        # encoder_outputs = self.t5.encoder(
-        #     input_ids,
-        #     attention_mask=attention_mask,
-        #     head_mask=head_mask,
-        #     inputs_embeds=inputs_embeds,
-        #     output_attentions=output_attentions,
-        #     output_hidden_states=output_hidden_states,
-        #     return_dict=return_dict,
-        # )
-
-
-        encoder_outputs = self.t5.encoder(input_ids, attention_mask=attention_mask)
-        # last_hidden_state = encoder_outputs.last_hidden_state
-        # use mean of hidden state
-        # pooled_output = mean_pooling(encoder_outputs, attention_mask)
-
-        # Use the hidden state of the first token as the sequence representation
-        pooled_output = encoder_outputs.last_hidden_state[:, 0, :]  # Shape: (batch_size, hidden_size)
-
-        # pooled_output = encoder_outputs[1]
-
-        pooled_output = self.dropout(pooled_output)
-        logits = self.classifier(pooled_output)
-
-        loss = None
-        if labels is not None:
-            if self.config.problem_type is None:
-                if self.num_labels == 1:
-                    self.config.problem_type = "regression"
-                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
-                    self.config.problem_type = "single_label_classification"
-                else:
-                    self.config.problem_type = "multi_label_classification"
-
-            if self.config.problem_type == "regression":
-                loss_fct = MSELoss()
-                if self.num_labels == 1:
-                    loss = loss_fct(logits.squeeze(), labels.squeeze())
-                else:
-                    loss = loss_fct(logits, labels)
-            elif self.config.problem_type == "single_label_classification":
-                loss_fct = CrossEntropyLoss()
-                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
-            elif self.config.problem_type == "multi_label_classification":
-                loss_fct = BCEWithLogitsLoss()
-                loss = loss_fct(logits, labels)
-        if not return_dict:
-            output = (logits,) + encoder_outputs[2:]
-            return ((loss,) + output) if loss is not None else output
-
-        return SequenceClassifierOutput(
-            loss=loss,
-            logits=logits,
-            hidden_states=encoder_outputs.hidden_states,
-            attentions=encoder_outputs.attentions,
-        )

train/frozen_t5_decoder/mapping_prediction/.gitignore

@@ -1,2 +0,0 @@
-__pycache__
-exports/

train/frozen_t5_decoder/mapping_prediction/inference.py

@@ -1,168 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-)
-import os
-from tqdm import tqdm
-from datasets import Dataset
-import numpy as np
-
-os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-
-class Inference():
-    tokenizer: T5TokenizerFast
-    model: torch.nn.Module
-    dataloader: DataLoader
-
-    def __init__(self, checkpoint_path):
-        self._create_tokenizer()
-        self._load_model(checkpoint_path)
-
-
-    def _create_tokenizer(self):
-        # %%
-        # load tokenizer
-        self.tokenizer = T5TokenizerFast.from_pretrained("t5-small", return_tensors="pt", clean_up_tokenization_spaces=True)
-        # Define additional special tokens
-        additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "SIG", "UNIT", "DATA_TYPE"]
-        # Add the additional special tokens to the tokenizer
-        self.tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    def _load_model(self, checkpoint_path: str):
-        # load model
-        # Define the directory and the pattern
-        model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)
-        model = torch.compile(model)
-        # set model to eval
-        self.model = model.eval()
-
-
-
-
-    def prepare_dataloader(self, input_df, batch_size, max_length):
-        """
-        *arguments*
-        - input_df: input dataframe containing fields 'tag_description', 'thing', 'property'
-        - batch_size: the batch size of dataloader output
-        - max_length: length of tokenizer output
-        """
-        print("preparing dataloader")
-        # convert each dataframe row into a dictionary
-        # outputs a list of dictionaries
-
-        def _process_df(df):
-            output_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = {
-                    'input' : f"{desc}{unit}",
-                    'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-                }
-                output_list.append(element)
-
-            return output_list
-
-        def _preprocess_function(example):
-            input = example['input']
-            target = example['output']
-            # text_target sets the corresponding label to inputs
-            # there is no need to create a separate 'labels'
-            model_inputs = self.tokenizer(
-                input,
-                text_target=target, 
-                max_length=max_length,
-                return_tensors="pt",
-                padding="max_length",
-                truncation=True,
-            )
-            return model_inputs
-
-        test_dataset = Dataset.from_list(_process_df(input_df))
-
-
-        # map maps function to each "row" in the dataset
-        # aka the data in the immediate nesting
-        datasets = test_dataset.map(
-            _preprocess_function,
-            batched=True,
-            num_proc=1,
-            remove_columns=test_dataset.column_names,
-        )
-        # datasets = _preprocess_function(test_dataset)
-        datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
-
-        # create dataloader
-        self.dataloader = DataLoader(datasets, batch_size=batch_size)
-
-
-    def generate(self):
-        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        MAX_GENERATE_LENGTH = 128
-
-        pred_generations = []
-        pred_labels = []
-
-        print("start generation")
-        for batch in tqdm(self.dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            pred_labels.extend(batch['labels'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-            self.model.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                outputs = self.model.generate(input_ids,
-                                        attention_mask=attention_mask,
-                                        max_length=MAX_GENERATE_LENGTH)
-                
-                # Decode the output and print the results
-                pred_generations.extend(outputs.to("cpu"))
-
-
-
-        # %%
-        # extract sequence and decode
-        def extract_seq(tokens, start_value, end_value):
-            if start_value not in tokens or end_value not in tokens:
-                return None  # Or handle this case according to your requirements
-            start_id = np.where(tokens == start_value)[0][0]
-            end_id = np.where(tokens == end_value)[0][0]
-
-            return tokens[start_id+1:end_id]
-
-
-        def process_tensor_output(tokens):
-            thing_seq = extract_seq(tokens, 32100, 32101) # 32100 = <THING_START>, 32101 = <THING_END>
-            property_seq = extract_seq(tokens, 32102, 32103) # 32102 = <PROPERTY_START>, 32103 = <PROPERTY_END>
-            p_thing = None
-            p_property = None
-            if (thing_seq is not None):
-                p_thing =  self.tokenizer.decode(thing_seq, skip_special_tokens=False)
-            if (property_seq is not None):
-                p_property =  self.tokenizer.decode(property_seq, skip_special_tokens=False)
-            return p_thing, p_property
-
-        # decode prediction labels
-        def decode_preds(tokens_list):
-            thing_prediction_list = []
-            property_prediction_list = []
-            for tokens in tokens_list:
-                p_thing, p_property = process_tensor_output(tokens)
-                thing_prediction_list.append(p_thing)
-                property_prediction_list.append(p_property)
-            return thing_prediction_list, property_prediction_list 
-
-        thing_prediction_list, property_prediction_list = decode_preds(pred_generations)
-        return thing_prediction_list, property_prediction_list
-

train/frozen_t5_decoder/mapping_prediction/output.txt

@@ -1,2 +0,0 @@
-
-Accuracy for fold 1: 0.0

train/frozen_t5_decoder/mapping_prediction/predict.py

@@ -1,74 +0,0 @@
-
-import pandas as pd
-import os
-import glob
-from inference import Inference
-
-checkpoint_directory =  '../'
-
-BATCH_SIZE = 512
-
-def infer_and_select(fold):
-    print(f"Inference for fold {fold}")
-    # import test data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
-
-    # get target data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-    # processing to help with selection later
-    train_df['thing_property'] = train_df['thing'] + " " + train_df['property']
-
-
-    ##########################################
-    # run inference
-    # checkpoint
-    # Use glob to find matching paths
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}b')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-
-    infer = Inference(checkpoint_path)
-    infer.prepare_dataloader(df, batch_size=BATCH_SIZE, max_length=128)
-    thing_prediction_list, property_prediction_list = infer.generate()
-
-    # add labels too
-    # thing_actual_list, property_actual_list = decode_preds(pred_labels)
-    # Convert the list to a Pandas DataFrame
-    df_out = pd.DataFrame({
-        'p_thing': thing_prediction_list, 
-        'p_property': property_prediction_list
-    })
-    # df_out['p_thing_correct'] = df_out['p_thing'] == df_out['thing']
-    # df_out['p_property_correct'] = df_out['p_property'] == df_out['property']
-    df = pd.concat([df, df_out], axis=1)
-
-    # we can save the t5 generation output here
-    df.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-    # here we want to evaluate mapping accuracy within the valid in mdm data only
-    in_mdm = df['MDM']
-    condition_correct_thing = df['p_thing'] == df['thing']
-    condition_correct_property = df['p_property'] == df['property']
-    prediction_mdm_correct = sum(condition_correct_thing & condition_correct_property & in_mdm)
-    pred_correct_proportion = prediction_mdm_correct/sum(in_mdm)
-
-    # write output to file output.txt
-    with open("output.txt", "a") as f:
-        print(f'Accuracy for fold {fold}: {pred_correct_proportion}', file=f)
-
-###########################################  
-# Execute for all folds
-
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1]:
-    infer_and_select(fold)

train/frozen_t5_decoder/train_decoder.py

@@ -1,236 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-from custom_t5.modeling_t5 import T5EncoderForSequenceClassification
-
-from safetensors.torch import load_file
-from transformers import (
-    T5Config,
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    EarlyStoppingCallback,
-    Seq2SeqTrainingArguments,
-    T5ForConditionalGeneration,
-    T5Model
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-# outputs a list of dictionaries
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        element = {
-            'input' : f"{desc}{unit}",
-            'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# function to perform training for a given fold
-def train(fold):
-    save_path = f'checkpoint_fold_{fold}b'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-    model_checkpoint = "t5-small"
-    tokenizer = T5TokenizerFast.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['input']
-        target = example['output']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            text_target=target, 
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns=split_datasets["train"].column_names,
-    )
-
-    # https://github.com/huggingface/transformers/pull/28414
-    # model_checkpoint = "google/t5-efficient-tiny"
-    # device_map set to auto to force it to load contiguous weights 
-    # model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, device_map='auto')
-
-    # directory = os.path.join(".", f'checkpoint_fold_{fold}a')
-    # # Use glob to find matching paths
-    # # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # # we are guaranteed to save only 1 checkpoint from training
-    # pattern = 'checkpoint-*'
-    # prev_checkpoint = glob.glob(os.path.join(directory, pattern))[0]
-    # # t5_classify = T5Model.from_pretrained(prev_checkpoint)
-    # # Load the checkpoint
-    # checkpoint_path = f"{prev_checkpoint}/model.safetensors"
-    # checkpoint = load_file(checkpoint_path)
-    # Filter out weights related to the classification head
-    # given name format: t5.encoder.embed_tokens.weight
-    # we want: encoder.embed.tokens.weight
-    # t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
-
-    model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
-    # model.load_state_dict(state_dict=t5_weights, strict=False)
-
-    # for key, param in model.state_dict().items():
-    #     if key in t5_weights:
-    #         print(f"{key}: Successfully overridden")
-    #     else:
-    #         print(f"{key}: Retained original weights")
-
-
-    # Freeze the decoder
-    for param in model.decoder.parameters():
-        param.requires_grad = False
-
-    # Freeze the shared embedding layer
-    for param in model.shared.parameters():
-        param.requires_grad = False
-
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-    metric = evaluate.load("sacrebleu")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        # In case the model returns more than the prediction logits
-        if isinstance(preds, tuple):
-            preds = preds[0]
-
-        decoded_preds = tokenizer.batch_decode(preds, 
-                                            skip_special_tokens=False)
-
-        # Replace -100s in the labels as we can't decode them
-        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-        decoded_labels = tokenizer.batch_decode(labels,
-                                                skip_special_tokens=False)
-
-        # Remove <PAD> tokens from decoded predictions and labels
-        decoded_preds = [pred.replace(tokenizer.pad_token, '').strip() for pred in decoded_preds]
-        decoded_labels = [[label.replace(tokenizer.pad_token, '').strip()] for label in decoded_labels]
-
-        # Some simple post-processing
-        # decoded_preds = [pred.strip() for pred in decoded_preds]
-        # decoded_labels = [[label.strip()] for label in decoded_labels]
-        # print(decoded_preds, decoded_labels)
-
-        result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-        return {"bleu": result["score"]}
-
-
-    # Generation Config
-    # from transformers import GenerationConfig
-    gen_config = model.generation_config
-    gen_config.max_length = 128
-
-    # compile
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # Trainer
-
-    args = Seq2SeqTrainingArguments(
-        f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=40,
-        predict_with_generate=True,
-        bf16=True,
-        push_to_hub=False,
-        generation_config=gen_config,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model,
-        args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1]:
-    print(fold)
-    train(fold)
-

train/frozen_t5_encoder/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

train/frozen_t5_encoder/mapping_prediction/.gitignore

@@ -1,2 +0,0 @@
-__pycache__
-exports/

train/frozen_t5_encoder/mapping_prediction/inference.py

@@ -1,168 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-)
-import os
-from tqdm import tqdm
-from datasets import Dataset
-import numpy as np
-
-os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-
-class Inference():
-    tokenizer: T5TokenizerFast
-    model: torch.nn.Module
-    dataloader: DataLoader
-
-    def __init__(self, checkpoint_path):
-        self._create_tokenizer()
-        self._load_model(checkpoint_path)
-
-
-    def _create_tokenizer(self):
-        # %%
-        # load tokenizer
-        self.tokenizer = T5TokenizerFast.from_pretrained("t5-small", return_tensors="pt", clean_up_tokenization_spaces=True)
-        # Define additional special tokens
-        additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "SIG", "UNIT", "DATA_TYPE"]
-        # Add the additional special tokens to the tokenizer
-        self.tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    def _load_model(self, checkpoint_path: str):
-        # load model
-        # Define the directory and the pattern
-        model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)
-        model = torch.compile(model)
-        # set model to eval
-        self.model = model.eval()
-
-
-
-
-    def prepare_dataloader(self, input_df, batch_size, max_length):
-        """
-        *arguments*
-        - input_df: input dataframe containing fields 'tag_description', 'thing', 'property'
-        - batch_size: the batch size of dataloader output
-        - max_length: length of tokenizer output
-        """
-        print("preparing dataloader")
-        # convert each dataframe row into a dictionary
-        # outputs a list of dictionaries
-
-        def _process_df(df):
-            output_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = {
-                    'input' : f"{desc}{unit}",
-                    'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-                }
-                output_list.append(element)
-
-            return output_list
-
-        def _preprocess_function(example):
-            input = example['input']
-            target = example['output']
-            # text_target sets the corresponding label to inputs
-            # there is no need to create a separate 'labels'
-            model_inputs = self.tokenizer(
-                input,
-                text_target=target, 
-                max_length=max_length,
-                return_tensors="pt",
-                padding="max_length",
-                truncation=True,
-            )
-            return model_inputs
-
-        test_dataset = Dataset.from_list(_process_df(input_df))
-
-
-        # map maps function to each "row" in the dataset
-        # aka the data in the immediate nesting
-        datasets = test_dataset.map(
-            _preprocess_function,
-            batched=True,
-            num_proc=1,
-            remove_columns=test_dataset.column_names,
-        )
-        # datasets = _preprocess_function(test_dataset)
-        datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
-
-        # create dataloader
-        self.dataloader = DataLoader(datasets, batch_size=batch_size)
-
-
-    def generate(self):
-        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        MAX_GENERATE_LENGTH = 128
-
-        pred_generations = []
-        pred_labels = []
-
-        print("start generation")
-        for batch in tqdm(self.dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            pred_labels.extend(batch['labels'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-            self.model.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                outputs = self.model.generate(input_ids,
-                                        attention_mask=attention_mask,
-                                        max_length=MAX_GENERATE_LENGTH)
-                
-                # Decode the output and print the results
-                pred_generations.extend(outputs.to("cpu"))
-
-
-
-        # %%
-        # extract sequence and decode
-        def extract_seq(tokens, start_value, end_value):
-            if start_value not in tokens or end_value not in tokens:
-                return None  # Or handle this case according to your requirements
-            start_id = np.where(tokens == start_value)[0][0]
-            end_id = np.where(tokens == end_value)[0][0]
-
-            return tokens[start_id+1:end_id]
-
-
-        def process_tensor_output(tokens):
-            thing_seq = extract_seq(tokens, 32100, 32101) # 32100 = <THING_START>, 32101 = <THING_END>
-            property_seq = extract_seq(tokens, 32102, 32103) # 32102 = <PROPERTY_START>, 32103 = <PROPERTY_END>
-            p_thing = None
-            p_property = None
-            if (thing_seq is not None):
-                p_thing =  self.tokenizer.decode(thing_seq, skip_special_tokens=False)
-            if (property_seq is not None):
-                p_property =  self.tokenizer.decode(property_seq, skip_special_tokens=False)
-            return p_thing, p_property
-
-        # decode prediction labels
-        def decode_preds(tokens_list):
-            thing_prediction_list = []
-            property_prediction_list = []
-            for tokens in tokens_list:
-                p_thing, p_property = process_tensor_output(tokens)
-                thing_prediction_list.append(p_thing)
-                property_prediction_list.append(p_property)
-            return thing_prediction_list, property_prediction_list 
-
-        thing_prediction_list, property_prediction_list = decode_preds(pred_generations)
-        return thing_prediction_list, property_prediction_list
-

train/frozen_t5_encoder/mapping_prediction/output.txt

@@ -1,6 +0,0 @@
-
-Accuracy for fold 1: 0.9342167534311405
-Accuracy for fold 2: 0.883177570093458
-Accuracy for fold 3: 0.963855421686747
-Accuracy for fold 4: 0.9705042816365367
-Accuracy for fold 5: 0.9051763628034815

train/frozen_t5_encoder/mapping_prediction/predict.py

@@ -1,74 +0,0 @@
-
-import pandas as pd
-import os
-import glob
-from inference import Inference
-
-checkpoint_directory =  '../'
-
-BATCH_SIZE = 512
-
-def infer_and_select(fold):
-    print(f"Inference for fold {fold}")
-    # import test data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
-
-    # get target data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-    # processing to help with selection later
-    train_df['thing_property'] = train_df['thing'] + " " + train_df['property']
-
-
-    ##########################################
-    # run inference
-    # checkpoint
-    # Use glob to find matching paths
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}b')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-
-    infer = Inference(checkpoint_path)
-    infer.prepare_dataloader(df, batch_size=BATCH_SIZE, max_length=128)
-    thing_prediction_list, property_prediction_list = infer.generate()
-
-    # add labels too
-    # thing_actual_list, property_actual_list = decode_preds(pred_labels)
-    # Convert the list to a Pandas DataFrame
-    df_out = pd.DataFrame({
-        'p_thing': thing_prediction_list, 
-        'p_property': property_prediction_list
-    })
-    # df_out['p_thing_correct'] = df_out['p_thing'] == df_out['thing']
-    # df_out['p_property_correct'] = df_out['p_property'] == df_out['property']
-    df = pd.concat([df, df_out], axis=1)
-
-    # we can save the t5 generation output here
-    df.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-    # here we want to evaluate mapping accuracy within the valid in mdm data only
-    in_mdm = df['MDM']
-    condition_correct_thing = df['p_thing'] == df['thing']
-    condition_correct_property = df['p_property'] == df['property']
-    prediction_mdm_correct = sum(condition_correct_thing & condition_correct_property & in_mdm)
-    pred_correct_proportion = prediction_mdm_correct/sum(in_mdm)
-
-    # write output to file output.txt
-    with open("output.txt", "a") as f:
-        print(f'Accuracy for fold {fold}: {pred_correct_proportion}', file=f)
-
-###########################################  
-# Execute for all folds
-
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    infer_and_select(fold)

train/frozen_t5_encoder/train_decoder.py

@@ -1,235 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-
-from safetensors.torch import load_file
-from transformers import (
-    T5Config,
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    EarlyStoppingCallback,
-    Seq2SeqTrainingArguments,
-    T5ForConditionalGeneration,
-    T5Model
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-# outputs a list of dictionaries
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        element = {
-            'input' : f"{desc}{unit}",
-            'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# function to perform training for a given fold
-def train(fold):
-    save_path = f'checkpoint_fold_{fold}b'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-    model_checkpoint = "t5-small"
-    tokenizer = T5TokenizerFast.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['input']
-        target = example['output']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            text_target=target, 
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns=split_datasets["train"].column_names,
-    )
-
-    # https://github.com/huggingface/transformers/pull/28414
-    # model_checkpoint = "google/t5-efficient-tiny"
-    # device_map set to auto to force it to load contiguous weights 
-    # model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, device_map='auto')
-
-    # directory = os.path.join(".", f'checkpoint_fold_{fold}a')
-    # # Use glob to find matching paths
-    # # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # # we are guaranteed to save only 1 checkpoint from training
-    # pattern = 'checkpoint-*'
-    # prev_checkpoint = glob.glob(os.path.join(directory, pattern))[0]
-    # # t5_classify = T5Model.from_pretrained(prev_checkpoint)
-    # # Load the checkpoint
-    # checkpoint_path = f"{prev_checkpoint}/model.safetensors"
-    # checkpoint = load_file(checkpoint_path)
-    # # Filter out weights related to the classification head
-    # # given name format: t5.encoder.embed_tokens.weight
-    # # we want: encoder.embed.tokens.weight
-    # t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
-
-    model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
-    # model.load_state_dict(state_dict=t5_weights, strict=False)
-
-    # for key, param in model.state_dict().items():
-    #     if key in t5_weights:
-    #         print(f"{key}: Successfully overridden")
-    #     else:
-    #         print(f"{key}: Retained original weights")
-
-
-    # Freeze the encoder
-    for param in model.encoder.parameters():
-        param.requires_grad = False
-
-    # Freeze the shared embedding layer
-    for param in model.shared.parameters():
-        param.requires_grad = False
-
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-    metric = evaluate.load("sacrebleu")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        # In case the model returns more than the prediction logits
-        if isinstance(preds, tuple):
-            preds = preds[0]
-
-        decoded_preds = tokenizer.batch_decode(preds, 
-                                            skip_special_tokens=False)
-
-        # Replace -100s in the labels as we can't decode them
-        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-        decoded_labels = tokenizer.batch_decode(labels,
-                                                skip_special_tokens=False)
-
-        # Remove <PAD> tokens from decoded predictions and labels
-        decoded_preds = [pred.replace(tokenizer.pad_token, '').strip() for pred in decoded_preds]
-        decoded_labels = [[label.replace(tokenizer.pad_token, '').strip()] for label in decoded_labels]
-
-        # Some simple post-processing
-        # decoded_preds = [pred.strip() for pred in decoded_preds]
-        # decoded_labels = [[label.strip()] for label in decoded_labels]
-        # print(decoded_preds, decoded_labels)
-
-        result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-        return {"bleu": result["score"]}
-
-
-    # Generation Config
-    # from transformers import GenerationConfig
-    gen_config = model.generation_config
-    gen_config.max_length = 128
-
-    # compile
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # Trainer
-
-    args = Seq2SeqTrainingArguments(
-        f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=40,
-        predict_with_generate=True,
-        bf16=True,
-        push_to_hub=False,
-        generation_config=gen_config,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model,
-        args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-

train/hybrid_t5_complete_desc_unit/mapping_prediction/output.txt

@@ -1,6 +1,6 @@
 
-Accuracy for fold 1: 0.9242782773308093
+Accuracy for fold 1: 0.9427354472314246
-Accuracy for fold 2: 0.9126168224299065
+Accuracy for fold 2: 0.8859813084112149
-Accuracy for fold 3: 0.9643574297188755
+Accuracy for fold 3: 0.9683734939759037
-Accuracy for fold 4: 0.9595623215984777
+Accuracy for fold 4: 0.9762131303520457
-Accuracy for fold 5: 0.8950984883188273
+Accuracy for fold 5: 0.907924874026569

train/hybrid_t5_complete_desc_unit/mapping_prediction/predict.py

@@ -13,7 +13,6 @@ def infer_and_select(fold):
     # import test data
     data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
     df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
 
     # get target data
     data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"

train/hybrid_t5_complete_desc_unit/train_decoder.py

@@ -120,22 +120,13 @@ def train(fold):
     checkpoint_path = f"{prev_checkpoint}/model.safetensors"
     checkpoint = load_file(checkpoint_path)
     # Filter out weights related to the classification head
-    # given name format: t5.encoder.embed_tokens.weight
+    t5_weights = {key: value for key, value in checkpoint.items() if "classifier" not in key}
-    # we want: encoder.embed.tokens.weight
+
-    t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
 
     model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
     model.load_state_dict(state_dict=t5_weights, strict=False)
-
+    # important! after extending tokens vocab
-    for key, param in model.state_dict().items():
+    model.resize_token_embeddings(len(tokenizer))
-        if key in t5_weights:
-            print(f"{key}: Successfully overridden")
-        else:
-            print(f"{key}: Retained original weights")
-
 
     # Freeze the encoder
     for param in model.encoder.parameters():
@@ -203,7 +194,7 @@ def train(fold):
         ddp_find_unused_parameters=False,
         weight_decay=0.01,
         save_total_limit=1,
-        num_train_epochs=40,
+        num_train_epochs=80,
         predict_with_generate=True,
         bf16=True,
         push_to_hub=False,

train/hybrid_t5_complete_desc_unit/train_encoder.py

@@ -35,13 +35,12 @@ torch.set_float32_matmul_precision('high')
 # import the full mdm-only file
 data_path = '../../data_import/exports/data_mapping_mdm.csv'
 full_df = pd.read_csv(data_path, skipinitialspace=True)
-# mdm_list = sorted(list((set(full_df['pattern']))))
+mdm_list = sorted(list((set(full_df['pattern']))))
 
 # # rather than use pattern, we use the real thing and property
-thing_property = full_df['thing'] + full_df['property']
+# thing_property = full_df['thing'] + full_df['property']
-thing_property = thing_property.to_list()
+# thing_property = thing_property.to_list()
-mdm_list = sorted(list(set(thing_property)))
+# mdm_list = sorted(list(set(thing_property)))
-print("number of classes: ", len(mdm_list))
 
 
 # %%
@@ -63,8 +62,8 @@ def process_df_to_dict(df, mdm_list):
     for _, row in df.iterrows():
         desc = f"<DESC>{row['tag_description']}<DESC>"
         unit = f"<UNIT>{row['unit']}<UNIT>"
-        pattern = f"{row['thing'] + row['property']}"
+        # pattern = f"{row['thing'] + row['property']}"
-        # pattern = f"{row['thing_pattern'] + ' ' + row['property_pattern']}"
+        pattern = f"{row['thing_pattern'] + ' ' + row['property_pattern']}"
         try:
             index = mdm_list.index(pattern)
         except ValueError:
@@ -138,7 +137,7 @@ def train(fold):
         remove_columns="text",
     )
 
-    # %% temp    # t5_classify = T5Model.from_pretrained(prev_checkpoint)
+    # %% temp
     # tokenized_datasets['train'].rename_columns()
 
     # %%
@@ -190,10 +189,10 @@ def train(fold):
         # save_strategy="epoch",
         load_best_model_at_end=False,
         learning_rate=1e-3,
-        per_device_train_batch_size=64,
+        per_device_train_batch_size=128,
-        per_device_eval_batch_size=64,
+        per_device_eval_batch_size=128,
         auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,    # t5_classify = T5Model.from_pretrained(prev_checkpoint)
+        ddp_find_unused_parameters=False,
         weight_decay=0.01,
         save_total_limit=1,
         num_train_epochs=80,

train/hybrid_t5_pattern_desc_unit/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

train/hybrid_t5_pattern_desc_unit/custom_t5/.gitignore

@@ -1 +0,0 @@
-__pycache__

train/hybrid_t5_pattern_desc_unit/custom_t5/modeling_t5.py

@@ -1,125 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Optional, Tuple, Union
-
-import torch
-import torch.utils.checkpoint
-from torch import nn
-from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
-
-from transformers import (
-    T5PreTrainedModel,
-    T5Model
-    
-)
-
-from transformers.modeling_outputs import (
-    SequenceClassifierOutput,
-)
-
-def mean_pooling(encoder_outputs, attention_mask):
-    """
-    Perform mean pooling over encoder outputs, considering the attention mask.
-    """
-    hidden_states = encoder_outputs.last_hidden_state  # Shape: (batch_size, seq_length, hidden_size)
-    mask = attention_mask.unsqueeze(-1)  # Shape: (batch_size, seq_length, 1)
-    masked_hidden_states = hidden_states * mask  # Zero out padding tokens
-    sum_hidden_states = masked_hidden_states.sum(dim=1)  # Sum over sequence length
-    sum_mask = mask.sum(dim=1)  # Sum the mask (number of non-padding tokens)
-    return sum_hidden_states / sum_mask  # Mean pooling
-
-
-class T5EncoderForSequenceClassification(T5PreTrainedModel):
-
-    def __init__(self, checkpoint, tokenizer, config, num_labels):
-        super().__init__(config)
-        self.num_labels = num_labels
-        self.config = config
-
-        # we force the loading of a pre-trained model here
-        self.t5 = T5Model.from_pretrained(checkpoint)
-        self.t5.resize_token_embeddings(len(tokenizer))
-        classifier_dropout = (
-            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
-        )
-        self.dropout = nn.Dropout(classifier_dropout)
-        self.classifier = nn.Linear(config.hidden_size, self.num_labels)
-
-
-    def forward(
-        self,
-        input_ids: Optional[torch.Tensor] = None,
-        attention_mask: Optional[torch.Tensor] = None,
-        token_type_ids: Optional[torch.Tensor] = None,
-        position_ids: Optional[torch.Tensor] = None,
-        head_mask: Optional[torch.Tensor] = None,
-        inputs_embeds: Optional[torch.Tensor] = None,
-        labels: Optional[torch.Tensor] = None,
-        output_attentions: Optional[bool] = None,
-        output_hidden_states: Optional[bool] = None,
-        return_dict: Optional[bool] = None,
-    ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]:
-        r"""
-        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
-            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
-            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
-            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
-        """
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
-
-        # encoder_outputs = self.t5.encoder(
-        #     input_ids,
-        #     attention_mask=attention_mask,
-        #     head_mask=head_mask,
-        #     inputs_embeds=inputs_embeds,
-        #     output_attentions=output_attentions,
-        #     output_hidden_states=output_hidden_states,
-        #     return_dict=return_dict,
-        # )
-
-
-        encoder_outputs = self.t5.encoder(input_ids, attention_mask=attention_mask)
-        # last_hidden_state = encoder_outputs.last_hidden_state
-        # use mean of hidden state
-        # pooled_output = mean_pooling(encoder_outputs, attention_mask)
-
-        # Use the hidden state of the first token as the sequence representation
-        pooled_output = encoder_outputs.last_hidden_state[:, 0, :]  # Shape: (batch_size, hidden_size)
-
-        # pooled_output = encoder_outputs[1]
-
-        pooled_output = self.dropout(pooled_output)
-        logits = self.classifier(pooled_output)
-
-        loss = None
-        if labels is not None:
-            if self.config.problem_type is None:
-                if self.num_labels == 1:
-                    self.config.problem_type = "regression"
-                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
-                    self.config.problem_type = "single_label_classification"
-                else:
-                    self.config.problem_type = "multi_label_classification"
-
-            if self.config.problem_type == "regression":
-                loss_fct = MSELoss()
-                if self.num_labels == 1:
-                    loss = loss_fct(logits.squeeze(), labels.squeeze())
-                else:
-                    loss = loss_fct(logits, labels)
-            elif self.config.problem_type == "single_label_classification":
-                loss_fct = CrossEntropyLoss()
-                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
-            elif self.config.problem_type == "multi_label_classification":
-                loss_fct = BCEWithLogitsLoss()
-                loss = loss_fct(logits, labels)
-        if not return_dict:
-            output = (logits,) + encoder_outputs[2:]
-            return ((loss,) + output) if loss is not None else output
-
-        return SequenceClassifierOutput(
-            loss=loss,
-            logits=logits,
-            hidden_states=encoder_outputs.hidden_states,
-            attentions=encoder_outputs.attentions,
-        )

train/hybrid_t5_pattern_desc_unit/mapping_prediction/.gitignore

@@ -1,2 +0,0 @@
-__pycache__
-exports/

train/hybrid_t5_pattern_desc_unit/mapping_prediction/inference.py

@@ -1,168 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-)
-import os
-from tqdm import tqdm
-from datasets import Dataset
-import numpy as np
-
-os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-
-class Inference():
-    tokenizer: T5TokenizerFast
-    model: torch.nn.Module
-    dataloader: DataLoader
-
-    def __init__(self, checkpoint_path):
-        self._create_tokenizer()
-        self._load_model(checkpoint_path)
-
-
-    def _create_tokenizer(self):
-        # %%
-        # load tokenizer
-        self.tokenizer = T5TokenizerFast.from_pretrained("t5-small", return_tensors="pt", clean_up_tokenization_spaces=True)
-        # Define additional special tokens
-        additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "SIG", "UNIT", "DATA_TYPE"]
-        # Add the additional special tokens to the tokenizer
-        self.tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    def _load_model(self, checkpoint_path: str):
-        # load model
-        # Define the directory and the pattern
-        model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)
-        model = torch.compile(model)
-        # set model to eval
-        self.model = model.eval()
-
-
-
-
-    def prepare_dataloader(self, input_df, batch_size, max_length):
-        """
-        *arguments*
-        - input_df: input dataframe containing fields 'tag_description', 'thing', 'property'
-        - batch_size: the batch size of dataloader output
-        - max_length: length of tokenizer output
-        """
-        print("preparing dataloader")
-        # convert each dataframe row into a dictionary
-        # outputs a list of dictionaries
-
-        def _process_df(df):
-            output_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = {
-                    'input' : f"{desc}{unit}",
-                    'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-                }
-                output_list.append(element)
-
-            return output_list
-
-        def _preprocess_function(example):
-            input = example['input']
-            target = example['output']
-            # text_target sets the corresponding label to inputs
-            # there is no need to create a separate 'labels'
-            model_inputs = self.tokenizer(
-                input,
-                text_target=target, 
-                max_length=max_length,
-                return_tensors="pt",
-                padding="max_length",
-                truncation=True,
-            )
-            return model_inputs
-
-        test_dataset = Dataset.from_list(_process_df(input_df))
-
-
-        # map maps function to each "row" in the dataset
-        # aka the data in the immediate nesting
-        datasets = test_dataset.map(
-            _preprocess_function,
-            batched=True,
-            num_proc=1,
-            remove_columns=test_dataset.column_names,
-        )
-        # datasets = _preprocess_function(test_dataset)
-        datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
-
-        # create dataloader
-        self.dataloader = DataLoader(datasets, batch_size=batch_size)
-
-
-    def generate(self):
-        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        MAX_GENERATE_LENGTH = 128
-
-        pred_generations = []
-        pred_labels = []
-
-        print("start generation")
-        for batch in tqdm(self.dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            pred_labels.extend(batch['labels'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-            self.model.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                outputs = self.model.generate(input_ids,
-                                        attention_mask=attention_mask,
-                                        max_length=MAX_GENERATE_LENGTH)
-                
-                # Decode the output and print the results
-                pred_generations.extend(outputs.to("cpu"))
-
-
-
-        # %%
-        # extract sequence and decode
-        def extract_seq(tokens, start_value, end_value):
-            if start_value not in tokens or end_value not in tokens:
-                return None  # Or handle this case according to your requirements
-            start_id = np.where(tokens == start_value)[0][0]
-            end_id = np.where(tokens == end_value)[0][0]
-
-            return tokens[start_id+1:end_id]
-
-
-        def process_tensor_output(tokens):
-            thing_seq = extract_seq(tokens, 32100, 32101) # 32100 = <THING_START>, 32101 = <THING_END>
-            property_seq = extract_seq(tokens, 32102, 32103) # 32102 = <PROPERTY_START>, 32103 = <PROPERTY_END>
-            p_thing = None
-            p_property = None
-            if (thing_seq is not None):
-                p_thing =  self.tokenizer.decode(thing_seq, skip_special_tokens=False)
-            if (property_seq is not None):
-                p_property =  self.tokenizer.decode(property_seq, skip_special_tokens=False)
-            return p_thing, p_property
-
-        # decode prediction labels
-        def decode_preds(tokens_list):
-            thing_prediction_list = []
-            property_prediction_list = []
-            for tokens in tokens_list:
-                p_thing, p_property = process_tensor_output(tokens)
-                thing_prediction_list.append(p_thing)
-                property_prediction_list.append(p_property)
-            return thing_prediction_list, property_prediction_list 
-
-        thing_prediction_list, property_prediction_list = decode_preds(pred_generations)
-        return thing_prediction_list, property_prediction_list
-

train/hybrid_t5_pattern_desc_unit/mapping_prediction/output.txt

@@ -1,6 +0,0 @@
-
-Accuracy for fold 1: 0.9394226218646474
-Accuracy for fold 2: 0.9107476635514019
-Accuracy for fold 3: 0.9548192771084337
-Accuracy for fold 4: 0.972882968601332
-Accuracy for fold 5: 0.8996793403573065

train/hybrid_t5_pattern_desc_unit/mapping_prediction/predict.py

@@ -1,74 +0,0 @@
-
-import pandas as pd
-import os
-import glob
-from inference import Inference
-
-checkpoint_directory =  '../'
-
-BATCH_SIZE = 512
-
-def infer_and_select(fold):
-    print(f"Inference for fold {fold}")
-    # import test data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
-
-    # get target data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-    # processing to help with selection later
-    train_df['thing_property'] = train_df['thing'] + " " + train_df['property']
-
-
-    ##########################################
-    # run inference
-    # checkpoint
-    # Use glob to find matching paths
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}b')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-
-    infer = Inference(checkpoint_path)
-    infer.prepare_dataloader(df, batch_size=BATCH_SIZE, max_length=128)
-    thing_prediction_list, property_prediction_list = infer.generate()
-
-    # add labels too
-    # thing_actual_list, property_actual_list = decode_preds(pred_labels)
-    # Convert the list to a Pandas DataFrame
-    df_out = pd.DataFrame({
-        'p_thing': thing_prediction_list, 
-        'p_property': property_prediction_list
-    })
-    # df_out['p_thing_correct'] = df_out['p_thing'] == df_out['thing']
-    # df_out['p_property_correct'] = df_out['p_property'] == df_out['property']
-    df = pd.concat([df, df_out], axis=1)
-
-    # we can save the t5 generation output here
-    df.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-    # here we want to evaluate mapping accuracy within the valid in mdm data only
-    in_mdm = df['MDM']
-    condition_correct_thing = df['p_thing'] == df['thing']
-    condition_correct_property = df['p_property'] == df['property']
-    prediction_mdm_correct = sum(condition_correct_thing & condition_correct_property & in_mdm)
-    pred_correct_proportion = prediction_mdm_correct/sum(in_mdm)
-
-    # write output to file output.txt
-    with open("output.txt", "a") as f:
-        print(f'Accuracy for fold {fold}: {pred_correct_proportion}', file=f)
-
-###########################################  
-# Execute for all folds
-
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    infer_and_select(fold)

train/hybrid_t5_pattern_desc_unit/train_decoder.py

@@ -1,234 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-from custom_t5.modeling_t5 import T5EncoderForSequenceClassification
-
-from safetensors.torch import load_file
-from transformers import (
-    T5Config,
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    EarlyStoppingCallback,
-    Seq2SeqTrainingArguments,
-    T5ForConditionalGeneration,
-    T5Model
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-# outputs a list of dictionaries
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        element = {
-            'input' : f"{desc}{unit}",
-            'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# function to perform training for a given fold
-def train(fold):
-    save_path = f'checkpoint_fold_{fold}b'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-    model_checkpoint = "t5-small"
-    tokenizer = T5TokenizerFast.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['input']
-        target = example['output']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            text_target=target, 
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns=split_datasets["train"].column_names,
-    )
-
-    # https://github.com/huggingface/transformers/pull/28414
-    # model_checkpoint = "google/t5-efficient-tiny"
-    # device_map set to auto to force it to load contiguous weights 
-    # model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, device_map='auto')
-
-    directory = os.path.join(".", f'checkpoint_fold_{fold}a')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    prev_checkpoint = glob.glob(os.path.join(directory, pattern))[0]
-    # Load the checkpoint
-    checkpoint_path = f"{prev_checkpoint}/model.safetensors"
-    checkpoint = load_file(checkpoint_path)
-    # Filter out weights related to the classification head
-    # given name format: t5.encoder.embed_tokens.weight
-    # we want: encoder.embed.tokens.weight
-    t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
-
-    model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
-    model.load_state_dict(state_dict=t5_weights, strict=False)
-
-    for key, param in model.state_dict().items():
-        if key in t5_weights:
-            print(f"{key}: Successfully overridden")
-        else:
-            print(f"{key}: Retained original weights")
-
-    # Freeze the encoder
-    for param in model.encoder.parameters():
-        param.requires_grad = False
-
-    # Freeze the shared embedding layer
-    for param in model.shared.parameters():
-        param.requires_grad = False
-
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-    metric = evaluate.load("sacrebleu")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        # In case the model returns more than the prediction logits
-        if isinstance(preds, tuple):
-            preds = preds[0]
-
-        decoded_preds = tokenizer.batch_decode(preds, 
-                                            skip_special_tokens=False)
-
-        # Replace -100s in the labels as we can't decode them
-        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-        decoded_labels = tokenizer.batch_decode(labels,
-                                                skip_special_tokens=False)
-
-        # Remove <PAD> tokens from decoded predictions and labels
-        decoded_preds = [pred.replace(tokenizer.pad_token, '').strip() for pred in decoded_preds]
-        decoded_labels = [[label.replace(tokenizer.pad_token, '').strip()] for label in decoded_labels]
-
-        # Some simple post-processing
-        # decoded_preds = [pred.strip() for pred in decoded_preds]
-        # decoded_labels = [[label.strip()] for label in decoded_labels]
-        # print(decoded_preds, decoded_labels)
-
-        result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-        return {"bleu": result["score"]}
-
-
-    # Generation Config
-    # from transformers import GenerationConfig
-    gen_config = model.generation_config
-    gen_config.max_length = 128
-
-    # compile
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # Trainer
-
-    args = Seq2SeqTrainingArguments(
-        f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=40,
-        predict_with_generate=True,
-        bf16=True,
-        push_to_hub=False,
-        generation_config=gen_config,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model,
-        args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-

train/hybrid_t5_pattern_desc_unit/train_encoder.py

@@ -1,228 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-from custom_t5.modeling_t5 import T5EncoderForSequenceClassification
-from transformers import (
-    AutoTokenizer,
-    AutoModelForSequenceClassification,
-    DataCollatorWithPadding,
-    Trainer,
-    EarlyStoppingCallback,
-    TrainingArguments,
-    T5Config,
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-# %%
-
-# we need to create the mdm_list
-# import the full mdm-only file
-data_path = '../../data_import/exports/data_mapping_mdm.csv'
-full_df = pd.read_csv(data_path, skipinitialspace=True)
-mdm_list = sorted(list((set(full_df['pattern']))))
-
-# # rather than use pattern, we use the real thing and property
-# thing_property = full_df['thing'] + full_df['property']
-# thing_property = thing_property.to_list()
-# mdm_list = sorted(list(set(thing_property)))
-
-
-# %%
-id2label = {}
-label2id = {}
-for idx, val in enumerate(mdm_list):
-    id2label[idx] = val
-    label2id[val] = idx
-
-# %%
-
-# outputs a list of dictionaries
-# processes dataframe into lists of dictionaries
-# each element maps input to output
-# input: tag_description
-# output: class label
-def process_df_to_dict(df, mdm_list):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        # pattern = f"{row['thing'] + row['property']}"
-        pattern = f"{row['thing_pattern'] + ' ' + row['property_pattern']}"
-        try:
-            index = mdm_list.index(pattern)
-        except ValueError:
-            print("Error: value not found in MDM list")
-            index = -1
-        element = {
-            'text' : f"{desc}{unit}",
-            'label': index,
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold, mdm_list):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df, mdm_list)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df, mdm_list)),
-    })
-    return combined_data
-
-
-# %%
-
-# function to perform training for a given fold
-def train(fold):
-
-    save_path = f'checkpoint_fold_{fold}a'
-    split_datasets = create_split_dataset(fold, mdm_list)
-
-    # prepare tokenizer
-
-    # model_checkpoint = "distilbert/distilbert-base-uncased"
-    # model_checkpoint = 'google-bert/bert-base-cased'
-    model_checkpoint = "t5-small"
-    tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['text']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns="text",
-    )
-
-    # %% temp
-    # tokenized_datasets['train'].rename_columns()
-
-    # %%
-    # create data collator
-
-    data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
-
-    # %%
-    # compute metrics
-    metric = evaluate.load("accuracy")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        preds = np.argmax(preds, axis=1)
-        return metric.compute(predictions=preds, references=labels)
-
-    # %%
-    # create id2label and label2id
-
-
-    # %%
-    # model = AutoModelForSequenceClassification.from_pretrained(
-    #     model_checkpoint,
-    #     num_labels=len(mdm_list),
-    #     id2label=id2label,
-    #     label2id=label2id)
-    model = T5EncoderForSequenceClassification(
-        checkpoint=model_checkpoint,
-        tokenizer=tokenizer,
-        config=T5Config.from_pretrained(model_checkpoint),
-        num_labels=len(mdm_list)
-    )
-    # important! after extending tokens vocab
-    # model.t5.resize_token_embeddings(len(tokenizer))
-
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # %%
-    # Trainer
-
-    training_args = TrainingArguments(
-        output_dir=f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=80,
-        bf16=True,
-        push_to_hub=False,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Trainer(
-        model,
-        training_args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        tokenizer=tokenizer,
-        data_collator=data_collator,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-
-
-# %%

train/mapping_t5_complete_desc_unit/mapping_prediction/predict.py

@@ -13,7 +13,6 @@ def infer_and_select(fold):
     # import test data
     data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
     df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
 
     # get target data
     data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"

train/modified_t5_decoder_12_layers/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

train/modified_t5_decoder_12_layers/mapping_prediction/.gitignore

@@ -1,2 +0,0 @@
-__pycache__
-exports/

train/modified_t5_decoder_12_layers/mapping_prediction/inference.py

@@ -1,168 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-)
-import os
-from tqdm import tqdm
-from datasets import Dataset
-import numpy as np
-
-os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-
-class Inference():
-    tokenizer: T5TokenizerFast
-    model: torch.nn.Module
-    dataloader: DataLoader
-
-    def __init__(self, checkpoint_path):
-        self._create_tokenizer()
-        self._load_model(checkpoint_path)
-
-
-    def _create_tokenizer(self):
-        # %%
-        # load tokenizer
-        self.tokenizer = T5TokenizerFast.from_pretrained("t5-small", return_tensors="pt", clean_up_tokenization_spaces=True)
-        # Define additional special tokens
-        additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "SIG", "UNIT", "DATA_TYPE"]
-        # Add the additional special tokens to the tokenizer
-        self.tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    def _load_model(self, checkpoint_path: str):
-        # load model
-        # Define the directory and the pattern
-        model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)
-        model = torch.compile(model)
-        # set model to eval
-        self.model = model.eval()
-
-
-
-
-    def prepare_dataloader(self, input_df, batch_size, max_length):
-        """
-        *arguments*
-        - input_df: input dataframe containing fields 'tag_description', 'thing', 'property'
-        - batch_size: the batch size of dataloader output
-        - max_length: length of tokenizer output
-        """
-        print("preparing dataloader")
-        # convert each dataframe row into a dictionary
-        # outputs a list of dictionaries
-
-        def _process_df(df):
-            output_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = {
-                    'input' : f"{desc}{unit}",
-                    'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-                }
-                output_list.append(element)
-
-            return output_list
-
-        def _preprocess_function(example):
-            input = example['input']
-            target = example['output']
-            # text_target sets the corresponding label to inputs
-            # there is no need to create a separate 'labels'
-            model_inputs = self.tokenizer(
-                input,
-                text_target=target, 
-                max_length=max_length,
-                return_tensors="pt",
-                padding="max_length",
-                truncation=True,
-            )
-            return model_inputs
-
-        test_dataset = Dataset.from_list(_process_df(input_df))
-
-
-        # map maps function to each "row" in the dataset
-        # aka the data in the immediate nesting
-        datasets = test_dataset.map(
-            _preprocess_function,
-            batched=True,
-            num_proc=1,
-            remove_columns=test_dataset.column_names,
-        )
-        # datasets = _preprocess_function(test_dataset)
-        datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
-
-        # create dataloader
-        self.dataloader = DataLoader(datasets, batch_size=batch_size)
-
-
-    def generate(self):
-        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        MAX_GENERATE_LENGTH = 128
-
-        pred_generations = []
-        pred_labels = []
-
-        print("start generation")
-        for batch in tqdm(self.dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            pred_labels.extend(batch['labels'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-            self.model.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                outputs = self.model.generate(input_ids,
-                                        attention_mask=attention_mask,
-                                        max_length=MAX_GENERATE_LENGTH)
-                
-                # Decode the output and print the results
-                pred_generations.extend(outputs.to("cpu"))
-
-
-
-        # %%
-        # extract sequence and decode
-        def extract_seq(tokens, start_value, end_value):
-            if start_value not in tokens or end_value not in tokens:
-                return None  # Or handle this case according to your requirements
-            start_id = np.where(tokens == start_value)[0][0]
-            end_id = np.where(tokens == end_value)[0][0]
-
-            return tokens[start_id+1:end_id]
-
-
-        def process_tensor_output(tokens):
-            thing_seq = extract_seq(tokens, 32100, 32101) # 32100 = <THING_START>, 32101 = <THING_END>
-            property_seq = extract_seq(tokens, 32102, 32103) # 32102 = <PROPERTY_START>, 32103 = <PROPERTY_END>
-            p_thing = None
-            p_property = None
-            if (thing_seq is not None):
-                p_thing =  self.tokenizer.decode(thing_seq, skip_special_tokens=False)
-            if (property_seq is not None):
-                p_property =  self.tokenizer.decode(property_seq, skip_special_tokens=False)
-            return p_thing, p_property
-
-        # decode prediction labels
-        def decode_preds(tokens_list):
-            thing_prediction_list = []
-            property_prediction_list = []
-            for tokens in tokens_list:
-                p_thing, p_property = process_tensor_output(tokens)
-                thing_prediction_list.append(p_thing)
-                property_prediction_list.append(p_property)
-            return thing_prediction_list, property_prediction_list 
-
-        thing_prediction_list, property_prediction_list = decode_preds(pred_generations)
-        return thing_prediction_list, property_prediction_list
-

train/modified_t5_decoder_12_layers/mapping_prediction/output.txt

@@ -1,6 +0,0 @@
-
-Accuracy for fold 1: 0.9403691433980123
-Accuracy for fold 2: 0.9046728971962616
-Accuracy for fold 3: 0.9678714859437751
-Accuracy for fold 4: 0.9695528068506185
-Accuracy for fold 5: 0.902427851580394

train/modified_t5_decoder_12_layers/mapping_prediction/predict.py

@@ -1,74 +0,0 @@
-
-import pandas as pd
-import os
-import glob
-from inference import Inference
-
-checkpoint_directory =  '../'
-
-BATCH_SIZE = 512
-
-def infer_and_select(fold):
-    print(f"Inference for fold {fold}")
-    # import test data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
-
-    # get target data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-    # processing to help with selection later
-    train_df['thing_property'] = train_df['thing'] + " " + train_df['property']
-
-
-    ##########################################
-    # run inference
-    # checkpoint
-    # Use glob to find matching paths
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}b')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-
-    infer = Inference(checkpoint_path)
-    infer.prepare_dataloader(df, batch_size=BATCH_SIZE, max_length=128)
-    thing_prediction_list, property_prediction_list = infer.generate()
-
-    # add labels too
-    # thing_actual_list, property_actual_list = decode_preds(pred_labels)
-    # Convert the list to a Pandas DataFrame
-    df_out = pd.DataFrame({
-        'p_thing': thing_prediction_list, 
-        'p_property': property_prediction_list
-    })
-    # df_out['p_thing_correct'] = df_out['p_thing'] == df_out['thing']
-    # df_out['p_property_correct'] = df_out['p_property'] == df_out['property']
-    df = pd.concat([df, df_out], axis=1)
-
-    # we can save the t5 generation output here
-    df.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-    # here we want to evaluate mapping accuracy within the valid in mdm data only
-    in_mdm = df['MDM']
-    condition_correct_thing = df['p_thing'] == df['thing']
-    condition_correct_property = df['p_property'] == df['property']
-    prediction_mdm_correct = sum(condition_correct_thing & condition_correct_property & in_mdm)
-    pred_correct_proportion = prediction_mdm_correct/sum(in_mdm)
-
-    # write output to file output.txt
-    with open("output.txt", "a") as f:
-        print(f'Accuracy for fold {fold}: {pred_correct_proportion}', file=f)
-
-###########################################  
-# Execute for all folds
-
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    infer_and_select(fold)

train/modified_t5_decoder_12_layers/train_decoder.py

@@ -1,255 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-
-from safetensors.torch import load_file
-
-from transformers.models.t5.modeling_t5 import T5Block
-from transformers import (
-    T5Config,
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    EarlyStoppingCallback,
-    Seq2SeqTrainingArguments,
-    T5ForConditionalGeneration,
-    T5Model
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-
-
-# %%
-
-# model_checkpoint = "t5-small"
-# model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-# model.config
-
-# %%
-# outputs a list of dictionaries
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        element = {
-            'input' : f"{desc}{unit}",
-            'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# function to perform training for a given fold
-def train(fold):
-    save_path = f'checkpoint_fold_{fold}b'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-    model_checkpoint = "t5-small"
-    tokenizer = T5TokenizerFast.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['input']
-        target = example['output']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            text_target=target, 
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns=split_datasets["train"].column_names,
-    )
-
-    # https://github.com/huggingface/transformers/pull/28414
-    # model_checkpoint = "google/t5-efficient-tiny"
-    # device_map set to auto to force it to load contiguous weights 
-    # model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, device_map='auto')
-
-    # directory = os.path.join(".", f'checkpoint_fold_{fold}a')
-    # # Use glob to find matching paths
-    # # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # # we are guaranteed to save only 1 checkpoint from training
-    # pattern = 'checkpoint-*'
-    # prev_checkpoint = glob.glob(os.path.join(directory, pattern))[0]
-    # # t5_classify = T5Model.from_pretrained(prev_checkpoint)
-    # # Load the checkpoint
-    # checkpoint_path = f"{prev_checkpoint}/model.safetensors"
-    # checkpoint = load_file(checkpoint_path)
-    # # Filter out weights related to the classification head
-    # # given name format: t5.encoder.embed_tokens.weight
-    # # we want: encoder.embed.tokens.weight
-    # t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
-
-
-    pretrained_model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-
-    # Access the decoder stack
-    # config = T5Config("t5-small")
-
-    config = pretrained_model.config
-    config.num_layers = 6
-    config.num_decoder_layers = 12  # set new decoder layer count
-
-    model = T5ForConditionalGeneration(config)
-
-    model.shared = pretrained_model.shared
-    model.encoder = pretrained_model.encoder
-
-    pretrained_decoder_weights = [layer.state_dict() for layer in pretrained_model.decoder.block]
-    for i, layer in enumerate(pretrained_decoder_weights):
-        model.decoder.block[i].load_state_dict(layer)  # Load pretrained weights
-
-
-    # print number of decoder blocks
-    print(f'Number of decoder blocks: {len(model.decoder.block)}')
-    print(f'num_layers: {model.config.num_layers}')
-    print(f'num_decoder_layers: {model.config.num_decoder_layers}')
-
-
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
-
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-    metric = evaluate.load("sacrebleu")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        # In case the model returns more than the prediction logits
-        if isinstance(preds, tuple):
-            preds = preds[0]
-
-        decoded_preds = tokenizer.batch_decode(preds, 
-                                            skip_special_tokens=False)
-
-        # Replace -100s in the labels as we can't decode them
-        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-        decoded_labels = tokenizer.batch_decode(labels,
-                                                skip_special_tokens=False)
-
-        # Remove <PAD> tokens from decoded predictions and labels
-        decoded_preds = [pred.replace(tokenizer.pad_token, '').strip() for pred in decoded_preds]
-        decoded_labels = [[label.replace(tokenizer.pad_token, '').strip()] for label in decoded_labels]
-
-        # Some simple post-processing
-        # decoded_preds = [pred.strip() for pred in decoded_preds]
-        # decoded_labels = [[label.strip()] for label in decoded_labels]
-        # print(decoded_preds, decoded_labels)
-
-        result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-        return {"bleu": result["score"]}
-
-
-    # Generation Config
-    # from transformers import GenerationConfig
-    gen_config = model.generation_config
-    gen_config.max_length = 128
-
-    # compile
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # Trainer
-
-    args = Seq2SeqTrainingArguments(
-        f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=40,
-        predict_with_generate=True,
-        bf16=True,
-        push_to_hub=False,
-        generation_config=gen_config,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model,
-        args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-

train/modified_t5_decoder_1_layers/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

train/modified_t5_decoder_1_layers/mapping_prediction/.gitignore

@@ -1,2 +0,0 @@
-__pycache__
-exports/

train/modified_t5_decoder_1_layers/mapping_prediction/inference.py

@@ -1,168 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-)
-import os
-from tqdm import tqdm
-from datasets import Dataset
-import numpy as np
-
-os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-
-class Inference():
-    tokenizer: T5TokenizerFast
-    model: torch.nn.Module
-    dataloader: DataLoader
-
-    def __init__(self, checkpoint_path):
-        self._create_tokenizer()
-        self._load_model(checkpoint_path)
-
-
-    def _create_tokenizer(self):
-        # %%
-        # load tokenizer
-        self.tokenizer = T5TokenizerFast.from_pretrained("t5-small", return_tensors="pt", clean_up_tokenization_spaces=True)
-        # Define additional special tokens
-        additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "SIG", "UNIT", "DATA_TYPE"]
-        # Add the additional special tokens to the tokenizer
-        self.tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    def _load_model(self, checkpoint_path: str):
-        # load model
-        # Define the directory and the pattern
-        model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)
-        model = torch.compile(model)
-        # set model to eval
-        self.model = model.eval()
-
-
-
-
-    def prepare_dataloader(self, input_df, batch_size, max_length):
-        """
-        *arguments*
-        - input_df: input dataframe containing fields 'tag_description', 'thing', 'property'
-        - batch_size: the batch size of dataloader output
-        - max_length: length of tokenizer output
-        """
-        print("preparing dataloader")
-        # convert each dataframe row into a dictionary
-        # outputs a list of dictionaries
-
-        def _process_df(df):
-            output_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = {
-                    'input' : f"{desc}{unit}",
-                    'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-                }
-                output_list.append(element)
-
-            return output_list
-
-        def _preprocess_function(example):
-            input = example['input']
-            target = example['output']
-            # text_target sets the corresponding label to inputs
-            # there is no need to create a separate 'labels'
-            model_inputs = self.tokenizer(
-                input,
-                text_target=target, 
-                max_length=max_length,
-                return_tensors="pt",
-                padding="max_length",
-                truncation=True,
-            )
-            return model_inputs
-
-        test_dataset = Dataset.from_list(_process_df(input_df))
-
-
-        # map maps function to each "row" in the dataset
-        # aka the data in the immediate nesting
-        datasets = test_dataset.map(
-            _preprocess_function,
-            batched=True,
-            num_proc=1,
-            remove_columns=test_dataset.column_names,
-        )
-        # datasets = _preprocess_function(test_dataset)
-        datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
-
-        # create dataloader
-        self.dataloader = DataLoader(datasets, batch_size=batch_size)
-
-
-    def generate(self):
-        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        MAX_GENERATE_LENGTH = 128
-
-        pred_generations = []
-        pred_labels = []
-
-        print("start generation")
-        for batch in tqdm(self.dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            pred_labels.extend(batch['labels'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-            self.model.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                outputs = self.model.generate(input_ids,
-                                        attention_mask=attention_mask,
-                                        max_length=MAX_GENERATE_LENGTH)
-                
-                # Decode the output and print the results
-                pred_generations.extend(outputs.to("cpu"))
-
-
-
-        # %%
-        # extract sequence and decode
-        def extract_seq(tokens, start_value, end_value):
-            if start_value not in tokens or end_value not in tokens:
-                return None  # Or handle this case according to your requirements
-            start_id = np.where(tokens == start_value)[0][0]
-            end_id = np.where(tokens == end_value)[0][0]
-
-            return tokens[start_id+1:end_id]
-
-
-        def process_tensor_output(tokens):
-            thing_seq = extract_seq(tokens, 32100, 32101) # 32100 = <THING_START>, 32101 = <THING_END>
-            property_seq = extract_seq(tokens, 32102, 32103) # 32102 = <PROPERTY_START>, 32103 = <PROPERTY_END>
-            p_thing = None
-            p_property = None
-            if (thing_seq is not None):
-                p_thing =  self.tokenizer.decode(thing_seq, skip_special_tokens=False)
-            if (property_seq is not None):
-                p_property =  self.tokenizer.decode(property_seq, skip_special_tokens=False)
-            return p_thing, p_property
-
-        # decode prediction labels
-        def decode_preds(tokens_list):
-            thing_prediction_list = []
-            property_prediction_list = []
-            for tokens in tokens_list:
-                p_thing, p_property = process_tensor_output(tokens)
-                thing_prediction_list.append(p_thing)
-                property_prediction_list.append(p_property)
-            return thing_prediction_list, property_prediction_list 
-
-        thing_prediction_list, property_prediction_list = decode_preds(pred_generations)
-        return thing_prediction_list, property_prediction_list
-

train/modified_t5_decoder_1_layers/mapping_prediction/output.txt

@@ -1,6 +0,0 @@
-
-Accuracy for fold 1: 0.8968291528632276
-Accuracy for fold 2: 0.8859813084112149
-Accuracy for fold 3: 0.9382530120481928
-Accuracy for fold 4: 0.9586108468125595
-Accuracy for fold 5: 0.8827301878149336

train/modified_t5_decoder_1_layers/mapping_prediction/predict.py

@@ -1,74 +0,0 @@
-
-import pandas as pd
-import os
-import glob
-from inference import Inference
-
-checkpoint_directory =  '../'
-
-BATCH_SIZE = 512
-
-def infer_and_select(fold):
-    print(f"Inference for fold {fold}")
-    # import test data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
-
-    # get target data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-    # processing to help with selection later
-    train_df['thing_property'] = train_df['thing'] + " " + train_df['property']
-
-
-    ##########################################
-    # run inference
-    # checkpoint
-    # Use glob to find matching paths
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}b')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-
-    infer = Inference(checkpoint_path)
-    infer.prepare_dataloader(df, batch_size=BATCH_SIZE, max_length=128)
-    thing_prediction_list, property_prediction_list = infer.generate()
-
-    # add labels too
-    # thing_actual_list, property_actual_list = decode_preds(pred_labels)
-    # Convert the list to a Pandas DataFrame
-    df_out = pd.DataFrame({
-        'p_thing': thing_prediction_list, 
-        'p_property': property_prediction_list
-    })
-    # df_out['p_thing_correct'] = df_out['p_thing'] == df_out['thing']
-    # df_out['p_property_correct'] = df_out['p_property'] == df_out['property']
-    df = pd.concat([df, df_out], axis=1)
-
-    # we can save the t5 generation output here
-    df.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-    # here we want to evaluate mapping accuracy within the valid in mdm data only
-    in_mdm = df['MDM']
-    condition_correct_thing = df['p_thing'] == df['thing']
-    condition_correct_property = df['p_property'] == df['property']
-    prediction_mdm_correct = sum(condition_correct_thing & condition_correct_property & in_mdm)
-    pred_correct_proportion = prediction_mdm_correct/sum(in_mdm)
-
-    # write output to file output.txt
-    with open("output.txt", "a") as f:
-        print(f'Accuracy for fold {fold}: {pred_correct_proportion}', file=f)
-
-###########################################  
-# Execute for all folds
-
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    infer_and_select(fold)

train/modified_t5_decoder_1_layers/train_decoder.py

@@ -1,255 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-
-from safetensors.torch import load_file
-
-from transformers.models.t5.modeling_t5 import T5Block
-from transformers import (
-    T5Config,
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    EarlyStoppingCallback,
-    Seq2SeqTrainingArguments,
-    T5ForConditionalGeneration,
-    T5Model
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-
-
-# %%
-
-# model_checkpoint = "t5-small"
-# model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-# model.config
-
-# %%
-# outputs a list of dictionaries
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        element = {
-            'input' : f"{desc}{unit}",
-            'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# function to perform training for a given fold
-def train(fold):
-    save_path = f'checkpoint_fold_{fold}b'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-    model_checkpoint = "t5-small"
-    tokenizer = T5TokenizerFast.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['input']
-        target = example['output']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            text_target=target, 
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns=split_datasets["train"].column_names,
-    )
-
-    # https://github.com/huggingface/transformers/pull/28414
-    # model_checkpoint = "google/t5-efficient-tiny"
-    # device_map set to auto to force it to load contiguous weights 
-    # model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, device_map='auto')
-
-    # directory = os.path.join(".", f'checkpoint_fold_{fold}a')
-    # # Use glob to find matching paths
-    # # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # # we are guaranteed to save only 1 checkpoint from training
-    # pattern = 'checkpoint-*'
-    # prev_checkpoint = glob.glob(os.path.join(directory, pattern))[0]
-    # # t5_classify = T5Model.from_pretrained(prev_checkpoint)
-    # # Load the checkpoint
-    # checkpoint_path = f"{prev_checkpoint}/model.safetensors"
-    # checkpoint = load_file(checkpoint_path)
-    # # Filter out weights related to the classification head
-    # # given name format: t5.encoder.embed_tokens.weight
-    # # we want: encoder.embed.tokens.weight
-    # t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
-
-
-    pretrained_model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-
-    # Access the decoder stack
-    # config = T5Config("t5-small")
-
-    config = pretrained_model.config
-    config.num_layers = 6
-    config.num_decoder_layers = 1  # set new decoder layer count
-
-    model = T5ForConditionalGeneration(config)
-
-    model.shared = pretrained_model.shared
-    model.encoder = pretrained_model.encoder
-
-    pretrained_decoder_weights = [layer.state_dict() for layer in pretrained_model.decoder.block]
-    for i, layer in enumerate(pretrained_decoder_weights[:config.num_decoder_layers]):
-        model.decoder.block[i].load_state_dict(layer)  # Load pretrained weights
-
-
-    # print number of decoder blocks
-    print(f'Number of decoder blocks: {len(model.decoder.block)}')
-    print(f'num_layers: {model.config.num_layers}')
-    print(f'num_decoder_layers: {model.config.num_decoder_layers}')
-
-
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
-
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-    metric = evaluate.load("sacrebleu")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        # In case the model returns more than the prediction logits
-        if isinstance(preds, tuple):
-            preds = preds[0]
-
-        decoded_preds = tokenizer.batch_decode(preds, 
-                                            skip_special_tokens=False)
-
-        # Replace -100s in the labels as we can't decode them
-        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-        decoded_labels = tokenizer.batch_decode(labels,
-                                                skip_special_tokens=False)
-
-        # Remove <PAD> tokens from decoded predictions and labels
-        decoded_preds = [pred.replace(tokenizer.pad_token, '').strip() for pred in decoded_preds]
-        decoded_labels = [[label.replace(tokenizer.pad_token, '').strip()] for label in decoded_labels]
-
-        # Some simple post-processing
-        # decoded_preds = [pred.strip() for pred in decoded_preds]
-        # decoded_labels = [[label.strip()] for label in decoded_labels]
-        # print(decoded_preds, decoded_labels)
-
-        result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-        return {"bleu": result["score"]}
-
-
-    # Generation Config
-    # from transformers import GenerationConfig
-    gen_config = model.generation_config
-    gen_config.max_length = 128
-
-    # compile
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # Trainer
-
-    args = Seq2SeqTrainingArguments(
-        f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=40,
-        predict_with_generate=True,
-        bf16=True,
-        push_to_hub=False,
-        generation_config=gen_config,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model,
-        args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-

train/modified_t5_decoder_2_layers/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

train/modified_t5_decoder_2_layers/mapping_prediction/.gitignore

@@ -1,2 +0,0 @@
-__pycache__
-exports/

train/modified_t5_decoder_2_layers/mapping_prediction/inference.py

@@ -1,168 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-)
-import os
-from tqdm import tqdm
-from datasets import Dataset
-import numpy as np
-
-os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-
-class Inference():
-    tokenizer: T5TokenizerFast
-    model: torch.nn.Module
-    dataloader: DataLoader
-
-    def __init__(self, checkpoint_path):
-        self._create_tokenizer()
-        self._load_model(checkpoint_path)
-
-
-    def _create_tokenizer(self):
-        # %%
-        # load tokenizer
-        self.tokenizer = T5TokenizerFast.from_pretrained("t5-small", return_tensors="pt", clean_up_tokenization_spaces=True)
-        # Define additional special tokens
-        additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "SIG", "UNIT", "DATA_TYPE"]
-        # Add the additional special tokens to the tokenizer
-        self.tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    def _load_model(self, checkpoint_path: str):
-        # load model
-        # Define the directory and the pattern
-        model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)
-        model = torch.compile(model)
-        # set model to eval
-        self.model = model.eval()
-
-
-
-
-    def prepare_dataloader(self, input_df, batch_size, max_length):
-        """
-        *arguments*
-        - input_df: input dataframe containing fields 'tag_description', 'thing', 'property'
-        - batch_size: the batch size of dataloader output
-        - max_length: length of tokenizer output
-        """
-        print("preparing dataloader")
-        # convert each dataframe row into a dictionary
-        # outputs a list of dictionaries
-
-        def _process_df(df):
-            output_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = {
-                    'input' : f"{desc}{unit}",
-                    'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-                }
-                output_list.append(element)
-
-            return output_list
-
-        def _preprocess_function(example):
-            input = example['input']
-            target = example['output']
-            # text_target sets the corresponding label to inputs
-            # there is no need to create a separate 'labels'
-            model_inputs = self.tokenizer(
-                input,
-                text_target=target, 
-                max_length=max_length,
-                return_tensors="pt",
-                padding="max_length",
-                truncation=True,
-            )
-            return model_inputs
-
-        test_dataset = Dataset.from_list(_process_df(input_df))
-
-
-        # map maps function to each "row" in the dataset
-        # aka the data in the immediate nesting
-        datasets = test_dataset.map(
-            _preprocess_function,
-            batched=True,
-            num_proc=1,
-            remove_columns=test_dataset.column_names,
-        )
-        # datasets = _preprocess_function(test_dataset)
-        datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
-
-        # create dataloader
-        self.dataloader = DataLoader(datasets, batch_size=batch_size)
-
-
-    def generate(self):
-        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        MAX_GENERATE_LENGTH = 128
-
-        pred_generations = []
-        pred_labels = []
-
-        print("start generation")
-        for batch in tqdm(self.dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            pred_labels.extend(batch['labels'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-            self.model.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                outputs = self.model.generate(input_ids,
-                                        attention_mask=attention_mask,
-                                        max_length=MAX_GENERATE_LENGTH)
-                
-                # Decode the output and print the results
-                pred_generations.extend(outputs.to("cpu"))
-
-
-
-        # %%
-        # extract sequence and decode
-        def extract_seq(tokens, start_value, end_value):
-            if start_value not in tokens or end_value not in tokens:
-                return None  # Or handle this case according to your requirements
-            start_id = np.where(tokens == start_value)[0][0]
-            end_id = np.where(tokens == end_value)[0][0]
-
-            return tokens[start_id+1:end_id]
-
-
-        def process_tensor_output(tokens):
-            thing_seq = extract_seq(tokens, 32100, 32101) # 32100 = <THING_START>, 32101 = <THING_END>
-            property_seq = extract_seq(tokens, 32102, 32103) # 32102 = <PROPERTY_START>, 32103 = <PROPERTY_END>
-            p_thing = None
-            p_property = None
-            if (thing_seq is not None):
-                p_thing =  self.tokenizer.decode(thing_seq, skip_special_tokens=False)
-            if (property_seq is not None):
-                p_property =  self.tokenizer.decode(property_seq, skip_special_tokens=False)
-            return p_thing, p_property
-
-        # decode prediction labels
-        def decode_preds(tokens_list):
-            thing_prediction_list = []
-            property_prediction_list = []
-            for tokens in tokens_list:
-                p_thing, p_property = process_tensor_output(tokens)
-                thing_prediction_list.append(p_thing)
-                property_prediction_list.append(p_property)
-            return thing_prediction_list, property_prediction_list 
-
-        thing_prediction_list, property_prediction_list = decode_preds(pred_generations)
-        return thing_prediction_list, property_prediction_list
-

train/modified_t5_decoder_2_layers/mapping_prediction/output.txt

@@ -1,6 +0,0 @@
-
-Accuracy for fold 1: 0.9318504495977283
-Accuracy for fold 2: 0.8859813084112149
-Accuracy for fold 3: 0.9678714859437751
-Accuracy for fold 4: 0.9738344433872502
-Accuracy for fold 5: 0.9015116811726981

train/modified_t5_decoder_2_layers/mapping_prediction/predict.py

@@ -1,74 +0,0 @@
-
-import pandas as pd
-import os
-import glob
-from inference import Inference
-
-checkpoint_directory =  '../'
-
-BATCH_SIZE = 512
-
-def infer_and_select(fold):
-    print(f"Inference for fold {fold}")
-    # import test data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
-
-    # get target data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-    # processing to help with selection later
-    train_df['thing_property'] = train_df['thing'] + " " + train_df['property']
-
-
-    ##########################################
-    # run inference
-    # checkpoint
-    # Use glob to find matching paths
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}b')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-
-    infer = Inference(checkpoint_path)
-    infer.prepare_dataloader(df, batch_size=BATCH_SIZE, max_length=128)
-    thing_prediction_list, property_prediction_list = infer.generate()
-
-    # add labels too
-    # thing_actual_list, property_actual_list = decode_preds(pred_labels)
-    # Convert the list to a Pandas DataFrame
-    df_out = pd.DataFrame({
-        'p_thing': thing_prediction_list, 
-        'p_property': property_prediction_list
-    })
-    # df_out['p_thing_correct'] = df_out['p_thing'] == df_out['thing']
-    # df_out['p_property_correct'] = df_out['p_property'] == df_out['property']
-    df = pd.concat([df, df_out], axis=1)
-
-    # we can save the t5 generation output here
-    df.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-    # here we want to evaluate mapping accuracy within the valid in mdm data only
-    in_mdm = df['MDM']
-    condition_correct_thing = df['p_thing'] == df['thing']
-    condition_correct_property = df['p_property'] == df['property']
-    prediction_mdm_correct = sum(condition_correct_thing & condition_correct_property & in_mdm)
-    pred_correct_proportion = prediction_mdm_correct/sum(in_mdm)
-
-    # write output to file output.txt
-    with open("output.txt", "a") as f:
-        print(f'Accuracy for fold {fold}: {pred_correct_proportion}', file=f)
-
-###########################################  
-# Execute for all folds
-
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    infer_and_select(fold)

train/modified_t5_decoder_2_layers/train_decoder.py

@@ -1,255 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-
-from safetensors.torch import load_file
-
-from transformers.models.t5.modeling_t5 import T5Block
-from transformers import (
-    T5Config,
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    EarlyStoppingCallback,
-    Seq2SeqTrainingArguments,
-    T5ForConditionalGeneration,
-    T5Model
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-
-
-# %%
-
-# model_checkpoint = "t5-small"
-# model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-# model.config
-
-# %%
-# outputs a list of dictionaries
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        element = {
-            'input' : f"{desc}{unit}",
-            'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# function to perform training for a given fold
-def train(fold):
-    save_path = f'checkpoint_fold_{fold}b'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-    model_checkpoint = "t5-small"
-    tokenizer = T5TokenizerFast.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['input']
-        target = example['output']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            text_target=target, 
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns=split_datasets["train"].column_names,
-    )
-
-    # https://github.com/huggingface/transformers/pull/28414
-    # model_checkpoint = "google/t5-efficient-tiny"
-    # device_map set to auto to force it to load contiguous weights 
-    # model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, device_map='auto')
-
-    # directory = os.path.join(".", f'checkpoint_fold_{fold}a')
-    # # Use glob to find matching paths
-    # # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # # we are guaranteed to save only 1 checkpoint from training
-    # pattern = 'checkpoint-*'
-    # prev_checkpoint = glob.glob(os.path.join(directory, pattern))[0]
-    # # t5_classify = T5Model.from_pretrained(prev_checkpoint)
-    # # Load the checkpoint
-    # checkpoint_path = f"{prev_checkpoint}/model.safetensors"
-    # checkpoint = load_file(checkpoint_path)
-    # # Filter out weights related to the classification head
-    # # given name format: t5.encoder.embed_tokens.weight
-    # # we want: encoder.embed.tokens.weight
-    # t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
-
-
-    pretrained_model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-
-    # Access the decoder stack
-    # config = T5Config("t5-small")
-
-    config = pretrained_model.config
-    config.num_layers = 6
-    config.num_decoder_layers = 2  # set new decoder layer count
-
-    model = T5ForConditionalGeneration(config)
-
-    model.shared = pretrained_model.shared
-    model.encoder = pretrained_model.encoder
-
-    pretrained_decoder_weights = [layer.state_dict() for layer in pretrained_model.decoder.block]
-    for i, layer in enumerate(pretrained_decoder_weights[:config.num_decoder_layers]):
-        model.decoder.block[i].load_state_dict(layer)  # Load pretrained weights
-
-
-    # print number of decoder blocks
-    print(f'Number of decoder blocks: {len(model.decoder.block)}')
-    print(f'num_layers: {model.config.num_layers}')
-    print(f'num_decoder_layers: {model.config.num_decoder_layers}')
-
-
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
-
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-    metric = evaluate.load("sacrebleu")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        # In case the model returns more than the prediction logits
-        if isinstance(preds, tuple):
-            preds = preds[0]
-
-        decoded_preds = tokenizer.batch_decode(preds, 
-                                            skip_special_tokens=False)
-
-        # Replace -100s in the labels as we can't decode them
-        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-        decoded_labels = tokenizer.batch_decode(labels,
-                                                skip_special_tokens=False)
-
-        # Remove <PAD> tokens from decoded predictions and labels
-        decoded_preds = [pred.replace(tokenizer.pad_token, '').strip() for pred in decoded_preds]
-        decoded_labels = [[label.replace(tokenizer.pad_token, '').strip()] for label in decoded_labels]
-
-        # Some simple post-processing
-        # decoded_preds = [pred.strip() for pred in decoded_preds]
-        # decoded_labels = [[label.strip()] for label in decoded_labels]
-        # print(decoded_preds, decoded_labels)
-
-        result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-        return {"bleu": result["score"]}
-
-
-    # Generation Config
-    # from transformers import GenerationConfig
-    gen_config = model.generation_config
-    gen_config.max_length = 128
-
-    # compile
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # Trainer
-
-    args = Seq2SeqTrainingArguments(
-        f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=40,
-        predict_with_generate=True,
-        bf16=True,
-        push_to_hub=False,
-        generation_config=gen_config,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model,
-        args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-

train/modified_t5_decoder_3_layers/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

train/modified_t5_decoder_3_layers/mapping_prediction/.gitignore

@@ -1,2 +0,0 @@
-__pycache__
-exports/

train/modified_t5_decoder_3_layers/mapping_prediction/inference.py

@@ -1,168 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-)
-import os
-from tqdm import tqdm
-from datasets import Dataset
-import numpy as np
-
-os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-
-class Inference():
-    tokenizer: T5TokenizerFast
-    model: torch.nn.Module
-    dataloader: DataLoader
-
-    def __init__(self, checkpoint_path):
-        self._create_tokenizer()
-        self._load_model(checkpoint_path)
-
-
-    def _create_tokenizer(self):
-        # %%
-        # load tokenizer
-        self.tokenizer = T5TokenizerFast.from_pretrained("t5-small", return_tensors="pt", clean_up_tokenization_spaces=True)
-        # Define additional special tokens
-        additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "SIG", "UNIT", "DATA_TYPE"]
-        # Add the additional special tokens to the tokenizer
-        self.tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    def _load_model(self, checkpoint_path: str):
-        # load model
-        # Define the directory and the pattern
-        model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)
-        model = torch.compile(model)
-        # set model to eval
-        self.model = model.eval()
-
-
-
-
-    def prepare_dataloader(self, input_df, batch_size, max_length):
-        """
-        *arguments*
-        - input_df: input dataframe containing fields 'tag_description', 'thing', 'property'
-        - batch_size: the batch size of dataloader output
-        - max_length: length of tokenizer output
-        """
-        print("preparing dataloader")
-        # convert each dataframe row into a dictionary
-        # outputs a list of dictionaries
-
-        def _process_df(df):
-            output_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = {
-                    'input' : f"{desc}{unit}",
-                    'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-                }
-                output_list.append(element)
-
-            return output_list
-
-        def _preprocess_function(example):
-            input = example['input']
-            target = example['output']
-            # text_target sets the corresponding label to inputs
-            # there is no need to create a separate 'labels'
-            model_inputs = self.tokenizer(
-                input,
-                text_target=target, 
-                max_length=max_length,
-                return_tensors="pt",
-                padding="max_length",
-                truncation=True,
-            )
-            return model_inputs
-
-        test_dataset = Dataset.from_list(_process_df(input_df))
-
-
-        # map maps function to each "row" in the dataset
-        # aka the data in the immediate nesting
-        datasets = test_dataset.map(
-            _preprocess_function,
-            batched=True,
-            num_proc=1,
-            remove_columns=test_dataset.column_names,
-        )
-        # datasets = _preprocess_function(test_dataset)
-        datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
-
-        # create dataloader
-        self.dataloader = DataLoader(datasets, batch_size=batch_size)
-
-
-    def generate(self):
-        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        MAX_GENERATE_LENGTH = 128
-
-        pred_generations = []
-        pred_labels = []
-
-        print("start generation")
-        for batch in tqdm(self.dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            pred_labels.extend(batch['labels'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-            self.model.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                outputs = self.model.generate(input_ids,
-                                        attention_mask=attention_mask,
-                                        max_length=MAX_GENERATE_LENGTH)
-                
-                # Decode the output and print the results
-                pred_generations.extend(outputs.to("cpu"))
-
-
-
-        # %%
-        # extract sequence and decode
-        def extract_seq(tokens, start_value, end_value):
-            if start_value not in tokens or end_value not in tokens:
-                return None  # Or handle this case according to your requirements
-            start_id = np.where(tokens == start_value)[0][0]
-            end_id = np.where(tokens == end_value)[0][0]
-
-            return tokens[start_id+1:end_id]
-
-
-        def process_tensor_output(tokens):
-            thing_seq = extract_seq(tokens, 32100, 32101) # 32100 = <THING_START>, 32101 = <THING_END>
-            property_seq = extract_seq(tokens, 32102, 32103) # 32102 = <PROPERTY_START>, 32103 = <PROPERTY_END>
-            p_thing = None
-            p_property = None
-            if (thing_seq is not None):
-                p_thing =  self.tokenizer.decode(thing_seq, skip_special_tokens=False)
-            if (property_seq is not None):
-                p_property =  self.tokenizer.decode(property_seq, skip_special_tokens=False)
-            return p_thing, p_property
-
-        # decode prediction labels
-        def decode_preds(tokens_list):
-            thing_prediction_list = []
-            property_prediction_list = []
-            for tokens in tokens_list:
-                p_thing, p_property = process_tensor_output(tokens)
-                thing_prediction_list.append(p_thing)
-                property_prediction_list.append(p_property)
-            return thing_prediction_list, property_prediction_list 
-
-        thing_prediction_list, property_prediction_list = decode_preds(pred_generations)
-        return thing_prediction_list, property_prediction_list
-

train/modified_t5_decoder_3_layers/mapping_prediction/output.txt

@@ -1,6 +0,0 @@
-
-Accuracy for fold 1: 0.9427354472314246
-Accuracy for fold 2: 0.9098130841121496
-Accuracy for fold 3: 0.964859437751004
-Accuracy for fold 4: 0.9719314938154139
-Accuracy for fold 5: 0.9070087036188731

train/modified_t5_decoder_3_layers/mapping_prediction/predict.py

@@ -1,74 +0,0 @@
-
-import pandas as pd
-import os
-import glob
-from inference import Inference
-
-checkpoint_directory =  '../'
-
-BATCH_SIZE = 512
-
-def infer_and_select(fold):
-    print(f"Inference for fold {fold}")
-    # import test data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
-
-    # get target data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-    # processing to help with selection later
-    train_df['thing_property'] = train_df['thing'] + " " + train_df['property']
-
-
-    ##########################################
-    # run inference
-    # checkpoint
-    # Use glob to find matching paths
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}b')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-
-    infer = Inference(checkpoint_path)
-    infer.prepare_dataloader(df, batch_size=BATCH_SIZE, max_length=128)
-    thing_prediction_list, property_prediction_list = infer.generate()
-
-    # add labels too
-    # thing_actual_list, property_actual_list = decode_preds(pred_labels)
-    # Convert the list to a Pandas DataFrame
-    df_out = pd.DataFrame({
-        'p_thing': thing_prediction_list, 
-        'p_property': property_prediction_list
-    })
-    # df_out['p_thing_correct'] = df_out['p_thing'] == df_out['thing']
-    # df_out['p_property_correct'] = df_out['p_property'] == df_out['property']
-    df = pd.concat([df, df_out], axis=1)
-
-    # we can save the t5 generation output here
-    df.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-    # here we want to evaluate mapping accuracy within the valid in mdm data only
-    in_mdm = df['MDM']
-    condition_correct_thing = df['p_thing'] == df['thing']
-    condition_correct_property = df['p_property'] == df['property']
-    prediction_mdm_correct = sum(condition_correct_thing & condition_correct_property & in_mdm)
-    pred_correct_proportion = prediction_mdm_correct/sum(in_mdm)
-
-    # write output to file output.txt
-    with open("output.txt", "a") as f:
-        print(f'Accuracy for fold {fold}: {pred_correct_proportion}', file=f)
-
-###########################################  
-# Execute for all folds
-
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    infer_and_select(fold)

train/modified_t5_decoder_3_layers/train_decoder.py

@@ -1,255 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-
-from safetensors.torch import load_file
-
-from transformers.models.t5.modeling_t5 import T5Block
-from transformers import (
-    T5Config,
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    EarlyStoppingCallback,
-    Seq2SeqTrainingArguments,
-    T5ForConditionalGeneration,
-    T5Model
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-
-
-# %%
-
-# model_checkpoint = "t5-small"
-# model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-# model.config
-
-# %%
-# outputs a list of dictionaries
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        element = {
-            'input' : f"{desc}{unit}",
-            'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# function to perform training for a given fold
-def train(fold):
-    save_path = f'checkpoint_fold_{fold}b'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-    model_checkpoint = "t5-small"
-    tokenizer = T5TokenizerFast.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['input']
-        target = example['output']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            text_target=target, 
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns=split_datasets["train"].column_names,
-    )
-
-    # https://github.com/huggingface/transformers/pull/28414
-    # model_checkpoint = "google/t5-efficient-tiny"
-    # device_map set to auto to force it to load contiguous weights 
-    # model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, device_map='auto')
-
-    # directory = os.path.join(".", f'checkpoint_fold_{fold}a')
-    # # Use glob to find matching paths
-    # # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # # we are guaranteed to save only 1 checkpoint from training
-    # pattern = 'checkpoint-*'
-    # prev_checkpoint = glob.glob(os.path.join(directory, pattern))[0]
-    # # t5_classify = T5Model.from_pretrained(prev_checkpoint)
-    # # Load the checkpoint
-    # checkpoint_path = f"{prev_checkpoint}/model.safetensors"
-    # checkpoint = load_file(checkpoint_path)
-    # # Filter out weights related to the classification head
-    # # given name format: t5.encoder.embed_tokens.weight
-    # # we want: encoder.embed.tokens.weight
-    # t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
-
-
-    pretrained_model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-
-    # Access the decoder stack
-    # config = T5Config("t5-small")
-
-    config = pretrained_model.config
-    config.num_layers = 6
-    config.num_decoder_layers = 3  # set new decoder layer count
-
-    model = T5ForConditionalGeneration(config)
-
-    model.shared = pretrained_model.shared
-    model.encoder = pretrained_model.encoder
-
-    pretrained_decoder_weights = [layer.state_dict() for layer in pretrained_model.decoder.block]
-    for i, layer in enumerate(pretrained_decoder_weights[:config.num_decoder_layers]):
-        model.decoder.block[i].load_state_dict(layer)  # Load pretrained weights
-
-
-    # print number of decoder blocks
-    print(f'Number of decoder blocks: {len(model.decoder.block)}')
-    print(f'num_layers: {model.config.num_layers}')
-    print(f'num_decoder_layers: {model.config.num_decoder_layers}')
-
-
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
-
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-    metric = evaluate.load("sacrebleu")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        # In case the model returns more than the prediction logits
-        if isinstance(preds, tuple):
-            preds = preds[0]
-
-        decoded_preds = tokenizer.batch_decode(preds, 
-                                            skip_special_tokens=False)
-
-        # Replace -100s in the labels as we can't decode them
-        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-        decoded_labels = tokenizer.batch_decode(labels,
-                                                skip_special_tokens=False)
-
-        # Remove <PAD> tokens from decoded predictions and labels
-        decoded_preds = [pred.replace(tokenizer.pad_token, '').strip() for pred in decoded_preds]
-        decoded_labels = [[label.replace(tokenizer.pad_token, '').strip()] for label in decoded_labels]
-
-        # Some simple post-processing
-        # decoded_preds = [pred.strip() for pred in decoded_preds]
-        # decoded_labels = [[label.strip()] for label in decoded_labels]
-        # print(decoded_preds, decoded_labels)
-
-        result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-        return {"bleu": result["score"]}
-
-
-    # Generation Config
-    # from transformers import GenerationConfig
-    gen_config = model.generation_config
-    gen_config.max_length = 128
-
-    # compile
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # Trainer
-
-    args = Seq2SeqTrainingArguments(
-        f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=40,
-        predict_with_generate=True,
-        bf16=True,
-        push_to_hub=False,
-        generation_config=gen_config,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model,
-        args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-

train/modified_t5_decoder_4_layers/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

train/modified_t5_decoder_4_layers/mapping_prediction/.gitignore

@@ -1,2 +0,0 @@
-__pycache__
-exports/

train/modified_t5_decoder_4_layers/mapping_prediction/inference.py

@@ -1,168 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-)
-import os
-from tqdm import tqdm
-from datasets import Dataset
-import numpy as np
-
-os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-
-class Inference():
-    tokenizer: T5TokenizerFast
-    model: torch.nn.Module
-    dataloader: DataLoader
-
-    def __init__(self, checkpoint_path):
-        self._create_tokenizer()
-        self._load_model(checkpoint_path)
-
-
-    def _create_tokenizer(self):
-        # %%
-        # load tokenizer
-        self.tokenizer = T5TokenizerFast.from_pretrained("t5-small", return_tensors="pt", clean_up_tokenization_spaces=True)
-        # Define additional special tokens
-        additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "SIG", "UNIT", "DATA_TYPE"]
-        # Add the additional special tokens to the tokenizer
-        self.tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    def _load_model(self, checkpoint_path: str):
-        # load model
-        # Define the directory and the pattern
-        model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)
-        model = torch.compile(model)
-        # set model to eval
-        self.model = model.eval()
-
-
-
-
-    def prepare_dataloader(self, input_df, batch_size, max_length):
-        """
-        *arguments*
-        - input_df: input dataframe containing fields 'tag_description', 'thing', 'property'
-        - batch_size: the batch size of dataloader output
-        - max_length: length of tokenizer output
-        """
-        print("preparing dataloader")
-        # convert each dataframe row into a dictionary
-        # outputs a list of dictionaries
-
-        def _process_df(df):
-            output_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = {
-                    'input' : f"{desc}{unit}",
-                    'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-                }
-                output_list.append(element)
-
-            return output_list
-
-        def _preprocess_function(example):
-            input = example['input']
-            target = example['output']
-            # text_target sets the corresponding label to inputs
-            # there is no need to create a separate 'labels'
-            model_inputs = self.tokenizer(
-                input,
-                text_target=target, 
-                max_length=max_length,
-                return_tensors="pt",
-                padding="max_length",
-                truncation=True,
-            )
-            return model_inputs
-
-        test_dataset = Dataset.from_list(_process_df(input_df))
-
-
-        # map maps function to each "row" in the dataset
-        # aka the data in the immediate nesting
-        datasets = test_dataset.map(
-            _preprocess_function,
-            batched=True,
-            num_proc=1,
-            remove_columns=test_dataset.column_names,
-        )
-        # datasets = _preprocess_function(test_dataset)
-        datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
-
-        # create dataloader
-        self.dataloader = DataLoader(datasets, batch_size=batch_size)
-
-
-    def generate(self):
-        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        MAX_GENERATE_LENGTH = 128
-
-        pred_generations = []
-        pred_labels = []
-
-        print("start generation")
-        for batch in tqdm(self.dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            pred_labels.extend(batch['labels'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-            self.model.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                outputs = self.model.generate(input_ids,
-                                        attention_mask=attention_mask,
-                                        max_length=MAX_GENERATE_LENGTH)
-                
-                # Decode the output and print the results
-                pred_generations.extend(outputs.to("cpu"))
-
-
-
-        # %%
-        # extract sequence and decode
-        def extract_seq(tokens, start_value, end_value):
-            if start_value not in tokens or end_value not in tokens:
-                return None  # Or handle this case according to your requirements
-            start_id = np.where(tokens == start_value)[0][0]
-            end_id = np.where(tokens == end_value)[0][0]
-
-            return tokens[start_id+1:end_id]
-
-
-        def process_tensor_output(tokens):
-            thing_seq = extract_seq(tokens, 32100, 32101) # 32100 = <THING_START>, 32101 = <THING_END>
-            property_seq = extract_seq(tokens, 32102, 32103) # 32102 = <PROPERTY_START>, 32103 = <PROPERTY_END>
-            p_thing = None
-            p_property = None
-            if (thing_seq is not None):
-                p_thing =  self.tokenizer.decode(thing_seq, skip_special_tokens=False)
-            if (property_seq is not None):
-                p_property =  self.tokenizer.decode(property_seq, skip_special_tokens=False)
-            return p_thing, p_property
-
-        # decode prediction labels
-        def decode_preds(tokens_list):
-            thing_prediction_list = []
-            property_prediction_list = []
-            for tokens in tokens_list:
-                p_thing, p_property = process_tensor_output(tokens)
-                thing_prediction_list.append(p_thing)
-                property_prediction_list.append(p_property)
-            return thing_prediction_list, property_prediction_list 
-
-        thing_prediction_list, property_prediction_list = decode_preds(pred_generations)
-        return thing_prediction_list, property_prediction_list
-

train/modified_t5_decoder_4_layers/mapping_prediction/output.txt

@@ -1,6 +0,0 @@
-
-Accuracy for fold 1: 0.9503076194983436
-Accuracy for fold 2: 0.9135514018691588
-Accuracy for fold 3: 0.9698795180722891
-Accuracy for fold 4: 0.9790675547098002
-Accuracy for fold 5: 0.907924874026569

train/modified_t5_decoder_4_layers/mapping_prediction/predict.py

@@ -1,74 +0,0 @@
-
-import pandas as pd
-import os
-import glob
-from inference import Inference
-
-checkpoint_directory =  '../'
-
-BATCH_SIZE = 512
-
-def infer_and_select(fold):
-    print(f"Inference for fold {fold}")
-    # import test data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    # df = df[df['MDM']].reset_index(drop=True)
-
-    # get target data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-    # processing to help with selection later
-    train_df['thing_property'] = train_df['thing'] + " " + train_df['property']
-
-
-    ##########################################
-    # run inference
-    # checkpoint
-    # Use glob to find matching paths
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}b')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-
-    infer = Inference(checkpoint_path)
-    infer.prepare_dataloader(df, batch_size=BATCH_SIZE, max_length=128)
-    thing_prediction_list, property_prediction_list = infer.generate()
-
-    # add labels too
-    # thing_actual_list, property_actual_list = decode_preds(pred_labels)
-    # Convert the list to a Pandas DataFrame
-    df_out = pd.DataFrame({
-        'p_thing': thing_prediction_list, 
-        'p_property': property_prediction_list
-    })
-    # df_out['p_thing_correct'] = df_out['p_thing'] == df_out['thing']
-    # df_out['p_property_correct'] = df_out['p_property'] == df_out['property']
-    df = pd.concat([df, df_out], axis=1)
-
-    # we can save the t5 generation output here
-    df.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-    # here we want to evaluate mapping accuracy within the valid in mdm data only
-    in_mdm = df['MDM']
-    condition_correct_thing = df['p_thing'] == df['thing']
-    condition_correct_property = df['p_property'] == df['property']
-    prediction_mdm_correct = sum(condition_correct_thing & condition_correct_property & in_mdm)
-    pred_correct_proportion = prediction_mdm_correct/sum(in_mdm)
-
-    # write output to file output.txt
-    with open("output.txt", "a") as f:
-        print(f'Accuracy for fold {fold}: {pred_correct_proportion}', file=f)
-
-###########################################  
-# Execute for all folds
-
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    infer_and_select(fold)

train/modified_t5_decoder_4_layers/train_decoder.py

@@ -1,255 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-
-from safetensors.torch import load_file
-
-from transformers.models.t5.modeling_t5 import T5Block
-from transformers import (
-    T5Config,
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    EarlyStoppingCallback,
-    Seq2SeqTrainingArguments,
-    T5ForConditionalGeneration,
-    T5Model
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-
-
-# %%
-
-# model_checkpoint = "t5-small"
-# model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-# model.config
-
-# %%
-# outputs a list of dictionaries
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        element = {
-            'input' : f"{desc}{unit}",
-            'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# function to perform training for a given fold
-def train(fold):
-    save_path = f'checkpoint_fold_{fold}b'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-    model_checkpoint = "t5-small"
-    tokenizer = T5TokenizerFast.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['input']
-        target = example['output']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            text_target=target, 
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns=split_datasets["train"].column_names,
-    )
-
-    # https://github.com/huggingface/transformers/pull/28414
-    # model_checkpoint = "google/t5-efficient-tiny"
-    # device_map set to auto to force it to load contiguous weights 
-    # model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, device_map='auto')
-
-    # directory = os.path.join(".", f'checkpoint_fold_{fold}a')
-    # # Use glob to find matching paths
-    # # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # # we are guaranteed to save only 1 checkpoint from training
-    # pattern = 'checkpoint-*'
-    # prev_checkpoint = glob.glob(os.path.join(directory, pattern))[0]
-    # # t5_classify = T5Model.from_pretrained(prev_checkpoint)
-    # # Load the checkpoint
-    # checkpoint_path = f"{prev_checkpoint}/model.safetensors"
-    # checkpoint = load_file(checkpoint_path)
-    # # Filter out weights related to the classification head
-    # # given name format: t5.encoder.embed_tokens.weight
-    # # we want: encoder.embed.tokens.weight
-    # t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
-
-
-    pretrained_model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-
-    # Access the decoder stack
-    # config = T5Config("t5-small")
-
-    config = pretrained_model.config
-    config.num_layers = 6
-    config.num_decoder_layers = 4  # set new decoder layer count
-
-    model = T5ForConditionalGeneration(config)
-
-    model.shared = pretrained_model.shared
-    model.encoder = pretrained_model.encoder
-
-    pretrained_decoder_weights = [layer.state_dict() for layer in pretrained_model.decoder.block]
-    for i, layer in enumerate(pretrained_decoder_weights[:config.num_decoder_layers]):
-        model.decoder.block[i].load_state_dict(layer)  # Load pretrained weights
-
-
-    # print number of decoder blocks
-    print(f'Number of decoder blocks: {len(model.decoder.block)}')
-    print(f'num_layers: {model.config.num_layers}')
-    print(f'num_decoder_layers: {model.config.num_decoder_layers}')
-
-
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
-
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-    metric = evaluate.load("sacrebleu")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        # In case the model returns more than the prediction logits
-        if isinstance(preds, tuple):
-            preds = preds[0]
-
-        decoded_preds = tokenizer.batch_decode(preds, 
-                                            skip_special_tokens=False)
-
-        # Replace -100s in the labels as we can't decode them
-        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-        decoded_labels = tokenizer.batch_decode(labels,
-                                                skip_special_tokens=False)
-
-        # Remove <PAD> tokens from decoded predictions and labels
-        decoded_preds = [pred.replace(tokenizer.pad_token, '').strip() for pred in decoded_preds]
-        decoded_labels = [[label.replace(tokenizer.pad_token, '').strip()] for label in decoded_labels]
-
-        # Some simple post-processing
-        # decoded_preds = [pred.strip() for pred in decoded_preds]
-        # decoded_labels = [[label.strip()] for label in decoded_labels]
-        # print(decoded_preds, decoded_labels)
-
-        result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-        return {"bleu": result["score"]}
-
-
-    # Generation Config
-    # from transformers import GenerationConfig
-    gen_config = model.generation_config
-    gen_config.max_length = 128
-
-    # compile
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # Trainer
-
-    args = Seq2SeqTrainingArguments(
-        f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=40,
-        predict_with_generate=True,
-        bf16=True,
-        push_to_hub=False,
-        generation_config=gen_config,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model,
-        args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-

train/modified_t5_decoder_8_layers/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

train/modified_t5_decoder_8_layers/mapping_prediction/.gitignore

@@ -1,2 +0,0 @@
-__pycache__
-exports/

train/modified_t5_decoder_8_layers/mapping_prediction/inference.py

@@ -1,168 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-)
-import os
-from tqdm import tqdm
-from datasets import Dataset
-import numpy as np
-
-os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-
-class Inference():
-    tokenizer: T5TokenizerFast
-    model: torch.nn.Module
-    dataloader: DataLoader
-
-    def __init__(self, checkpoint_path):
-        self._create_tokenizer()
-        self._load_model(checkpoint_path)
-
-
-    def _create_tokenizer(self):
-        # %%
-        # load tokenizer
-        self.tokenizer = T5TokenizerFast.from_pretrained("t5-small", return_tensors="pt", clean_up_tokenization_spaces=True)
-        # Define additional special tokens
-        additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "SIG", "UNIT", "DATA_TYPE"]
-        # Add the additional special tokens to the tokenizer
-        self.tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    def _load_model(self, checkpoint_path: str):
-        # load model
-        # Define the directory and the pattern
-        model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)
-        model = torch.compile(model)
-        # set model to eval
-        self.model = model.eval()
-
-
-
-
-    def prepare_dataloader(self, input_df, batch_size, max_length):
-        """
-        *arguments*
-        - input_df: input dataframe containing fields 'tag_description', 'thing', 'property'
-        - batch_size: the batch size of dataloader output
-        - max_length: length of tokenizer output
-        """
-        print("preparing dataloader")
-        # convert each dataframe row into a dictionary
-        # outputs a list of dictionaries
-
-        def _process_df(df):
-            output_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = {
-                    'input' : f"{desc}{unit}",
-                    'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-                }
-                output_list.append(element)
-
-            return output_list
-
-        def _preprocess_function(example):
-            input = example['input']
-            target = example['output']
-            # text_target sets the corresponding label to inputs
-            # there is no need to create a separate 'labels'
-            model_inputs = self.tokenizer(
-                input,
-                text_target=target, 
-                max_length=max_length,
-                return_tensors="pt",
-                padding="max_length",
-                truncation=True,
-            )
-            return model_inputs
-
-        test_dataset = Dataset.from_list(_process_df(input_df))
-
-
-        # map maps function to each "row" in the dataset
-        # aka the data in the immediate nesting
-        datasets = test_dataset.map(
-            _preprocess_function,
-            batched=True,
-            num_proc=1,
-            remove_columns=test_dataset.column_names,
-        )
-        # datasets = _preprocess_function(test_dataset)
-        datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
-
-        # create dataloader
-        self.dataloader = DataLoader(datasets, batch_size=batch_size)
-
-
-    def generate(self):
-        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        MAX_GENERATE_LENGTH = 128
-
-        pred_generations = []
-        pred_labels = []
-
-        print("start generation")
-        for batch in tqdm(self.dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            pred_labels.extend(batch['labels'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-            self.model.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                outputs = self.model.generate(input_ids,
-                                        attention_mask=attention_mask,
-                                        max_length=MAX_GENERATE_LENGTH)
-                
-                # Decode the output and print the results
-                pred_generations.extend(outputs.to("cpu"))
-
-
-
-        # %%
-        # extract sequence and decode
-        def extract_seq(tokens, start_value, end_value):
-            if start_value not in tokens or end_value not in tokens:
-                return None  # Or handle this case according to your requirements
-            start_id = np.where(tokens == start_value)[0][0]
-            end_id = np.where(tokens == end_value)[0][0]
-
-            return tokens[start_id+1:end_id]
-
-
-        def process_tensor_output(tokens):
-            thing_seq = extract_seq(tokens, 32100, 32101) # 32100 = <THING_START>, 32101 = <THING_END>
-            property_seq = extract_seq(tokens, 32102, 32103) # 32102 = <PROPERTY_START>, 32103 = <PROPERTY_END>
-            p_thing = None
-            p_property = None
-            if (thing_seq is not None):
-                p_thing =  self.tokenizer.decode(thing_seq, skip_special_tokens=False)
-            if (property_seq is not None):
-                p_property =  self.tokenizer.decode(property_seq, skip_special_tokens=False)
-            return p_thing, p_property
-
-        # decode prediction labels
-        def decode_preds(tokens_list):
-            thing_prediction_list = []
-            property_prediction_list = []
-            for tokens in tokens_list:
-                p_thing, p_property = process_tensor_output(tokens)
-                thing_prediction_list.append(p_thing)
-                property_prediction_list.append(p_property)
-            return thing_prediction_list, property_prediction_list 
-
-        thing_prediction_list, property_prediction_list = decode_preds(pred_generations)
-        return thing_prediction_list, property_prediction_list
-

train/modified_t5_decoder_8_layers/mapping_prediction/output.txt

@@ -1,6 +0,0 @@
-
-Accuracy for fold 1: 0.9441552295314718
-Accuracy for fold 2: 0.9121495327102803
-Accuracy for fold 3: 0.963855421686747
-Accuracy for fold 4: 0.9752616555661275
-Accuracy for fold 5: 0.907924874026569

train/modified_t5_decoder_8_layers/mapping_prediction/predict.py

@@ -1,74 +0,0 @@
-
-import pandas as pd
-import os
-import glob
-from inference import Inference
-
-checkpoint_directory =  '../'
-
-BATCH_SIZE = 512
-
-def infer_and_select(fold):
-    print(f"Inference for fold {fold}")
-    # import test data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
-
-    # get target data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-    # processing to help with selection later
-    train_df['thing_property'] = train_df['thing'] + " " + train_df['property']
-
-
-    ##########################################
-    # run inference
-    # checkpoint
-    # Use glob to find matching paths
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}b')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-
-    infer = Inference(checkpoint_path)
-    infer.prepare_dataloader(df, batch_size=BATCH_SIZE, max_length=128)
-    thing_prediction_list, property_prediction_list = infer.generate()
-
-    # add labels too
-    # thing_actual_list, property_actual_list = decode_preds(pred_labels)
-    # Convert the list to a Pandas DataFrame
-    df_out = pd.DataFrame({
-        'p_thing': thing_prediction_list, 
-        'p_property': property_prediction_list
-    })
-    # df_out['p_thing_correct'] = df_out['p_thing'] == df_out['thing']
-    # df_out['p_property_correct'] = df_out['p_property'] == df_out['property']
-    df = pd.concat([df, df_out], axis=1)
-
-    # we can save the t5 generation output here
-    df.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-    # here we want to evaluate mapping accuracy within the valid in mdm data only
-    in_mdm = df['MDM']
-    condition_correct_thing = df['p_thing'] == df['thing']
-    condition_correct_property = df['p_property'] == df['property']
-    prediction_mdm_correct = sum(condition_correct_thing & condition_correct_property & in_mdm)
-    pred_correct_proportion = prediction_mdm_correct/sum(in_mdm)
-
-    # write output to file output.txt
-    with open("output.txt", "a") as f:
-        print(f'Accuracy for fold {fold}: {pred_correct_proportion}', file=f)
-
-###########################################  
-# Execute for all folds
-
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    infer_and_select(fold)

train/modified_t5_decoder_8_layers/train_decoder.py

@@ -1,255 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "3"
-
-import torch
-
-from safetensors.torch import load_file
-
-from transformers.models.t5.modeling_t5 import T5Block
-from transformers import (
-    T5Config,
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    EarlyStoppingCallback,
-    Seq2SeqTrainingArguments,
-    T5ForConditionalGeneration,
-    T5Model
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-
-
-# %%
-
-# model_checkpoint = "t5-small"
-# model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-# model.config
-
-# %%
-# outputs a list of dictionaries
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        element = {
-            'input' : f"{desc}{unit}",
-            'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# function to perform training for a given fold
-def train(fold):
-    save_path = f'checkpoint_fold_{fold}b'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-    model_checkpoint = "t5-small"
-    tokenizer = T5TokenizerFast.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['input']
-        target = example['output']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            text_target=target, 
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns=split_datasets["train"].column_names,
-    )
-
-    # https://github.com/huggingface/transformers/pull/28414
-    # model_checkpoint = "google/t5-efficient-tiny"
-    # device_map set to auto to force it to load contiguous weights 
-    # model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, device_map='auto')
-
-    # directory = os.path.join(".", f'checkpoint_fold_{fold}a')
-    # # Use glob to find matching paths
-    # # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # # we are guaranteed to save only 1 checkpoint from training
-    # pattern = 'checkpoint-*'
-    # prev_checkpoint = glob.glob(os.path.join(directory, pattern))[0]
-    # # t5_classify = T5Model.from_pretrained(prev_checkpoint)
-    # # Load the checkpoint
-    # checkpoint_path = f"{prev_checkpoint}/model.safetensors"
-    # checkpoint = load_file(checkpoint_path)
-    # # Filter out weights related to the classification head
-    # # given name format: t5.encoder.embed_tokens.weight
-    # # we want: encoder.embed.tokens.weight
-    # t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
-
-
-    pretrained_model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-
-    # Access the decoder stack
-    # config = T5Config("t5-small")
-
-    config = pretrained_model.config
-    config.num_layers = 6
-    config.num_decoder_layers = 8  # set new decoder layer count
-
-    model = T5ForConditionalGeneration(config)
-
-    model.shared = pretrained_model.shared
-    model.encoder = pretrained_model.encoder
-
-    pretrained_decoder_weights = [layer.state_dict() for layer in pretrained_model.decoder.block]
-    for i, layer in enumerate(pretrained_decoder_weights):
-        model.decoder.block[i].load_state_dict(layer)  # Load pretrained weights
-
-
-    # print number of decoder blocks
-    print(f'Number of decoder blocks: {len(model.decoder.block)}')
-    print(f'num_layers: {model.config.num_layers}')
-    print(f'num_decoder_layers: {model.config.num_decoder_layers}')
-
-
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
-
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-    metric = evaluate.load("sacrebleu")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        # In case the model returns more than the prediction logits
-        if isinstance(preds, tuple):
-            preds = preds[0]
-
-        decoded_preds = tokenizer.batch_decode(preds, 
-                                            skip_special_tokens=False)
-
-        # Replace -100s in the labels as we can't decode them
-        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-        decoded_labels = tokenizer.batch_decode(labels,
-                                                skip_special_tokens=False)
-
-        # Remove <PAD> tokens from decoded predictions and labels
-        decoded_preds = [pred.replace(tokenizer.pad_token, '').strip() for pred in decoded_preds]
-        decoded_labels = [[label.replace(tokenizer.pad_token, '').strip()] for label in decoded_labels]
-
-        # Some simple post-processing
-        # decoded_preds = [pred.strip() for pred in decoded_preds]
-        # decoded_labels = [[label.strip()] for label in decoded_labels]
-        # print(decoded_preds, decoded_labels)
-
-        result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-        return {"bleu": result["score"]}
-
-
-    # Generation Config
-    # from transformers import GenerationConfig
-    gen_config = model.generation_config
-    gen_config.max_length = 128
-
-    # compile
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # Trainer
-
-    args = Seq2SeqTrainingArguments(
-        f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=40,
-        predict_with_generate=True,
-        bf16=True,
-        push_to_hub=False,
-        generation_config=gen_config,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model,
-        args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-

train/modified_t5_decoder_9_layers/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

train/modified_t5_decoder_9_layers/mapping_prediction/.gitignore

@@ -1,2 +0,0 @@
-__pycache__
-exports/

train/modified_t5_decoder_9_layers/mapping_prediction/inference.py

@@ -1,168 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-)
-import os
-from tqdm import tqdm
-from datasets import Dataset
-import numpy as np
-
-os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-
-class Inference():
-    tokenizer: T5TokenizerFast
-    model: torch.nn.Module
-    dataloader: DataLoader
-
-    def __init__(self, checkpoint_path):
-        self._create_tokenizer()
-        self._load_model(checkpoint_path)
-
-
-    def _create_tokenizer(self):
-        # %%
-        # load tokenizer
-        self.tokenizer = T5TokenizerFast.from_pretrained("t5-small", return_tensors="pt", clean_up_tokenization_spaces=True)
-        # Define additional special tokens
-        additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "SIG", "UNIT", "DATA_TYPE"]
-        # Add the additional special tokens to the tokenizer
-        self.tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    def _load_model(self, checkpoint_path: str):
-        # load model
-        # Define the directory and the pattern
-        model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)
-        model = torch.compile(model)
-        # set model to eval
-        self.model = model.eval()
-
-
-
-
-    def prepare_dataloader(self, input_df, batch_size, max_length):
-        """
-        *arguments*
-        - input_df: input dataframe containing fields 'tag_description', 'thing', 'property'
-        - batch_size: the batch size of dataloader output
-        - max_length: length of tokenizer output
-        """
-        print("preparing dataloader")
-        # convert each dataframe row into a dictionary
-        # outputs a list of dictionaries
-
-        def _process_df(df):
-            output_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = {
-                    'input' : f"{desc}{unit}",
-                    'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-                }
-                output_list.append(element)
-
-            return output_list
-
-        def _preprocess_function(example):
-            input = example['input']
-            target = example['output']
-            # text_target sets the corresponding label to inputs
-            # there is no need to create a separate 'labels'
-            model_inputs = self.tokenizer(
-                input,
-                text_target=target, 
-                max_length=max_length,
-                return_tensors="pt",
-                padding="max_length",
-                truncation=True,
-            )
-            return model_inputs
-
-        test_dataset = Dataset.from_list(_process_df(input_df))
-
-
-        # map maps function to each "row" in the dataset
-        # aka the data in the immediate nesting
-        datasets = test_dataset.map(
-            _preprocess_function,
-            batched=True,
-            num_proc=1,
-            remove_columns=test_dataset.column_names,
-        )
-        # datasets = _preprocess_function(test_dataset)
-        datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
-
-        # create dataloader
-        self.dataloader = DataLoader(datasets, batch_size=batch_size)
-
-
-    def generate(self):
-        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        MAX_GENERATE_LENGTH = 128
-
-        pred_generations = []
-        pred_labels = []
-
-        print("start generation")
-        for batch in tqdm(self.dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            pred_labels.extend(batch['labels'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-            self.model.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                outputs = self.model.generate(input_ids,
-                                        attention_mask=attention_mask,
-                                        max_length=MAX_GENERATE_LENGTH)
-                
-                # Decode the output and print the results
-                pred_generations.extend(outputs.to("cpu"))
-
-
-
-        # %%
-        # extract sequence and decode
-        def extract_seq(tokens, start_value, end_value):
-            if start_value not in tokens or end_value not in tokens:
-                return None  # Or handle this case according to your requirements
-            start_id = np.where(tokens == start_value)[0][0]
-            end_id = np.where(tokens == end_value)[0][0]
-
-            return tokens[start_id+1:end_id]
-
-
-        def process_tensor_output(tokens):
-            thing_seq = extract_seq(tokens, 32100, 32101) # 32100 = <THING_START>, 32101 = <THING_END>
-            property_seq = extract_seq(tokens, 32102, 32103) # 32102 = <PROPERTY_START>, 32103 = <PROPERTY_END>
-            p_thing = None
-            p_property = None
-            if (thing_seq is not None):
-                p_thing =  self.tokenizer.decode(thing_seq, skip_special_tokens=False)
-            if (property_seq is not None):
-                p_property =  self.tokenizer.decode(property_seq, skip_special_tokens=False)
-            return p_thing, p_property
-
-        # decode prediction labels
-        def decode_preds(tokens_list):
-            thing_prediction_list = []
-            property_prediction_list = []
-            for tokens in tokens_list:
-                p_thing, p_property = process_tensor_output(tokens)
-                thing_prediction_list.append(p_thing)
-                property_prediction_list.append(p_property)
-            return thing_prediction_list, property_prediction_list 
-
-        thing_prediction_list, property_prediction_list = decode_preds(pred_generations)
-        return thing_prediction_list, property_prediction_list
-

train/modified_t5_decoder_9_layers/mapping_prediction/output.txt

@@ -1,6 +0,0 @@
-
-Accuracy for fold 1: 0.9441552295314718
-Accuracy for fold 2: 0.9121495327102803
-Accuracy for fold 3: 0.963855421686747
-Accuracy for fold 4: 0.9752616555661275
-Accuracy for fold 5: 0.907924874026569

train/modified_t5_decoder_9_layers/mapping_prediction/predict.py

@@ -1,74 +0,0 @@
-
-import pandas as pd
-import os
-import glob
-from inference import Inference
-
-checkpoint_directory =  '../'
-
-BATCH_SIZE = 512
-
-def infer_and_select(fold):
-    print(f"Inference for fold {fold}")
-    # import test data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
-
-    # get target data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-    # processing to help with selection later
-    train_df['thing_property'] = train_df['thing'] + " " + train_df['property']
-
-
-    ##########################################
-    # run inference
-    # checkpoint
-    # Use glob to find matching paths
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}b')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-
-    infer = Inference(checkpoint_path)
-    infer.prepare_dataloader(df, batch_size=BATCH_SIZE, max_length=128)
-    thing_prediction_list, property_prediction_list = infer.generate()
-
-    # add labels too
-    # thing_actual_list, property_actual_list = decode_preds(pred_labels)
-    # Convert the list to a Pandas DataFrame
-    df_out = pd.DataFrame({
-        'p_thing': thing_prediction_list, 
-        'p_property': property_prediction_list
-    })
-    # df_out['p_thing_correct'] = df_out['p_thing'] == df_out['thing']
-    # df_out['p_property_correct'] = df_out['p_property'] == df_out['property']
-    df = pd.concat([df, df_out], axis=1)
-
-    # we can save the t5 generation output here
-    df.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-    # here we want to evaluate mapping accuracy within the valid in mdm data only
-    in_mdm = df['MDM']
-    condition_correct_thing = df['p_thing'] == df['thing']
-    condition_correct_property = df['p_property'] == df['property']
-    prediction_mdm_correct = sum(condition_correct_thing & condition_correct_property & in_mdm)
-    pred_correct_proportion = prediction_mdm_correct/sum(in_mdm)
-
-    # write output to file output.txt
-    with open("output.txt", "a") as f:
-        print(f'Accuracy for fold {fold}: {pred_correct_proportion}', file=f)
-
-###########################################  
-# Execute for all folds
-
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    infer_and_select(fold)

train/modified_t5_decoder_9_layers/train_decoder.py

@@ -1,255 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-
-from safetensors.torch import load_file
-
-from transformers.models.t5.modeling_t5 import T5Block
-from transformers import (
-    T5Config,
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    EarlyStoppingCallback,
-    Seq2SeqTrainingArguments,
-    T5ForConditionalGeneration,
-    T5Model
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-
-
-# %%
-
-# model_checkpoint = "t5-small"
-# model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-# model.config
-
-# %%
-# outputs a list of dictionaries
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        element = {
-            'input' : f"{desc}{unit}",
-            'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# function to perform training for a given fold
-def train(fold):
-    save_path = f'checkpoint_fold_{fold}b'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-    model_checkpoint = "t5-small"
-    tokenizer = T5TokenizerFast.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['input']
-        target = example['output']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            text_target=target, 
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns=split_datasets["train"].column_names,
-    )
-
-    # https://github.com/huggingface/transformers/pull/28414
-    # model_checkpoint = "google/t5-efficient-tiny"
-    # device_map set to auto to force it to load contiguous weights 
-    # model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, device_map='auto')
-
-    # directory = os.path.join(".", f'checkpoint_fold_{fold}a')
-    # # Use glob to find matching paths
-    # # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # # we are guaranteed to save only 1 checkpoint from training
-    # pattern = 'checkpoint-*'
-    # prev_checkpoint = glob.glob(os.path.join(directory, pattern))[0]
-    # # t5_classify = T5Model.from_pretrained(prev_checkpoint)
-    # # Load the checkpoint
-    # checkpoint_path = f"{prev_checkpoint}/model.safetensors"
-    # checkpoint = load_file(checkpoint_path)
-    # # Filter out weights related to the classification head
-    # # given name format: t5.encoder.embed_tokens.weight
-    # # we want: encoder.embed.tokens.weight
-    # t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
-
-
-    pretrained_model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-
-    # Access the decoder stack
-    # config = T5Config("t5-small")
-
-    config = pretrained_model.config
-    config.num_layers = 6
-    config.num_decoder_layers = 9  # set new decoder layer count
-
-    model = T5ForConditionalGeneration(config)
-
-    model.shared = pretrained_model.shared
-    model.encoder = pretrained_model.encoder
-
-    pretrained_decoder_weights = [layer.state_dict() for layer in pretrained_model.decoder.block]
-    for i, layer in enumerate(pretrained_decoder_weights):
-        model.decoder.block[i].load_state_dict(layer)  # Load pretrained weights
-
-
-    # print number of decoder blocks
-    print(f'Number of decoder blocks: {len(model.decoder.block)}')
-    print(f'num_layers: {model.config.num_layers}')
-    print(f'num_decoder_layers: {model.config.num_decoder_layers}')
-
-
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
-
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-    metric = evaluate.load("sacrebleu")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        # In case the model returns more than the prediction logits
-        if isinstance(preds, tuple):
-            preds = preds[0]
-
-        decoded_preds = tokenizer.batch_decode(preds, 
-                                            skip_special_tokens=False)
-
-        # Replace -100s in the labels as we can't decode them
-        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-        decoded_labels = tokenizer.batch_decode(labels,
-                                                skip_special_tokens=False)
-
-        # Remove <PAD> tokens from decoded predictions and labels
-        decoded_preds = [pred.replace(tokenizer.pad_token, '').strip() for pred in decoded_preds]
-        decoded_labels = [[label.replace(tokenizer.pad_token, '').strip()] for label in decoded_labels]
-
-        # Some simple post-processing
-        # decoded_preds = [pred.strip() for pred in decoded_preds]
-        # decoded_labels = [[label.strip()] for label in decoded_labels]
-        # print(decoded_preds, decoded_labels)
-
-        result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-        return {"bleu": result["score"]}
-
-
-    # Generation Config
-    # from transformers import GenerationConfig
-    gen_config = model.generation_config
-    gen_config.max_length = 128
-
-    # compile
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # Trainer
-
-    args = Seq2SeqTrainingArguments(
-        f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=40,
-        predict_with_generate=True,
-        bf16=True,
-        push_to_hub=False,
-        generation_config=gen_config,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model,
-        args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-

train/random_t5_encoder/.gitignore

@@ -1,2 +0,0 @@
-checkpoint*
-tensorboard-log

train/random_t5_encoder/mapping_prediction/.gitignore

@@ -1,2 +0,0 @@
-__pycache__
-exports/

train/random_t5_encoder/mapping_prediction/inference.py

@@ -1,168 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-)
-import os
-from tqdm import tqdm
-from datasets import Dataset
-import numpy as np
-
-os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-
-class Inference():
-    tokenizer: T5TokenizerFast
-    model: torch.nn.Module
-    dataloader: DataLoader
-
-    def __init__(self, checkpoint_path):
-        self._create_tokenizer()
-        self._load_model(checkpoint_path)
-
-
-    def _create_tokenizer(self):
-        # %%
-        # load tokenizer
-        self.tokenizer = T5TokenizerFast.from_pretrained("t5-small", return_tensors="pt", clean_up_tokenization_spaces=True)
-        # Define additional special tokens
-        additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "SIG", "UNIT", "DATA_TYPE"]
-        # Add the additional special tokens to the tokenizer
-        self.tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    def _load_model(self, checkpoint_path: str):
-        # load model
-        # Define the directory and the pattern
-        model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)
-        model = torch.compile(model)
-        # set model to eval
-        self.model = model.eval()
-
-
-
-
-    def prepare_dataloader(self, input_df, batch_size, max_length):
-        """
-        *arguments*
-        - input_df: input dataframe containing fields 'tag_description', 'thing', 'property'
-        - batch_size: the batch size of dataloader output
-        - max_length: length of tokenizer output
-        """
-        print("preparing dataloader")
-        # convert each dataframe row into a dictionary
-        # outputs a list of dictionaries
-
-        def _process_df(df):
-            output_list = []
-            for _, row in df.iterrows():
-                desc = f"<DESC>{row['tag_description']}<DESC>"
-                unit = f"<UNIT>{row['unit']}<UNIT>"
-                element = {
-                    'input' : f"{desc}{unit}",
-                    'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-                }
-                output_list.append(element)
-
-            return output_list
-
-        def _preprocess_function(example):
-            input = example['input']
-            target = example['output']
-            # text_target sets the corresponding label to inputs
-            # there is no need to create a separate 'labels'
-            model_inputs = self.tokenizer(
-                input,
-                text_target=target, 
-                max_length=max_length,
-                return_tensors="pt",
-                padding="max_length",
-                truncation=True,
-            )
-            return model_inputs
-
-        test_dataset = Dataset.from_list(_process_df(input_df))
-
-
-        # map maps function to each "row" in the dataset
-        # aka the data in the immediate nesting
-        datasets = test_dataset.map(
-            _preprocess_function,
-            batched=True,
-            num_proc=1,
-            remove_columns=test_dataset.column_names,
-        )
-        # datasets = _preprocess_function(test_dataset)
-        datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
-
-        # create dataloader
-        self.dataloader = DataLoader(datasets, batch_size=batch_size)
-
-
-    def generate(self):
-        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        MAX_GENERATE_LENGTH = 128
-
-        pred_generations = []
-        pred_labels = []
-
-        print("start generation")
-        for batch in tqdm(self.dataloader):
-            # Inference in batches
-            input_ids = batch['input_ids']
-            attention_mask = batch['attention_mask']
-            # save labels too
-            pred_labels.extend(batch['labels'])
-            
-
-            # Move to GPU if available
-            input_ids = input_ids.to(device)
-            attention_mask = attention_mask.to(device)
-            self.model.to(device)
-
-            # Perform inference
-            with torch.no_grad():
-                outputs = self.model.generate(input_ids,
-                                        attention_mask=attention_mask,
-                                        max_length=MAX_GENERATE_LENGTH)
-                
-                # Decode the output and print the results
-                pred_generations.extend(outputs.to("cpu"))
-
-
-
-        # %%
-        # extract sequence and decode
-        def extract_seq(tokens, start_value, end_value):
-            if start_value not in tokens or end_value not in tokens:
-                return None  # Or handle this case according to your requirements
-            start_id = np.where(tokens == start_value)[0][0]
-            end_id = np.where(tokens == end_value)[0][0]
-
-            return tokens[start_id+1:end_id]
-
-
-        def process_tensor_output(tokens):
-            thing_seq = extract_seq(tokens, 32100, 32101) # 32100 = <THING_START>, 32101 = <THING_END>
-            property_seq = extract_seq(tokens, 32102, 32103) # 32102 = <PROPERTY_START>, 32103 = <PROPERTY_END>
-            p_thing = None
-            p_property = None
-            if (thing_seq is not None):
-                p_thing =  self.tokenizer.decode(thing_seq, skip_special_tokens=False)
-            if (property_seq is not None):
-                p_property =  self.tokenizer.decode(property_seq, skip_special_tokens=False)
-            return p_thing, p_property
-
-        # decode prediction labels
-        def decode_preds(tokens_list):
-            thing_prediction_list = []
-            property_prediction_list = []
-            for tokens in tokens_list:
-                p_thing, p_property = process_tensor_output(tokens)
-                thing_prediction_list.append(p_thing)
-                property_prediction_list.append(p_property)
-            return thing_prediction_list, property_prediction_list 
-
-        thing_prediction_list, property_prediction_list = decode_preds(pred_generations)
-        return thing_prediction_list, property_prediction_list
-

train/random_t5_encoder/mapping_prediction/output.txt

@@ -1,6 +0,0 @@
-
-Accuracy for fold 1: 0.0
-Accuracy for fold 2: 0.0
-Accuracy for fold 3: 0.0
-Accuracy for fold 4: 0.0
-Accuracy for fold 5: 0.0

train/random_t5_encoder/mapping_prediction/predict.py

@@ -1,74 +0,0 @@
-
-import pandas as pd
-import os
-import glob
-from inference import Inference
-
-checkpoint_directory =  '../'
-
-BATCH_SIZE = 512
-
-def infer_and_select(fold):
-    print(f"Inference for fold {fold}")
-    # import test data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
-    df = pd.read_csv(data_path, skipinitialspace=True)
-    df = df[df['MDM']].reset_index(drop=True)
-
-    # get target data
-    data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-    # processing to help with selection later
-    train_df['thing_property'] = train_df['thing'] + " " + train_df['property']
-
-
-    ##########################################
-    # run inference
-    # checkpoint
-    # Use glob to find matching paths
-    directory = os.path.join(checkpoint_directory, f'checkpoint_fold_{fold}b')
-    # Use glob to find matching paths
-    # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # we are guaranteed to save only 1 checkpoint from training
-    pattern = 'checkpoint-*'
-    checkpoint_path = glob.glob(os.path.join(directory, pattern))[0]
-
-
-    infer = Inference(checkpoint_path)
-    infer.prepare_dataloader(df, batch_size=BATCH_SIZE, max_length=128)
-    thing_prediction_list, property_prediction_list = infer.generate()
-
-    # add labels too
-    # thing_actual_list, property_actual_list = decode_preds(pred_labels)
-    # Convert the list to a Pandas DataFrame
-    df_out = pd.DataFrame({
-        'p_thing': thing_prediction_list, 
-        'p_property': property_prediction_list
-    })
-    # df_out['p_thing_correct'] = df_out['p_thing'] == df_out['thing']
-    # df_out['p_property_correct'] = df_out['p_property'] == df_out['property']
-    df = pd.concat([df, df_out], axis=1)
-
-    # we can save the t5 generation output here
-    df.to_csv(f"exports/result_group_{fold}.csv", index=False)
-
-    # here we want to evaluate mapping accuracy within the valid in mdm data only
-    in_mdm = df['MDM']
-    condition_correct_thing = df['p_thing'] == df['thing']
-    condition_correct_property = df['p_property'] == df['property']
-    prediction_mdm_correct = sum(condition_correct_thing & condition_correct_property & in_mdm)
-    pred_correct_proportion = prediction_mdm_correct/sum(in_mdm)
-
-    # write output to file output.txt
-    with open("output.txt", "a") as f:
-        print(f'Accuracy for fold {fold}: {pred_correct_proportion}', file=f)
-
-###########################################  
-# Execute for all folds
-
-# reset file before writing to it
-with open("output.txt", "w") as f:
-    print('', file=f)
-
-for fold in [1,2,3,4,5]:
-    infer_and_select(fold)

train/random_t5_encoder/train_decoder.py

@@ -1,246 +0,0 @@
-# %%
-
-# from datasets import load_from_disk
-import os
-import glob
-
-os.environ['NCCL_P2P_DISABLE'] = '1'
-os.environ['NCCL_IB_DISABLE'] = '1'
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
-
-import torch
-
-from safetensors.torch import load_file
-
-from transformers import T5ForConditionalGeneration, T5Config
-from transformers import (
-    T5Config,
-    T5TokenizerFast,
-    AutoModelForSeq2SeqLM,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    EarlyStoppingCallback,
-    Seq2SeqTrainingArguments,
-    T5ForConditionalGeneration,
-    T5Model
-)
-import evaluate
-import numpy as np
-import pandas as pd
-# import matplotlib.pyplot as plt
-from datasets import Dataset, DatasetDict
-
-
-
-torch.set_float32_matmul_precision('high')
-
-# outputs a list of dictionaries
-def process_df_to_dict(df):
-    output_list = []
-    for _, row in df.iterrows():
-        desc = f"<DESC>{row['tag_description']}<DESC>"
-        unit = f"<UNIT>{row['unit']}<UNIT>"
-        element = {
-            'input' : f"{desc}{unit}",
-            'output': f"<THING_START>{row['thing']}<THING_END><PROPERTY_START>{row['property']}<PROPERTY_END>",
-        }
-        output_list.append(element)
-
-    return output_list
-
-
-def create_split_dataset(fold):
-    # train 
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train_all.csv"
-    train_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    # valid
-    data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
-    validation_df = pd.read_csv(data_path, skipinitialspace=True)
-
-    combined_data = DatasetDict({
-        'train': Dataset.from_list(process_df_to_dict(train_df)),
-        'validation' : Dataset.from_list(process_df_to_dict(validation_df)),
-    })
-    return combined_data
-
-
-# function to perform training for a given fold
-def train(fold):
-    save_path = f'checkpoint_fold_{fold}b'
-    split_datasets = create_split_dataset(fold)
-
-    # prepare tokenizer
-    model_checkpoint = "t5-small"
-    tokenizer = T5TokenizerFast.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
-    # Define additional special tokens
-    additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
-    # Add the additional special tokens to the tokenizer
-    tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
-
-    max_length = 120
-
-    # given a dataset entry, run it through the tokenizer
-    def preprocess_function(example):
-        input = example['input']
-        target = example['output']
-        # text_target sets the corresponding label to inputs
-        # there is no need to create a separate 'labels'
-        model_inputs = tokenizer(
-            input,
-            text_target=target, 
-            max_length=max_length,
-            truncation=True,
-            padding="max_length"
-        )
-        return model_inputs
-
-    # map maps function to each "row" in the dataset
-    # aka the data in the immediate nesting
-    tokenized_datasets = split_datasets.map(
-        preprocess_function,
-        batched=True,
-        num_proc=8,
-        remove_columns=split_datasets["train"].column_names,
-    )
-
-    # https://github.com/huggingface/transformers/pull/28414
-    # model_checkpoint = "google/t5-efficient-tiny"
-    # device_map set to auto to force it to load contiguous weights 
-    # model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, device_map='auto')
-
-    # directory = os.path.join(".", f'checkpoint_fold_{fold}a')
-    # # Use glob to find matching paths
-    # # path is usually checkpoint_fold_1/checkpoint-<step number>
-    # # we are guaranteed to save only 1 checkpoint from training
-    # pattern = 'checkpoint-*'
-    # prev_checkpoint = glob.glob(os.path.join(directory, pattern))[0]
-    # # t5_classify = T5Model.from_pretrained(prev_checkpoint)
-    # # Load the checkpoint
-    # checkpoint_path = f"{prev_checkpoint}/model.safetensors"
-    # checkpoint = load_file(checkpoint_path)
-    # # Filter out weights related to the classification head
-    # # given name format: t5.encoder.embed_tokens.weight
-    # # we want: encoder.embed.tokens.weight
-    # t5_weights= {key.replace("t5.", "", 1): value for key, value in checkpoint.items() if "classifier" not in key}
-
-    model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
-    # change the token embedding size to match the shape
-    model.resize_token_embeddings(len(tokenizer))
-
-
-    # Create a T5 model with random weights
-    config = T5Config.from_pretrained("t5-small")  # Use T5 configuration
-    random_model = T5ForConditionalGeneration(config)  # Model initialized with random weights
-    random_model.resize_token_embeddings(len(tokenizer))
-
-    model.encoder = random_model.encoder
-    model.shared = random_model.shared
-
-    # model.load_state_dict(state_dict=t5_weights, strict=False)
-
-    # for key, param in model.state_dict().items():
-    #     if key in t5_weights:
-    #         print(f"{key}: Successfully overridden")
-    #     else:
-    #         print(f"{key}: Retained original weights")
-
-
-    # Freeze the encoder
-    for param in model.encoder.parameters():
-        param.requires_grad = False
-
-    # Freeze the shared embedding layer
-    for param in model.shared.parameters():
-        param.requires_grad = False
-
-
-    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-    metric = evaluate.load("sacrebleu")
-
-
-    def compute_metrics(eval_preds):
-        preds, labels = eval_preds
-        # In case the model returns more than the prediction logits
-        if isinstance(preds, tuple):
-            preds = preds[0]
-
-        decoded_preds = tokenizer.batch_decode(preds, 
-                                            skip_special_tokens=False)
-
-        # Replace -100s in the labels as we can't decode them
-        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-        decoded_labels = tokenizer.batch_decode(labels,
-                                                skip_special_tokens=False)
-
-        # Remove <PAD> tokens from decoded predictions and labels
-        decoded_preds = [pred.replace(tokenizer.pad_token, '').strip() for pred in decoded_preds]
-        decoded_labels = [[label.replace(tokenizer.pad_token, '').strip()] for label in decoded_labels]
-
-        # Some simple post-processing
-        # decoded_preds = [pred.strip() for pred in decoded_preds]
-        # decoded_labels = [[label.strip()] for label in decoded_labels]
-        # print(decoded_preds, decoded_labels)
-
-        result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-        return {"bleu": result["score"]}
-
-
-    # Generation Config
-    # from transformers import GenerationConfig
-    gen_config = model.generation_config
-    gen_config.max_length = 128
-
-    # compile
-    # model = torch.compile(model, backend="inductor", dynamic=True)
-
-
-    # Trainer
-
-    args = Seq2SeqTrainingArguments(
-        f"{save_path}",
-        # eval_strategy="epoch",
-        eval_strategy="no",
-        logging_dir="tensorboard-log",
-        logging_strategy="epoch",
-        # save_strategy="epoch",
-        load_best_model_at_end=False,
-        learning_rate=1e-3,
-        per_device_train_batch_size=64,
-        per_device_eval_batch_size=64,
-        auto_find_batch_size=False,
-        ddp_find_unused_parameters=False,
-        weight_decay=0.01,
-        save_total_limit=1,
-        num_train_epochs=40,
-        predict_with_generate=True,
-        bf16=True,
-        push_to_hub=False,
-        generation_config=gen_config,
-        remove_unused_columns=False,
-    )
-
-
-    trainer = Seq2SeqTrainer(
-        model,
-        args,
-        train_dataset=tokenized_datasets["train"],
-        eval_dataset=tokenized_datasets["validation"],
-        data_collator=data_collator,
-        tokenizer=tokenizer,
-        compute_metrics=compute_metrics,
-        # callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
-    )
-
-    # uncomment to load training from checkpoint
-    # checkpoint_path = 'default_40_1/checkpoint-5600'
-    # trainer.train(resume_from_checkpoint=checkpoint_path)
-
-    trainer.train()
-
-# execute training
-for fold in [1,2,3,4,5]:
-    print(fold)
-    train(fold)
-

train/train.bash

@@ -1,27 +1,16 @@
 #!/bin/bash
 
-cd hybrid_t5_complete_desc_unit
+cd classification_bert_complete_desc
-micromamba run -n hug accelerate launch train_encoder.py
+micromamba run -n hug accelerate launch train.py
-micromamba run -n hug accelerate launch train_decoder.py
 cd ..
 
-cd hybrid_t5_pattern_desc_unit
+cd classification_bert_complete_desc_unit
-micromamba run -n hug accelerate launch train_encoder.py
+micromamba run -n hug accelerate launch train.py
-micromamba run -n hug accelerate launch train_decoder.py
 cd ..
 
-
+cd classification_bert_complete_desc_unit_name
-# cd classification_bert_complete_desc
+micromamba run -n hug accelerate launch train.py
-# micromamba run -n hug accelerate launch train.py
+cd ..
-# cd ..
-
-# cd classification_bert_complete_desc_unit
-# micromamba run -n hug accelerate launch train.py
-# cd ..
-
-# cd classification_bert_complete_desc_unit_name
-# micromamba run -n hug accelerate launch train.py
-# cd ..
 
 # cd mapping_t5_complete_desc
 # micromamba run -n hug accelerate launch train.py