Feat: implement find-back for analysis in find_closest.py
Feat: implement bert classification
This commit is contained in:
parent
22429ea536
commit
59bbf1f403
|
@ -107,7 +107,7 @@ def find_closest(cos_sim_matrix, condition_source, condition_target):
|
||||||
subset_matrix = cos_sim_matrix[np.ix_(condition_source, condition_target)]
|
subset_matrix = cos_sim_matrix[np.ix_(condition_source, condition_target)]
|
||||||
# we select top k here
|
# we select top k here
|
||||||
# Get the indices of the top 5 maximum values along axis 1
|
# Get the indices of the top 5 maximum values along axis 1
|
||||||
top_k = 5
|
top_k = 3
|
||||||
top_k_indices = np.argsort(subset_matrix, axis=1)[:, -top_k:] # Get indices of top k values
|
top_k_indices = np.argsort(subset_matrix, axis=1)[:, -top_k:] # Get indices of top k values
|
||||||
# note that top_k_indices is a nested list because of the 2d nature of the matrix
|
# note that top_k_indices is a nested list because of the 2d nature of the matrix
|
||||||
# the result is flipped
|
# the result is flipped
|
||||||
|
@ -135,15 +135,20 @@ def find_back_element_with_print(select_idx):
|
||||||
condition_target=condition_target)
|
condition_target=condition_target)
|
||||||
|
|
||||||
training_data_pattern_list = train_df.iloc[top_k_indices[0]]['pattern'].to_list()
|
training_data_pattern_list = train_df.iloc[top_k_indices[0]]['pattern'].to_list()
|
||||||
|
training_desc_list = train_df.iloc[top_k_indices[0]]['tag_description'].to_list()
|
||||||
|
|
||||||
test_data_pattern_list = test_df[test_df.index == select_idx]['pattern'].to_list()
|
test_data_pattern_list = test_df[test_df.index == select_idx]['pattern'].to_list()
|
||||||
predicted_test_data = test_df[test_df.index == select_idx]['p_thing'] + test_df[test_df.index == select_idx]['p_property']
|
test_desc_list = test_df[test_df.index == select_idx]['tag_description'].to_list()
|
||||||
|
predicted_test_data = test_df[test_df.index == select_idx]['p_thing'] + ' ' + test_df[test_df.index == select_idx]['p_property']
|
||||||
|
predicted_test_data = predicted_test_data.to_list()[0]
|
||||||
|
|
||||||
print("*" * 80)
|
print("*" * 80)
|
||||||
print("idx:", select_idx)
|
print("idx:", select_idx)
|
||||||
print(training_data_pattern_list)
|
print("train desc", training_desc_list)
|
||||||
print(test_data_pattern_list)
|
print("train thing+property", training_data_pattern_list)
|
||||||
print(predicted_test_data)
|
print("test desc", test_desc_list)
|
||||||
|
print("test thing+property", test_data_pattern_list)
|
||||||
|
print("predicted thing+property", predicted_test_data)
|
||||||
|
|
||||||
test_pattern = test_data_pattern_list[0]
|
test_pattern = test_data_pattern_list[0]
|
||||||
|
|
||||||
|
@ -154,7 +159,7 @@ def find_back_element_with_print(select_idx):
|
||||||
else:
|
else:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
find_back_element_with_print(2884)
|
find_back_element_with_print(0)
|
||||||
|
|
||||||
# %%
|
# %%
|
||||||
def find_back_element(select_idx):
|
def find_back_element(select_idx):
|
||||||
|
@ -194,15 +199,13 @@ for select_idx in error_thing_df.index:
|
||||||
print("status:", result)
|
print("status:", result)
|
||||||
pattern_in_train.append(result)
|
pattern_in_train.append(result)
|
||||||
|
|
||||||
# %%
|
|
||||||
sum(pattern_in_train)/len(pattern_in_train)
|
|
||||||
|
|
||||||
###
|
###
|
||||||
# for error property
|
# for error property
|
||||||
# %%
|
# %%
|
||||||
pattern_in_train = []
|
pattern_in_train = []
|
||||||
for select_idx in error_property_df.index:
|
for select_idx in error_property_df.index:
|
||||||
result = find_back_element(select_idx)
|
result = find_back_element_with_print(select_idx)
|
||||||
|
print("status:", result)
|
||||||
pattern_in_train.append(result)
|
pattern_in_train.append(result)
|
||||||
|
|
||||||
# %%
|
# %%
|
||||||
|
|
|
@ -0,0 +1,2 @@
|
||||||
|
checkpoint*
|
||||||
|
tensorboard-log
|
|
@ -0,0 +1,228 @@
|
||||||
|
# %%
|
||||||
|
|
||||||
|
# from datasets import load_from_disk
|
||||||
|
import os
|
||||||
|
import glob
|
||||||
|
|
||||||
|
os.environ['NCCL_P2P_DISABLE'] = '1'
|
||||||
|
os.environ['NCCL_IB_DISABLE'] = '1'
|
||||||
|
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
|
||||||
|
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
|
||||||
|
|
||||||
|
import torch
|
||||||
|
from torch.utils.data import DataLoader
|
||||||
|
|
||||||
|
from transformers import (
|
||||||
|
AutoTokenizer,
|
||||||
|
AutoModelForSequenceClassification,
|
||||||
|
DataCollatorWithPadding,
|
||||||
|
Trainer,
|
||||||
|
EarlyStoppingCallback,
|
||||||
|
TrainingArguments
|
||||||
|
)
|
||||||
|
import evaluate
|
||||||
|
import numpy as np
|
||||||
|
import pandas as pd
|
||||||
|
# import matplotlib.pyplot as plt
|
||||||
|
from datasets import Dataset, DatasetDict
|
||||||
|
|
||||||
|
from tqdm import tqdm
|
||||||
|
|
||||||
|
torch.set_float32_matmul_precision('high')
|
||||||
|
|
||||||
|
# %%
|
||||||
|
|
||||||
|
# we need to create the mdm_list
|
||||||
|
# import the full mdm-only file
|
||||||
|
data_path = '../../../data_import/exports/data_mapping_mdm.csv'
|
||||||
|
full_df = pd.read_csv(data_path, skipinitialspace=True)
|
||||||
|
mdm_list = sorted(list((set(full_df['pattern']))))
|
||||||
|
|
||||||
|
# %%
|
||||||
|
id2label = {}
|
||||||
|
label2id = {}
|
||||||
|
for idx, val in enumerate(mdm_list):
|
||||||
|
id2label[idx] = val
|
||||||
|
label2id[val] = idx
|
||||||
|
|
||||||
|
# %%
|
||||||
|
|
||||||
|
# outputs a list of dictionaries
|
||||||
|
# processes dataframe into lists of dictionaries
|
||||||
|
# each element maps input to output
|
||||||
|
# input: tag_description
|
||||||
|
# output: class label
|
||||||
|
def process_df_to_dict(df, mdm_list):
|
||||||
|
output_list = []
|
||||||
|
for _, row in df.iterrows():
|
||||||
|
desc = f"<DESC>{row['tag_description']}<DESC>"
|
||||||
|
pattern = row['pattern']
|
||||||
|
try:
|
||||||
|
index = mdm_list.index(pattern)
|
||||||
|
except ValueError:
|
||||||
|
index = -1
|
||||||
|
element = {
|
||||||
|
'text' : f"{desc}",
|
||||||
|
'label': index,
|
||||||
|
}
|
||||||
|
output_list.append(element)
|
||||||
|
|
||||||
|
return output_list
|
||||||
|
|
||||||
|
|
||||||
|
def create_dataset(fold, mdm_list):
|
||||||
|
data_path = f"../../../data_preprocess/exports/dataset/group_{fold}/test_all.csv"
|
||||||
|
test_df = pd.read_csv(data_path, skipinitialspace=True)
|
||||||
|
# we only use the mdm subset
|
||||||
|
test_df = test_df[test_df['MDM']].reset_index(drop=True)
|
||||||
|
|
||||||
|
test_dataset = Dataset.from_list(process_df_to_dict(test_df, mdm_list))
|
||||||
|
|
||||||
|
return test_dataset
|
||||||
|
|
||||||
|
|
||||||
|
# %%
|
||||||
|
|
||||||
|
# function to perform training for a given fold
|
||||||
|
# def train(fold):
|
||||||
|
fold = 1
|
||||||
|
|
||||||
|
test_dataset = create_dataset(fold, mdm_list)
|
||||||
|
|
||||||
|
# prepare tokenizer
|
||||||
|
|
||||||
|
checkpoint_directory = f'../checkpoint_fold_{fold}'
|
||||||
|
# Use glob to find matching paths
|
||||||
|
# path is usually checkpoint_fold_1/checkpoint-<step number>
|
||||||
|
# we are guaranteed to save only 1 checkpoint from training
|
||||||
|
pattern = 'checkpoint-*'
|
||||||
|
model_checkpoint = glob.glob(os.path.join(checkpoint_directory, pattern))[0]
|
||||||
|
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
|
||||||
|
# Define additional special tokens
|
||||||
|
# additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
|
||||||
|
# Add the additional special tokens to the tokenizer
|
||||||
|
# tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
|
||||||
|
|
||||||
|
# %%
|
||||||
|
# compute max token length
|
||||||
|
max_length = 0
|
||||||
|
for sample in test_dataset['text']:
|
||||||
|
# Tokenize the sample and get the length
|
||||||
|
input_ids = tokenizer(sample, truncation=False, add_special_tokens=True)["input_ids"]
|
||||||
|
length = len(input_ids)
|
||||||
|
|
||||||
|
# Update max_length if this sample is longer
|
||||||
|
if length > max_length:
|
||||||
|
max_length = length
|
||||||
|
|
||||||
|
print(max_length)
|
||||||
|
|
||||||
|
# %%
|
||||||
|
|
||||||
|
max_length = 64
|
||||||
|
|
||||||
|
# given a dataset entry, run it through the tokenizer
|
||||||
|
def preprocess_function(example):
|
||||||
|
input = example['text']
|
||||||
|
# text_target sets the corresponding label to inputs
|
||||||
|
# there is no need to create a separate 'labels'
|
||||||
|
model_inputs = tokenizer(
|
||||||
|
input,
|
||||||
|
max_length=max_length,
|
||||||
|
# truncation=True,
|
||||||
|
padding='max_length'
|
||||||
|
)
|
||||||
|
return model_inputs
|
||||||
|
|
||||||
|
# map maps function to each "row" in the dataset
|
||||||
|
# aka the data in the immediate nesting
|
||||||
|
datasets = test_dataset.map(
|
||||||
|
preprocess_function,
|
||||||
|
batched=True,
|
||||||
|
num_proc=8,
|
||||||
|
remove_columns="text",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])
|
||||||
|
|
||||||
|
# %% temp
|
||||||
|
# tokenized_datasets['train'].rename_columns()
|
||||||
|
|
||||||
|
# %%
|
||||||
|
# create data collator
|
||||||
|
|
||||||
|
data_collator = DataCollatorWithPadding(tokenizer=tokenizer, padding="max_length")
|
||||||
|
|
||||||
|
# %%
|
||||||
|
# compute metrics
|
||||||
|
# metric = evaluate.load("accuracy")
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# def compute_metrics(eval_preds):
|
||||||
|
# preds, labels = eval_preds
|
||||||
|
# preds = np.argmax(preds, axis=1)
|
||||||
|
# return metric.compute(predictions=preds, references=labels)
|
||||||
|
|
||||||
|
model = AutoModelForSequenceClassification.from_pretrained(
|
||||||
|
model_checkpoint,
|
||||||
|
num_labels=len(mdm_list),
|
||||||
|
id2label=id2label,
|
||||||
|
label2id=label2id)
|
||||||
|
# important! after extending tokens vocab
|
||||||
|
model.resize_token_embeddings(len(tokenizer))
|
||||||
|
|
||||||
|
model = model.eval()
|
||||||
|
|
||||||
|
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||||
|
model.to(device)
|
||||||
|
|
||||||
|
pred_labels = []
|
||||||
|
actual_labels = []
|
||||||
|
|
||||||
|
|
||||||
|
BATCH_SIZE = 64
|
||||||
|
dataloader = DataLoader(datasets, batch_size=BATCH_SIZE, shuffle=False)
|
||||||
|
for batch in tqdm(dataloader):
|
||||||
|
# Inference in batches
|
||||||
|
input_ids = batch['input_ids']
|
||||||
|
attention_mask = batch['attention_mask']
|
||||||
|
# save labels too
|
||||||
|
actual_labels.extend(batch['label'])
|
||||||
|
|
||||||
|
|
||||||
|
# Move to GPU if available
|
||||||
|
input_ids = input_ids.to(device)
|
||||||
|
attention_mask = attention_mask.to(device)
|
||||||
|
|
||||||
|
# Perform inference
|
||||||
|
with torch.no_grad():
|
||||||
|
logits = model(
|
||||||
|
input_ids,
|
||||||
|
attention_mask).logits
|
||||||
|
predicted_class_ids = logits.argmax(dim=1).to("cpu")
|
||||||
|
pred_labels.extend(predicted_class_ids)
|
||||||
|
|
||||||
|
pred_labels = [tensor.item() for tensor in pred_labels]
|
||||||
|
|
||||||
|
|
||||||
|
# %%
|
||||||
|
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix
|
||||||
|
y_true = actual_labels
|
||||||
|
y_pred = pred_labels
|
||||||
|
|
||||||
|
# Compute metrics
|
||||||
|
accuracy = accuracy_score(y_true, y_pred)
|
||||||
|
f1 = f1_score(y_true, y_pred, average='macro')
|
||||||
|
precision = precision_score(y_true, y_pred, average='macro')
|
||||||
|
recall = recall_score(y_true, y_pred, average='macro')
|
||||||
|
|
||||||
|
# Print the results
|
||||||
|
print(f'Accuracy: {accuracy:.2f}')
|
||||||
|
print(f'F1 Score: {f1:.2f}')
|
||||||
|
print(f'Precision: {precision:.2f}')
|
||||||
|
print(f'Recall: {recall:.2f}')
|
||||||
|
|
||||||
|
|
||||||
|
# %%
|
|
@ -0,0 +1,211 @@
|
||||||
|
# %%
|
||||||
|
|
||||||
|
# from datasets import load_from_disk
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.environ['NCCL_P2P_DISABLE'] = '1'
|
||||||
|
os.environ['NCCL_IB_DISABLE'] = '1'
|
||||||
|
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
|
||||||
|
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
|
||||||
|
|
||||||
|
import torch
|
||||||
|
from transformers import (
|
||||||
|
AutoTokenizer,
|
||||||
|
AutoModelForSequenceClassification,
|
||||||
|
DataCollatorWithPadding,
|
||||||
|
Trainer,
|
||||||
|
EarlyStoppingCallback,
|
||||||
|
TrainingArguments
|
||||||
|
)
|
||||||
|
import evaluate
|
||||||
|
import numpy as np
|
||||||
|
import pandas as pd
|
||||||
|
# import matplotlib.pyplot as plt
|
||||||
|
from datasets import Dataset, DatasetDict
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
torch.set_float32_matmul_precision('high')
|
||||||
|
|
||||||
|
# %%
|
||||||
|
|
||||||
|
# we need to create the mdm_list
|
||||||
|
# import the full mdm-only file
|
||||||
|
data_path = '../../data_import/exports/data_mapping_mdm.csv'
|
||||||
|
full_df = pd.read_csv(data_path, skipinitialspace=True)
|
||||||
|
mdm_list = sorted(list((set(full_df['pattern']))))
|
||||||
|
|
||||||
|
# %%
|
||||||
|
id2label = {}
|
||||||
|
label2id = {}
|
||||||
|
for idx, val in enumerate(mdm_list):
|
||||||
|
id2label[idx] = val
|
||||||
|
label2id[val] = idx
|
||||||
|
|
||||||
|
# %%
|
||||||
|
|
||||||
|
# outputs a list of dictionaries
|
||||||
|
# processes dataframe into lists of dictionaries
|
||||||
|
# each element maps input to output
|
||||||
|
# input: tag_description
|
||||||
|
# output: class label
|
||||||
|
def process_df_to_dict(df, mdm_list):
|
||||||
|
output_list = []
|
||||||
|
for _, row in df.iterrows():
|
||||||
|
desc = f"<DESC>{row['tag_description']}<DESC>"
|
||||||
|
pattern = row['pattern']
|
||||||
|
try:
|
||||||
|
index = mdm_list.index(pattern)
|
||||||
|
except ValueError:
|
||||||
|
index = -1
|
||||||
|
element = {
|
||||||
|
'text' : f"{desc}",
|
||||||
|
'label': index,
|
||||||
|
}
|
||||||
|
output_list.append(element)
|
||||||
|
|
||||||
|
return output_list
|
||||||
|
|
||||||
|
|
||||||
|
def create_split_dataset(fold, mdm_list):
|
||||||
|
# train
|
||||||
|
data_path = f"../../data_preprocess/exports/dataset/group_{fold}/train.csv"
|
||||||
|
train_df = pd.read_csv(data_path, skipinitialspace=True)
|
||||||
|
|
||||||
|
# valid
|
||||||
|
data_path = f"../../data_preprocess/exports/dataset/group_{fold}/valid.csv"
|
||||||
|
validation_df = pd.read_csv(data_path, skipinitialspace=True)
|
||||||
|
|
||||||
|
combined_data = DatasetDict({
|
||||||
|
'train': Dataset.from_list(process_df_to_dict(train_df, mdm_list)),
|
||||||
|
'validation' : Dataset.from_list(process_df_to_dict(validation_df, mdm_list)),
|
||||||
|
})
|
||||||
|
return combined_data
|
||||||
|
|
||||||
|
|
||||||
|
# %%
|
||||||
|
|
||||||
|
# function to perform training for a given fold
|
||||||
|
# def train(fold):
|
||||||
|
fold = 1
|
||||||
|
|
||||||
|
save_path = f'checkpoint_fold_{fold}'
|
||||||
|
split_datasets = create_split_dataset(fold, mdm_list)
|
||||||
|
|
||||||
|
# prepare tokenizer
|
||||||
|
|
||||||
|
model_checkpoint = "distilbert/distilbert-base-uncased"
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt", clean_up_tokenization_spaces=True)
|
||||||
|
# Define additional special tokens
|
||||||
|
# additional_special_tokens = ["<THING_START>", "<THING_END>", "<PROPERTY_START>", "<PROPERTY_END>", "<NAME>", "<DESC>", "<SIG>", "<UNIT>", "<DATA_TYPE>"]
|
||||||
|
# Add the additional special tokens to the tokenizer
|
||||||
|
# tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
|
||||||
|
|
||||||
|
max_length = 120
|
||||||
|
|
||||||
|
# given a dataset entry, run it through the tokenizer
|
||||||
|
def preprocess_function(example):
|
||||||
|
input = example['text']
|
||||||
|
# text_target sets the corresponding label to inputs
|
||||||
|
# there is no need to create a separate 'labels'
|
||||||
|
model_inputs = tokenizer(
|
||||||
|
input,
|
||||||
|
max_length=max_length,
|
||||||
|
truncation=True,
|
||||||
|
padding=True
|
||||||
|
)
|
||||||
|
return model_inputs
|
||||||
|
|
||||||
|
# map maps function to each "row" in the dataset
|
||||||
|
# aka the data in the immediate nesting
|
||||||
|
tokenized_datasets = split_datasets.map(
|
||||||
|
preprocess_function,
|
||||||
|
batched=True,
|
||||||
|
num_proc=8,
|
||||||
|
remove_columns="text",
|
||||||
|
)
|
||||||
|
|
||||||
|
# %% temp
|
||||||
|
# tokenized_datasets['train'].rename_columns()
|
||||||
|
# %% temp
|
||||||
|
tokenized_datasets['train']['input_ids']
|
||||||
|
|
||||||
|
# %%
|
||||||
|
# create data collator
|
||||||
|
|
||||||
|
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
|
||||||
|
|
||||||
|
# %%
|
||||||
|
# compute metrics
|
||||||
|
metric = evaluate.load("accuracy")
|
||||||
|
|
||||||
|
|
||||||
|
def compute_metrics(eval_preds):
|
||||||
|
preds, labels = eval_preds
|
||||||
|
preds = np.argmax(preds, axis=1)
|
||||||
|
return metric.compute(predictions=preds, references=labels)
|
||||||
|
|
||||||
|
# %%
|
||||||
|
# create id2label and label2id
|
||||||
|
|
||||||
|
|
||||||
|
# %%
|
||||||
|
model = AutoModelForSequenceClassification.from_pretrained(
|
||||||
|
model_checkpoint,
|
||||||
|
num_labels=len(mdm_list),
|
||||||
|
id2label=id2label,
|
||||||
|
label2id=label2id)
|
||||||
|
# important! after extending tokens vocab
|
||||||
|
model.resize_token_embeddings(len(tokenizer))
|
||||||
|
|
||||||
|
# model = torch.compile(model, backend="inductor", dynamic=True)
|
||||||
|
|
||||||
|
|
||||||
|
# %%
|
||||||
|
# Trainer
|
||||||
|
|
||||||
|
training_args = TrainingArguments(
|
||||||
|
output_dir=f"{save_path}",
|
||||||
|
eval_strategy="epoch",
|
||||||
|
logging_dir="tensorboard-log",
|
||||||
|
logging_strategy="epoch",
|
||||||
|
save_strategy="epoch",
|
||||||
|
load_best_model_at_end=True,
|
||||||
|
learning_rate=2e-5,
|
||||||
|
per_device_train_batch_size=64,
|
||||||
|
per_device_eval_batch_size=64,
|
||||||
|
auto_find_batch_size=False,
|
||||||
|
ddp_find_unused_parameters=False,
|
||||||
|
weight_decay=0.01,
|
||||||
|
save_total_limit=1,
|
||||||
|
num_train_epochs=40,
|
||||||
|
bf16=True,
|
||||||
|
push_to_hub=False,
|
||||||
|
remove_unused_columns=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
trainer = Trainer(
|
||||||
|
model,
|
||||||
|
training_args,
|
||||||
|
train_dataset=tokenized_datasets["train"],
|
||||||
|
eval_dataset=tokenized_datasets["validation"],
|
||||||
|
tokenizer=tokenizer,
|
||||||
|
data_collator=data_collator,
|
||||||
|
compute_metrics=compute_metrics,
|
||||||
|
# callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
|
||||||
|
)
|
||||||
|
|
||||||
|
# uncomment to load training from checkpoint
|
||||||
|
# checkpoint_path = 'default_40_1/checkpoint-5600'
|
||||||
|
# trainer.train(resume_from_checkpoint=checkpoint_path)
|
||||||
|
|
||||||
|
trainer.train()
|
||||||
|
|
||||||
|
# # execute training
|
||||||
|
# for fold in [1,2,3,4,5]:
|
||||||
|
# print(fold)
|
||||||
|
# train(fold)
|
||||||
|
|
||||||
|
|
||||||
|
# %%
|
|
@ -90,7 +90,7 @@ class Inference():
|
||||||
datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
|
datasets.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
|
||||||
|
|
||||||
# create dataloader
|
# create dataloader
|
||||||
self.dataloader = DataLoader(datasets, batch_size=batch_size)
|
self.dataloader = DataLoader(datasets, batch_size=batch_size, shuffle=False)
|
||||||
|
|
||||||
|
|
||||||
def generate(self):
|
def generate(self):
|
||||||
|
|
Loading…
Reference in New Issue