Load

In [ ]:

import torch
from torch.utils.data import Dataset, DataLoader
import torch.nn as nn
from sklearn.model_selection import train_test_split
import pandas as pd
from datasets import DatasetDict

import torch from torch.utils.data import Dataset, DataLoader import torch.nn as nn from sklearn.model_selection import train_test_split import pandas as pd from datasets import DatasetDict

In [ ]:

pad_token = 0
start_token = 1
end_token = 2 
class MyDataset(Dataset):
    def __init__(self, description, diagnosis, tokenizer, max_len):
        self.description = description
        self.diagnosis = diagnosis
        self.tokenizer = tokenizer
        self.max_len = max_len

    def __len__(self):
        return len(self.description)

    def __getitem__(self, idx):
        description_encoding = self.tokenizer(
            self.description[idx],
            add_special_tokens=True,
            max_length=self.max_len,
            padding='max_length',
            truncation=True,
            return_tensors='pt'
        )
        diagnosis_encoding = self.tokenizer(
            self.diagnosis[idx],
            add_special_tokens=True,
            max_length=self.max_len,
            padding='max_length',
            truncation=True,
            return_tensors='pt'
        )
        # 紧凑化
        return {
            'input_ids': description_encoding['input_ids'].flatten(),
            'attention_mask': description_encoding['attention_mask'].flatten(),
            'labels': diagnosis_encoding['input_ids'].flatten().tolist()
        }

pad_token = 0 start_token = 1 end_token = 2 class MyDataset(Dataset): def __init__(self, description, diagnosis, tokenizer, max_len): self.description = description self.diagnosis = diagnosis self.tokenizer = tokenizer self.max_len = max_len def __len__(self): return len(self.description) def __getitem__(self, idx): description_encoding = self.tokenizer( self.description[idx], add_special_tokens=True, max_length=self.max_len, padding='max_length', truncation=True, return_tensors='pt' ) diagnosis_encoding = self.tokenizer( self.diagnosis[idx], add_special_tokens=True, max_length=self.max_len, padding='max_length', truncation=True, return_tensors='pt' ) # 紧凑化 return { 'input_ids': description_encoding['input_ids'].flatten(), 'attention_mask': description_encoding['attention_mask'].flatten(), 'labels': diagnosis_encoding['input_ids'].flatten().tolist() }

In [ ]:

class MyDataset1(Dataset):
    def __init__(self, data, max_len=160):
        self.description = data['description'].tolist()
        self.diagnosis = data['diagnosis'].tolist()
        self.max_len = max_len
        
    def __len__(self):
        return len(self.description)

    def __getitem__(self, idx):
        #先处理description
        description_list = [int(num) for num in self.description[idx].split()]
        description_len = len(description_list)
        description = description_list[:self.max_len]
        description = torch.tensor(description)
        description = torch.cat((description, torch.tensor([pad_token] * (self.max_len - len(description)), dtype=torch.long)))
        #再处理diagnosis
        diagnosis_list = [int(num) for num in self.diagnosis[idx].split()]        
        diagnosis_len = len(diagnosis_list)
        diagnosis = diagnosis_list[:self.max_len]
        diagnosis = torch.tensor(diagnosis)
        diagnosis = torch.cat((diagnosis, torch.tensor([pad_token] * (self.max_len - len(diagnosis)), dtype=torch.long)))
        # 终止符
        description[description_len] = end_token
        diagnosis[diagnosis_len] = end_token
        return description, diagnosis, description_len, diagnosis_len

class MyDataset1(Dataset): def __init__(self, data, max_len=160): self.description = data['description'].tolist() self.diagnosis = data['diagnosis'].tolist() self.max_len = max_len def __len__(self): return len(self.description) def __getitem__(self, idx): #先处理description description_list = [int(num) for num in self.description[idx].split()] description_len = len(description_list) description = description_list[:self.max_len] description = torch.tensor(description) description = torch.cat((description, torch.tensor([pad_token] * (self.max_len - len(description)), dtype=torch.long))) #再处理diagnosis diagnosis_list = [int(num) for num in self.diagnosis[idx].split()] diagnosis_len = len(diagnosis_list) diagnosis = diagnosis_list[:self.max_len] diagnosis = torch.tensor(diagnosis) diagnosis = torch.cat((diagnosis, torch.tensor([pad_token] * (self.max_len - len(diagnosis)), dtype=torch.long))) # 终止符 description[description_len] = end_token diagnosis[diagnosis_len] = end_token return description, diagnosis, description_len, diagnosis_len

In [ ]:

def load(batch_size=64):
    train_data = pd.read_csv('train.csv')
    test_data = pd.read_csv('test.csv')
    train_data, val_data = train_test_split(train_data, test_size=0.2, random_state=42, shuffle=True)
    train_dataset = MyDataset1(train_data)
    val_dataset = MyDataset1(val_data)
    test_dataset = MyDataset1(test_data)
    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)
    return train_loader, val_loader, test_loader

def load(batch_size=64): train_data = pd.read_csv('train.csv') test_data = pd.read_csv('test.csv') train_data, val_data = train_test_split(train_data, test_size=0.2, random_state=42, shuffle=True) train_dataset = MyDataset1(train_data) val_dataset = MyDataset1(val_data) test_dataset = MyDataset1(test_data) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True) test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True) return train_loader, val_loader, test_loader

In [ ]:

def load_easy(tokenizer, MAX_LEN):
    train_data = pd.read_csv('train.csv')
    test_data = pd.read_csv('test.csv')
    # 划分训练集和验证集
    X = train_data['description'].values
    y = train_data['diagnosis'].values
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=True, random_state=42)
    train_dataset = MyDataset(X_train, y_train, tokenizer, MAX_LEN)
    val_dataset = MyDataset(X_test, y_test, tokenizer, MAX_LEN)
    test_dataset = MyDataset(test_data['description'].values, test_data['diagnosis'].values, tokenizer, MAX_LEN)
    return train_dataset, val_dataset, test_dataset

def load_easy(tokenizer, MAX_LEN): train_data = pd.read_csv('train.csv') test_data = pd.read_csv('test.csv') # 划分训练集和验证集 X = train_data['description'].values y = train_data['diagnosis'].values X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=True, random_state=42) train_dataset = MyDataset(X_train, y_train, tokenizer, MAX_LEN) val_dataset = MyDataset(X_test, y_test, tokenizer, MAX_LEN) test_dataset = MyDataset(test_data['description'].values, test_data['diagnosis'].values, tokenizer, MAX_LEN) return train_dataset, val_dataset, test_dataset

In [ ]: