我修改过的代码在下面（只是在原来的基础上添加了菜单和常量），但我正在寻找实现 F1 分数的正确方法（位于# TODO / F1 =）
我正在寻找整个运行的宏 F1，而不是每个时期。
from __future__ import print_function
from __future__ import division
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
from torchvision import datasets, models, transforms
import time
import os
import copy
import neptune
from GPUtil import showUtilization as gpu_usage
from numba import cuda
import numpy as np
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score
loss_function = nn.CrossEntropyLoss
optimizer_used = optim.SGD
# Optimizer momentum
momentum = 0.9
learning_rate = 0.001
# Clear memory
def free_gpu_cache():
print("Initial GPU Usage")
gpu_usage()
torch.cuda.empty_cache()
cuda.select_device(0)
cuda.close()
cuda.select_device(0)
print("GPU Usage after emptying the cache")
gpu_usage()
free_gpu_cache()
print("PyTorch Version: ", torch.__version__)
print("Torchvision Version: ", torchvision.__version__)
`print("\n1. resnet18\n2. resnet152\n3. alexnet\n4. vgg\n5. squeezenet\n6. densenet\n7. inception\n")`
extraction_method = input("Enter the model you want to train: ")
if extraction_method == "1":
model_name = "resnet18"
elif extraction_method == "2":
model_name = "resnet152"
elif extraction_method == "3":
model_name = "alexnet"
elif extraction_method == "4":
model_name = "vgg"
elif extraction_method == "5":
model_name = "squeezenet"
elif extraction_method == "6":
model_name = "densenet"
elif extraction_method == "7":
model_name = "inception"
# num_classes is the number of classes in the dataset
# feature_extract is a boolean that defines if fine-tuning or feature extracting. If feature_extract = False,
# the model is fine-tuned and all model parameters are updated.
# If feature_extract = True, only the last layer parameters are updated, the others remain fixed.
# Top level data directory. Format of the directory conforms must be same as the image folder structure
print("\n1. Trained on full mammogram images, tested on ultrasound\n2. Trained on ROI mammogram images, tested on "
"ultrasound\n3. Trained on ultrasound images, tested on ultrasound\n")
extraction_method = input("Enter the image training type required: ")
if extraction_method == "1":
data_dir = "data/mammogram"
test_set = "data/mammogram/val"
num_classes = 2
elif extraction_method == "2":
data_dir = "data/mgroi"
test_set = "data/mgroi/val"
num_classes = 2
elif extraction_method == "3":
data_dir = "data/ultrasound"
test_set = "data/ultrasound/val"
num_classes = 3
else:
exit()
batch_size = int(input("Enter batch size: "))
num_epochs = int(input("Enter number of epochs: "))
# Feature extracting. When False, fine-tune the whole model,
# when True only update the reshaped layer params
`print("\n1. Train from scratch / fine-tune whole model\n2. Feature extract / reshape final layer parameters\n")`
extraction_method = input("Enter the training required: ")
if extraction_method == "1":
feature_extract = False
elif extraction_method == "2":
feature_extract = True
else:
exit()
# Start Neptune run to log data
# run = neptune.init_run(
# project="tim-osmond/Train-MG-Test-US",
# api_token="x",
# )
#
# # Neptune parameters to log
# params = {
# "epochs": num_epochs,
# "model name": model_name,
# "data directory": data_dir,
# "batch size": batch_size,
# "feature extract": feature_extract
# }
# run["parameters"] = params
#
# # Neptune does not support using static names so added as run commands as work around
# run["optimizer"] = stringify_unsupported(optimizer_used)
# run["criterion"] = stringify_unsupported(loss_function)
# run["learning rate"] = stringify_unsupported(learning_rate)
# run["momentum"] = stringify_unsupported(momentum)
# The train_model function takes a PyTorch model, a dictionary of dataloaders, a loss function, an optimizer,
# the specified number of epochs to train and validate for, and a boolean flag for when the model is an Inception model.
# The is_inception flag is used to accommodate the Inception v3 model, as that architecture uses an auxiliary output
# and the overall model loss respects both the auxiliary output and the final output. The function trains for the
# specified number of epochs and after each epoch runs a full validation step. It also keeps track of the best
# performing model (in terms of validation accuracy), and at the end of training returns the best performing model.
# After each epoch, the training and validation accuracies are printed.
# TODO check why epochs=25
def train_model(model, dataloaders, criterion, optimizer, epochs=25, is_inception=False):
since = time.time()
val_acc_history = []
best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0
for epoch in range(epochs):
print('\nEpoch {}/{}'.format(epoch + 1, epochs))
print('*' * 10)
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
running_loss = 0.0
running_corrects = 0
# Iterate over data.
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
labels = labels.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
# Get model outputs and calculate loss
# Special case for inception because in training it has an auxiliary output. In train
# mode calculate the loss by summing the final output and the auxiliary output
# but in testing only consider the final output.
if is_inception and phase == 'train':
` # From https://discuss.pytorch.org/t/how-to-optimize-inception-model-with-auxiliary-classifiers/7958`
outputs, aux_outputs = model(inputs)
loss1 = criterion(outputs, labels)
loss2 = criterion(aux_outputs, labels)
loss = loss1 + 0.4 * loss2
else:
outputs = model(inputs)
loss = criterion(outputs, labels)
_, preds = torch.max(outputs, 1)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.item() * inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / len(dataloaders[phase].dataset)
epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)
**# TODO**
F1 = 2 * (precision * recall) / (precision + recall)
tp = np.sum((actual==label) & (predicted==label))
fp = np.sum((actual != label) & (predicted == label))
fn = np.sum((predicted != label) & (actual == label))
precision = tp / (tp + fp)
recall = tp / (tp + fn)
f1 = 2 * (precision * recall) / (precision + recall)
print(f'F1 Score = {f1}')
# TODO Neptune stats
# if phase == 'train':
# run["training loss"].append(epoch_loss)
# run["training accuracy"].append(epoch_acc)
#
# if phase == 'val':
# run["validation loss"].append(epoch_loss)
# run["validation accuracy"].append(epoch_acc)
# log F1 score to neptune
# run["f1_score"] = f1
print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))
# deep copy the model
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
if phase == 'val':
val_acc_history.append(epoch_acc)
print()
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
# load best model weights
model.load_state_dict(best_model_wts)
return model, val_acc_history
`# This helper function sets the .requires_grad attribute of the parameters in the model to False when feature extracting.`
# By default, when loading a pretrained model all the parameters have .requires_grad=True,
`# which is fine if training from scratch or fine-tuning. However, if feature extracting and only `want`
# to compute gradients for the newly initialized layer then we want all the other parameters to not require`
# gradients.
def set_parameter_requires_grad(model, feature_extracting):
if feature_extracting:
for param in model.parameters():
param.requires_grad = False
`# Notice, many of the models have similar output structures, but each must be handled slightly differently. Also,
# check out the printed model architecture of the reshaped network and make sure the number of output features is the
# same as the number of classes in the dataset.`
def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):
# Initialize these variables which will be set in this if statement. Each of these
# variables is model specific.
model_ft = None
input_size = 0
if model_name == "resnet18":
# Resnet18
model_ft = models.resnet18(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, num_classes)
input_size = 224
elif model_name == "resnet152":
# Resnet152
model_ft = models.resnet152(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, num_classes)
input_size = 224
elif model_name == "alexnet":
# Alexnet
model_ft = models.alexnet(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
num_ftrs = model_ft.classifier[6].in_features
model_ft.classifier[6] = nn.Linear(num_ftrs, num_classes)
input_size = 224
elif model_name == "vgg":
# VGG11_bn
model_ft = models.vgg11_bn(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
num_ftrs = model_ft.classifier[6].in_features
model_ft.classifier[6] = nn.Linear(num_ftrs, num_classes)
input_size = 224
elif model_name == "squeezenet":
# Squeezenet
model_ft = models.squeezenet1_0(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
model_ft.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1, 1), stride=(1, 1))
model_ft.num_classes = num_classes
input_size = 224
elif model_name == "densenet":
# Densenet
model_ft = models.densenet121(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
num_ftrs = model_ft.classifier.in_features
model_ft.classifier = nn.Linear(num_ftrs, num_classes)
input_size = 224
elif model_name == "inception":
# Inception v3
# Be careful, expects (299,299) sized images and has auxiliary output
model_ft = models.inception_v3(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
# Handle the auxilary net
num_ftrs = model_ft.AuxLogits.fc.in_features
model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)
# Handle the primary net
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, num_classes)
input_size = 299
return model_ft, input_size
# Initialize the model for this run
model_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True)
# Print the model just instantiated
print(model_ft)
# Initialize the data transforms, image datasets, and the dataloaders.
# Notice, the models were pretrained with the hard-coded normalization values, as described here.
# Data augmentation and normalization for training
# Just normalization for validation
data_transforms = {
'train': transforms.Compose([
transforms.RandomResizedCrop(input_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
'val': transforms.Compose([
transforms.Resize(input_size),
transforms.CenterCrop(input_size),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
}
print("\nInitializing Datasets and Dataloaders...")
# Create training and validation datasets
image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'val']}
# Create training and validation dataloaders
dataloaders_dict = {
x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True, num_workers=4) for x in
['train', 'val']}
# Detect if GPU available
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
`# Create an optimizer that only updates the desired parameters. After loading the pretrained model,
# but before reshaping, if feature_extract=True we manually set all the parameter’s. requires_grad attributes to False.
# Then the reinitialized layer’s parameters have .requires_grad=True by default. So now we know that all parameters
# that have .requires_grad=True should be optimized. Next, we make a list of such parameters and input this list to
# the optimizer algorithm constructor.
# To verify this, check out the printed parameters to learn. When fine-tuning, this list should be long and include
# all the model parameters. However, when feature extracting this list should be short and only include the
# weights and biases of the reshaped layers.
# Send the model to GPU`
model_ft = model_ft.to(device)
# Gather the parameters to be optimized/updated in this run. If we are
# fine-tuning we will be updating all parameters. However, if we are
# doing feature extract method, we will only update the parameters
# that we have just initialized, i.e. the parameters with requires_grad
# is True.
params_to_update = model_ft.parameters()
print("Params to learn:\n")
if feature_extract:
params_to_update = []
for name, param in model_ft.named_parameters():
if param.requires_grad == True:
params_to_update.append(param)
print("\t", name)
else:
for name, param in model_ft.named_parameters():
if param.requires_grad == True:
print("\t", name)
# Observe that all parameters are being optimized
optimizer = optimizer_used(params_to_update, lr=learning_rate, momentum=momentum)
# Set up the loss fxn
criterion = loss_function()
# Run the training and validation function for the set number of epochs.
`# The default learning rate is not optimal for all the models, so to achieve maximum accuracy it would be`
# necessary to tune for each model separately.
# Train and evaluate
model_ft, hist = train_model(model_ft, dataloaders_dict, criterion, optimizer, epochs=num_epochs,
is_inception=(model_name == "inception"))
# Finish export to Neptune
# run.stop()
我搜索了很多选项，但我对从这个模型中提取精度和召回率感到困惑。