Shortcuts

Source code for nets.solver.solver

from abc import ABC, abstractmethod

# Data science
from sklearn.metrics import f1_score, recall_score, precision_score, accuracy_score, confusion_matrix
import time
import json
import os
import copy

import nets
from nets.utils import append2dict, progress_bar, describe_stats


[docs]def get_metrics(gold_labels, predictions, labels=None): """Compute metrics for given predictions. This method returns basic metrics as a dictionary. Args: gold_labels (Tensor): true labels. predictions (Tensor): predicted labels. labels (list, optional): list of classes indices. Returns: dict """ if isinstance(gold_labels, nets.Tensor): gold_labels = gold_labels.numpy() if isinstance(predictions, nets.Tensor): predictions = predictions.numpy() metrics = {"accuracy": float(accuracy_score(gold_labels, predictions)), "precision": float(precision_score(gold_labels, predictions, labels=labels, average='macro')), "recall": float(recall_score(gold_labels, predictions, labels=labels, average='macro')), "macro_f1": float(f1_score(gold_labels, predictions, labels=labels, average='macro')), "confusion_matrix": confusion_matrix(gold_labels, predictions, labels=labels).tolist()} return metrics
[docs]class Solver(ABC): r"""Train and evaluate models. Args: model (Module): model to optimize or test. criterion (Loss): loss function. optimizer (Optimizer): optimizer for weights and biases. Attributes:: model (Module): model to optimize or test. checkpoint (dict): checkpoint of the best model tested. criterion (Loss): loss function. optimizer (Optimizer): optimizer for weights and biases. performance (dict): - train (dict): - loss (list(float)) - accuracy (list(float)) - precision (list(float)) - recall (list(float)) - macro_f1 (list(float)) - confusion_matrix (list(list(list(int)))) - eval (dict): - loss (list(float)) - accuracy (list(float)) - precision (list(float)) - recall (list(float)) - macro_f1 (list(float)) - confusion_matrix (list(list(list(int)))) """ def __init__(self, model=None, criterion=None, optimizer=None): self.model = model self.best_model = None self.checkpoint = {'epoch': None, 'model_name': None, 'model_state_dict': None, 'optimizer_name': None, 'optimizer_state_dict': None, 'train': None, 'eval': None } self.criterion = criterion self.optimizer = optimizer # Performances self.performance = None self.reset()
[docs] def reset(self): """Reset the performance dictionary. Returns: None """ self.performance = {"train": {"loss": [], "accuracy": [], "precision": [], "recall": [], "macro_f1": [], "confusion_matrix": []}, "eval": {"loss": [], "accuracy": [], "precision": [], "recall": [], "macro_f1": [], "confusion_matrix": []}}
[docs] @abstractmethod def train(self, iterator, *args, **kwargs): r"""Train one time the model on iterator data. Args: iterator (Iterator): iterator containing batch samples of data. Returns: dict: the performance and metrics of the training session. """ raise NotImplementedError
[docs] @abstractmethod def evaluate(self, iterator, *args, **kwargs): r"""Evaluate one time the model on iterator data. Args: iterator (Iterator): iterator containing batch samples of data. Returns: dict: the performance and metrics of the training session. """ raise NotImplementedError
def _update_checkpoint(self, epoch, results_train=None, results_eval=None): r"""Update the model's checkpoint. Keep track of its epoch, state, optimizer, and performances. In addition, it saves the current model in `best_model`. Args: epoch (int): epoch at the current training state. results_train (dict, optional): metrics for the training session at epoch. The default is None. results_eval (dict, optional): metrics for the evaluation session at epoch. The default is None. """ self.best_model = copy.deepcopy(self.model) self.checkpoint = {'epoch': epoch, 'model_name': self.best_model.__class__.__name__, 'model_state_dict': self.best_model.state_dict(), 'optimizer_name': self.optimizer.__class__.__name__, 'criterion_name': self.criterion.__class__.__name__, 'train': results_train, 'eval': results_eval }
[docs] def save(self, filename=None, dirpath=".", checkpoint=True): r"""Save the best torch model. Args: filename (str, optional): name of the model. The default is "model.pt". dirpath (str, optional): path to the desired foldre location. The default is ".". Returns: None. """ filename = 'model_' + self.checkpoint['model_name'] + '.pt' if filename is None else filename # Save the best model path = os.path.join(dirpath, filename) nets.save(self.best_model, path) # Save its checkpoint if checkpoint: checkname = 'checkpoint_' + filename.split('.')[-2].split('_')[-1] \ + '_epoch' + str(self.checkpoint['epoch']) + '.pt' checkpath = os.path.join(dirpath, checkname) nets.save(self.checkpoint, checkpath)
[docs] def get_accuracy(self, y_tilde, y): r"""Compute accuracy from predicted classes and gold labels. Args: y_tilde (Tensor): 1D tensor containing the predicted classes for each predictions in the batch. This tensor should be computed through `get_predicted_classes(y_hat)` method. y (Tensor): gold labels. Note that y_tilde an y must have the same shape. Returns: float: the mean of correct answers. """ assert y_tilde.shape == y.shape, ("predicted classes and gold labels should have the same shape") correct = (y_tilde == y).astype(float) # convert into float for division acc = correct.sum() / len(correct) return acc
[docs] def run(self, epochs, train_iterator, eval_iterator, *args, **kwargs): r"""Train and evaluate a model X times. During the training, both training and evaluation results are saved under the `performance` attribute. Args: epochs (int): number of times the model will be trained. train_iterator (Iterator): iterator containing batch samples of data. eval_iterator (Iterator): iterator containing batch samples of data. verbose (bool, optional): if `True` display a progress bar and metrics at each epoch. The default is True. Returns: None Examples:: >>> performer = MySolver(model, criterion, optimizer) >>> # Train & eval EPOCHS times >>> EPOCHS = 10 >>> performer.run(EPOCHS, train_iterator, eval_iterator, verbose=True) Epoch: 1/10 Training: 100% | [==================================================] | Time: 2m 26s Validation: 100% | [==================================================] | Time: 0m 4s Stats Training: | Loss: 0.349 | Acc: 84.33% | Prec.: 84.26% | Rec.: 84.33% | F1: 84.26% Stats Evaluation: | Loss: 0.627 | Acc: 72.04% | Prec.: 72.22% | Rec.: 72.17% | F1: 72.22% >>> # ... """ # By default, print a log each epoch verbose = True if 'verbose' not in {*kwargs} else kwargs['verbose'] # Keep track of the best model best_eval_accuracy = 0 # Train and evaluate the model epochs times for epoch in range(epochs): if verbose: print("Epoch:\t{0:3d}/{1}".format(epoch + 1, epochs)) # Train and evaluate the model results_train = self.train(train_iterator, *args, **kwargs) results_eval = self.evaluate(eval_iterator, *args, **kwargs) # Update the eval dictionary by adding the results at the # current epoch append2dict(self.performance["train"], results_train) append2dict(self.performance["eval"], results_eval) if verbose: print("\tStats Train: | " + describe_stats(results_train)) print("\t Stats Eval: | " + describe_stats(results_eval)) print() # We copy in memory the best model if best_eval_accuracy < self.performance["eval"]["accuracy"][-1]: best_eval_accuracy = self.performance["eval"]["accuracy"][-1] self._update_checkpoint(epoch + 1, results_train, results_eval)

Docs

Access comprehensive developer documentation for Nets

View Docs

Tutorials

Get beginners tutorials and create state-of-the-art models

View Tutorials

Resources

Check the GitHub page and contribute to the project

View GitHub