{
"cells": [
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The autoreload extension is already loaded. To reload it, use:\n",
" %reload_ext autoreload\n"
]
}
],
"source": [
"import os\n",
"import numpy as np\n",
"import torch\n",
"import torch.nn as nn\n",
"import torchvision\n",
"import PIL\n",
"from torchvision import transforms\n",
"from sklearn.metrics import average_precision_score\n",
"from PIL import Image, ImageDraw\n",
"import matplotlib.pyplot as plt\n",
"from kaggle_submission import output_submission_csv\n",
"from classifier import SimpleClassifier, Classifier#, AlexNet\n",
"from voc_dataloader import VocDataset, VOC_CLASSES\n",
"\n",
"%matplotlib inline\n",
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Part 1B: Design your own network\n",
"\n",
"In this notebook, your task is to create and train your own model for multi-label classification on VOC Pascal.\n",
"\n",
"## What to do\n",
"1. You will make change on network architecture in ```classifier.py```.\n",
"2. You may also want to change other hyperparameters to assist your training to get a better performances. Hints will be given in the below instructions.\n",
"\n",
"## What to submit\n",
"Check the submission template for details what to submit. "
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def train_classifier(train_loader, classifier, criterion, optimizer):\n",
" classifier.train()\n",
" loss_ = 0.0\n",
" losses = []\n",
" for i, (images, labels, _) in enumerate(train_loader):\n",
" images, labels = images.to(device), labels.to(device)\n",
" optimizer.zero_grad()\n",
" logits = classifier(images)\n",
" loss = criterion(logits, labels)\n",
" loss.backward()\n",
" optimizer.step()\n",
" losses.append(loss)\n",
" return torch.stack(losses).mean().item()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"def test_classifier(test_loader, classifier, criterion, print_ind_classes=True, print_total=True):\n",
" classifier.eval()\n",
" losses = []\n",
" with torch.no_grad():\n",
" y_true = np.zeros((0,21))\n",
" y_score = np.zeros((0,21))\n",
" for i, (images, labels, _) in enumerate(test_loader):\n",
" images, labels = images.to(device), labels.to(device)\n",
" logits = classifier(images)\n",
" y_true = np.concatenate((y_true, labels.cpu().numpy()), axis=0)\n",
" y_score = np.concatenate((y_score, logits.cpu().numpy()), axis=0)\n",
" loss = criterion(logits, labels)\n",
" losses.append(loss.item())\n",
" aps = []\n",
" # ignore first class which is background\n",
" for i in range(1, y_true.shape[1]):\n",
" ap = average_precision_score(y_true[:, i], y_score[:, i])\n",
" if print_ind_classes:\n",
" print('------- Class: {:<12} AP: {:>8.4f} -------'.format(VOC_CLASSES[i], ap))\n",
" aps.append(ap)\n",
" \n",
" mAP = np.mean(aps)\n",
" test_loss = np.mean(losses)\n",
" if print_total:\n",
" print('mAP: {0:.4f}'.format(mAP))\n",
" print('Avg loss: {}'.format(test_loss))\n",
" \n",
" return mAP, test_loss, aps"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"def plot_losses(train, val, test_frequency, num_epochs):\n",
" plt.plot(train, label=\"train\")\n",
" indices = [i for i in range(num_epochs) if ((i+1)%test_frequency == 0 or i ==0)]\n",
" plt.plot(indices, val, label=\"val\")\n",
" plt.title(\"Loss Plot\")\n",
" plt.ylabel(\"Loss\")\n",
" plt.xlabel(\"Epoch\")\n",
" plt.legend()\n",
" plt.show()\n",
" \n",
"def plot_mAP(train, val, test_frequency, num_epochs):\n",
" indices = [i for i in range(num_epochs) if ((i+1)%test_frequency == 0 or i ==0)]\n",
" plt.plot(indices, train, label=\"train\")\n",
" plt.plot(indices, val, label=\"val\")\n",
" plt.title(\"mAP Plot\")\n",
" plt.ylabel(\"mAP\")\n",
" plt.xlabel(\"Epoch\")\n",
" plt.legend()\n",
" plt.show()\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"\n",
"def train(classifier, num_epochs, train_loader, val_loader, criterion, optimizer, test_frequency=5):\n",
" train_losses = []\n",
" train_mAPs = []\n",
" val_losses = []\n",
" val_mAPs = []\n",
"\n",
" for epoch in range(1,num_epochs+1):\n",
" print(\"Starting epoch number \" + str(epoch))\n",
" train_loss = train_classifier(train_loader, classifier, criterion, optimizer)\n",
" train_losses.append(train_loss)\n",
" print(\"Loss for Training on Epoch \" +str(epoch) + \" is \"+ str(train_loss))\n",
" if(epoch%test_frequency==0 or epoch==1):\n",
" mAP_train, _, _ = test_classifier(train_loader, classifier, criterion, False, False)\n",
" train_mAPs.append(mAP_train)\n",
" mAP_val, val_loss, _ = test_classifier(val_loader, classifier, criterion)\n",
" print('Evaluating classifier')\n",
" print(\"Mean Precision Score for Testing on Epoch \" +str(epoch) + \" is \"+ str(mAP_val))\n",
" val_losses.append(val_loss)\n",
" val_mAPs.append(mAP_val)\n",
" \n",
" return classifier, train_losses, val_losses, train_mAPs, val_mAPs"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Developing Your Own Model"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Goal\n",
"To meet the benchmark for this assignment you will need to improve the network. Note you should have noticed pretrained Alenxt performs really well, but training Alexnet from scratch performs much worse. We hope you can design a better architecture over both the simple classifier and AlexNet to train from scratch.\n",
"\n",
"### How to start\n",
"You may take inspiration from other published architectures and architectures discussed in lecture. However, you are NOT allowed to use predefined models (e.g. models from torchvision) or use pretrained weights. Training must be done from scratch with your own custom model.\n",
"\n",
"#### Some hints\n",
"There are a variety of different approaches you should try to improve performance from the simple classifier:\n",
"\n",
"* Network architecture changes\n",
" * Number of layers: try adding layers to make your network deeper\n",
" * Batch normalization: adding batch norm between layers will likely give you a significant performance increase\n",
" * Residual connections: as you increase the depth of your network, you will find that having residual connections like those in ResNet architectures will be helpful\n",
"* Optimizer: Instead of plain SGD, you may want to add a learning rate schedule, add momentum, or use one of the other optimizers you have learned about like Adam. Check the `torch.optim` package for other optimizers\n",
"* Data augmentation: You should use the `torchvision.transforms` module to try adding random resized crops and horizontal flips of the input data. Check `transforms.RandomResizedCrop` and `transforms.RandomHorizontalFlip` for this. Feel free to apply more [transforms](https://pytorch.org/docs/stable/torchvision/transforms.html) for data augmentation which can lead to better performance. \n",
"* Epochs: Once you have found a generally good hyperparameter setting try training for more epochs\n",
"* Loss function: You might want to add weighting to the `MultiLabelSoftMarginLoss` for classes that are less well represented or experiment with a different loss function\n",
"\n",
"\n",
"\n",
"#### Note\n",
"We will soon be providing some initial expectations of mAP values as a function of epoch so you can get an early idea whether your implementation works without waiting a long time for training to converge.\n",
"\n",
"### What to submit \n",
"Submit your best model to Kaggle and save all plots for the writeup.\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")\n",
"\n",
"\n",
"normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
" std= [0.229, 0.224, 0.225])\n",
"\n",
"train_transform = transforms.Compose([\n",
" transforms.Resize(227),\n",
" transforms.ColorJitter(hue=.05, saturation=.05),\n",
" transforms.RandomHorizontalFlip(),\n",
" transforms.RandomRotation(20, resample=PIL.Image.BILINEAR),\n",
" transforms.CenterCrop(227),\n",
" transforms.ToTensor(),\n",
" normalize\n",
" ])\n",
"\n",
"test_transform = transforms.Compose([\n",
" transforms.Resize(227),\n",
" transforms.CenterCrop(227),\n",
" transforms.ToTensor(),\n",
" normalize,\n",
" ])\n",
"\n",
"ds_train = VocDataset('VOCdevkit_2007/VOC2007/','train',train_transform)\n",
"ds_val = VocDataset('VOCdevkit_2007/VOC2007/','val',test_transform)\n",
"ds_test = VocDataset('VOCdevkit_2007/VOC2007test/','test', test_transform)\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"#num_epochs = 100\n",
"#test_frequency = 5\n",
"batch_size = 64\n",
"\n",
"train_loader = torch.utils.data.DataLoader(dataset=ds_train,\n",
" batch_size=batch_size, \n",
" shuffle=True,\n",
" num_workers=1)\n",
"\n",
"val_loader = torch.utils.data.DataLoader(dataset=ds_val,\n",
" batch_size=batch_size, \n",
" shuffle=True,\n",
" num_workers=1)\n",
"\n",
"test_loader = torch.utils.data.DataLoader(dataset=ds_test,\n",
" batch_size=batch_size, \n",
" shuffle=False,\n",
" num_workers=1)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"AlexNet(\n",
" (features): Sequential(\n",
" (0): Conv2d(3, 64, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))\n",
" (1): ReLU(inplace=True)\n",
" (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
" (3): Conv2d(64, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
" (4): ReLU(inplace=True)\n",
" (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
" (6): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (7): ReLU(inplace=True)\n",
" (8): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (9): ReLU(inplace=True)\n",
" (10): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (11): ReLU(inplace=True)\n",
" (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (avgpool): AdaptiveAvgPool2d(output_size=(6, 6))\n",
" (classifier): Sequential(\n",
" (0): Dropout(p=0.5, inplace=False)\n",
" (1): Linear(in_features=9216, out_features=4096, bias=True)\n",
" (2): ReLU(inplace=True)\n",
" (3): Dropout(p=0.5, inplace=False)\n",
" (4): Linear(in_features=4096, out_features=4096, bias=True)\n",
" (5): ReLU(inplace=True)\n",
" (6): Linear(in_features=4096, out_features=21, bias=True)\n",
" )\n",
")\n",
"\n",
"\n",
"\n",
"VGG(\n",
" (features): Sequential(\n",
" (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (2): ReLU(inplace=True)\n",
" (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (5): ReLU(inplace=True)\n",
" (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
" (7): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (8): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (9): ReLU(inplace=True)\n",
" (10): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (11): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (12): ReLU(inplace=True)\n",
" (13): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
" (14): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (15): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (16): ReLU(inplace=True)\n",
" (17): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (18): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (19): ReLU(inplace=True)\n",
" (20): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (21): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (22): ReLU(inplace=True)\n",
" (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
" (24): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (25): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (26): ReLU(inplace=True)\n",
" (27): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (29): ReLU(inplace=True)\n",
" (30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (32): ReLU(inplace=True)\n",
" (33): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
" (34): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (35): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (36): ReLU(inplace=True)\n",
" (37): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (38): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (39): ReLU(inplace=True)\n",
" (40): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (41): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (42): ReLU(inplace=True)\n",
" (43): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (avgpool): AdaptiveAvgPool2d(output_size=(7, 7))\n",
" (classifier): Sequential(\n",
" (0): Linear(in_features=25088, out_features=4096, bias=True)\n",
" (1): ReLU(inplace=True)\n",
" (2): Dropout(p=0.5, inplace=False)\n",
" (3): Linear(in_features=4096, out_features=4096, bias=True)\n",
" (4): ReLU(inplace=True)\n",
" (5): Dropout(p=0.5, inplace=False)\n",
" (6): Linear(in_features=4096, out_features=1000, bias=True)\n",
" )\n",
")\n",
"=======================================================================\n",
"\n",
"\n",
"ResNet(\n",
" (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
" (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (relu): ReLU(inplace=True)\n",
" (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
" (layer1): Sequential(\n",
" (0): BasicBlock(\n",
" (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (relu): ReLU(inplace=True)\n",
" (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" )\n",
" (1): BasicBlock(\n",
" (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (relu): ReLU(inplace=True)\n",
" (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" )\n",
" )\n",
" (layer2): Sequential(\n",
" (0): BasicBlock(\n",
" (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
" (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (relu): ReLU(inplace=True)\n",
" (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (downsample): Sequential(\n",
" (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
" (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" )\n",
" )\n",
" (1): BasicBlock(\n",
" (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (relu): ReLU(inplace=True)\n",
" (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" )\n",
" )\n",
" (layer3): Sequential(\n",
" (0): BasicBlock(\n",
" (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
" (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (relu): ReLU(inplace=True)\n",
" (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (downsample): Sequential(\n",
" (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
" (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" )\n",
" )\n",
" (1): BasicBlock(\n",
" (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (relu): ReLU(inplace=True)\n",
" (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" )\n",
" )\n",
" (layer4): Sequential(\n",
" (0): BasicBlock(\n",
" (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
" (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (relu): ReLU(inplace=True)\n",
" (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (downsample): Sequential(\n",
" (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
" (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" )\n",
" )\n",
" (1): BasicBlock(\n",
" (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (relu): ReLU(inplace=True)\n",
" (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
" (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" )\n",
" )\n",
" (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
" (fc): Linear(in_features=512, out_features=1000, bias=True)\n",
")\n"
]
}
],
"source": [
"################################################\n",
"#My own experiment\n",
"################################################\n",
"num_epochs = 20\n",
"test_frequency = 5\n",
"\n",
"# Change classifier to AlexNet\n",
"classifier_2 = torchvision.models.alexnet(pretrained=False)\n",
"classifier_2.classifier._modules['6'] = nn.Linear(4096, 21) \n",
"classifier_2 = classifier_2.to(device)\n",
"\n",
"criterion_2 = nn.MultiLabelSoftMarginLoss()\n",
"\n",
"optimizer_2 = torch.optim.Adam(classifier_2.parameters(), lr=1e-4,betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False)\n",
"#classifier_2, train_losses_2, val_losses_2, train_mAPs_2, val_mAPs_2 = train(classifier_2, num_epochs, train_loader, val_loader, criterion_2, optimizer_2, test_frequency)\n",
"print(classifier_2)\n",
"print()\n",
"print()\n",
"print()\n",
"vgg=torchvision.models.vgg16_bn(pretrained=False)\n",
"print(vgg)\n",
"print(\"=======================================================================\")\n",
"print()\n",
"print()\n",
"print( torchvision.models.resnet18(pretrained=False))"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"----------------------------------------------------------------\n",
" Layer (type) Output Shape Param #\n",
"================================================================\n",
" Conv2d-1 [-1, 64, 56, 56] 23,296\n",
" ReLU-2 [-1, 64, 56, 56] 0\n",
" MaxPool2d-3 [-1, 64, 27, 27] 0\n",
" Conv2d-4 [-1, 192, 27, 27] 307,392\n",
" ReLU-5 [-1, 192, 27, 27] 0\n",
" MaxPool2d-6 [-1, 192, 13, 13] 0\n",
" Conv2d-7 [-1, 384, 13, 13] 663,936\n",
" ReLU-8 [-1, 384, 13, 13] 0\n",
" Conv2d-9 [-1, 256, 13, 13] 884,992\n",
" ReLU-10 [-1, 256, 13, 13] 0\n",
" Conv2d-11 [-1, 256, 13, 13] 590,080\n",
" ReLU-12 [-1, 256, 13, 13] 0\n",
" MaxPool2d-13 [-1, 256, 6, 6] 0\n",
"AdaptiveAvgPool2d-14 [-1, 256, 6, 6] 0\n",
" Dropout-15 [-1, 9216] 0\n",
" Linear-16 [-1, 4096] 37,752,832\n",
" ReLU-17 [-1, 4096] 0\n",
" Dropout-18 [-1, 4096] 0\n",
" Linear-19 [-1, 4096] 16,781,312\n",
" ReLU-20 [-1, 4096] 0\n",
" Linear-21 [-1, 21] 86,037\n",
"================================================================\n",
"Total params: 57,089,877\n",
"Trainable params: 57,089,877\n",
"Non-trainable params: 0\n",
"----------------------------------------------------------------\n",
"Input size (MB): 0.59\n",
"Forward/backward pass size (MB): 8.48\n",
"Params size (MB): 217.78\n",
"Estimated Total Size (MB): 226.85\n",
"----------------------------------------------------------------\n"
]
}
],
"source": [
"############################\n",
"# Options for my classifier begin here\n",
"#############################\n",
"# declare what device to use: gpu/cpu\n",
"from torchsummary import summary\n",
"summary(classifier_2, input_size=(3, 227, 227))\n"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Classifier(\n",
" (features): Sequential(\n",
" (0): Conv2d(3, 64, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))\n",
" (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)\n",
" (2): ReLU(inplace=True)\n",
" (3): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
" (4): Conv2d(64, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
" (5): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)\n",
" (6): ReLU(inplace=True)\n",
" (7): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
" (8): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (9): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)\n",
" (10): ReLU(inplace=True)\n",
" (11): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (12): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)\n",
" (13): ReLU(inplace=True)\n",
" (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
" (15): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)\n",
" (16): ReLU(inplace=True)\n",
" (17): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (avgpool): AdaptiveAvgPool2d(output_size=(6, 6))\n",
" (classifier): Sequential(\n",
" (0): Dropout(p=0.5, inplace=False)\n",
" (1): Linear(in_features=9216, out_features=4096, bias=True)\n",
" (2): ReLU(inplace=True)\n",
" (3): Dropout(p=0.5, inplace=False)\n",
" (4): Linear(in_features=4096, out_features=4096, bias=True)\n",
" (5): ReLU(inplace=True)\n",
" (6): Linear(in_features=4096, out_features=21, bias=True)\n",
" )\n",
")\n",
"----------------------------------------------------------------\n",
" Layer (type) Output Shape Param #\n",
"================================================================\n",
" Conv2d-1 [-1, 64, 56, 56] 23,296\n",
" BatchNorm2d-2 [-1, 64, 56, 56] 128\n",
" ReLU-3 [-1, 64, 56, 56] 0\n",
" MaxPool2d-4 [-1, 64, 27, 27] 0\n",
" Conv2d-5 [-1, 192, 27, 27] 307,392\n",
" BatchNorm2d-6 [-1, 192, 27, 27] 384\n",
" ReLU-7 [-1, 192, 27, 27] 0\n",
" MaxPool2d-8 [-1, 192, 13, 13] 0\n",
" Conv2d-9 [-1, 384, 13, 13] 663,936\n",
" BatchNorm2d-10 [-1, 384, 13, 13] 768\n",
" ReLU-11 [-1, 384, 13, 13] 0\n",
" Conv2d-12 [-1, 256, 13, 13] 884,992\n",
" BatchNorm2d-13 [-1, 256, 13, 13] 512\n",
" ReLU-14 [-1, 256, 13, 13] 0\n",
" Conv2d-15 [-1, 256, 13, 13] 590,080\n",
" BatchNorm2d-16 [-1, 256, 13, 13] 512\n",
" ReLU-17 [-1, 256, 13, 13] 0\n",
" MaxPool2d-18 [-1, 256, 6, 6] 0\n",
"AdaptiveAvgPool2d-19 [-1, 256, 6, 6] 0\n",
" Dropout-20 [-1, 9216] 0\n",
" Linear-21 [-1, 4096] 37,752,832\n",
" ReLU-22 [-1, 4096] 0\n",
" Dropout-23 [-1, 4096] 0\n",
" Linear-24 [-1, 4096] 16,781,312\n",
" ReLU-25 [-1, 4096] 0\n",
" Linear-26 [-1, 21] 86,037\n",
"================================================================\n",
"Total params: 57,092,181\n",
"Trainable params: 57,092,181\n",
"Non-trainable params: 0\n",
"----------------------------------------------------------------\n",
"Input size (MB): 0.59\n",
"Forward/backward pass size (MB): 12.23\n",
"Params size (MB): 217.79\n",
"Estimated Total Size (MB): 230.61\n",
"----------------------------------------------------------------\n",
"Starting epoch number 1\n",
"Loss for Training on Epoch 1 is 0.2579552233219147\n",
"------- Class: aeroplane AP: 0.3357 -------\n",
"------- Class: bicycle AP: 0.1210 -------\n",
"------- Class: bird AP: 0.1101 -------\n",
"------- Class: boat AP: 0.2499 -------\n",
"------- Class: bottle AP: 0.0802 -------\n",
"------- Class: bus AP: 0.1295 -------\n",
"------- Class: car AP: 0.2697 -------\n",
"------- Class: cat AP: 0.1959 -------\n",
"------- Class: chair AP: 0.2112 -------\n",
"------- Class: cow AP: 0.0403 -------\n",
"------- Class: diningtable AP: 0.1898 -------\n",
"------- Class: dog AP: 0.1587 -------\n",
"------- Class: horse AP: 0.1195 -------\n",
"------- Class: motorbike AP: 0.1534 -------\n",
"------- Class: person AP: 0.5468 -------\n",
"------- Class: pottedplant AP: 0.1208 -------\n",
"------- Class: sheep AP: 0.0379 -------\n",
"------- Class: sofa AP: 0.1344 -------\n",
"------- Class: train AP: 0.2385 -------\n",
"------- Class: tvmonitor AP: 0.0639 -------\n",
"mAP: 0.1754\n",
"Avg loss: 0.22726271748542787\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 1 is 0.17536151124143498\n",
"Starting epoch number 2\n",
"Loss for Training on Epoch 2 is 0.21826979517936707\n",
"Starting epoch number 3\n",
"Loss for Training on Epoch 3 is 0.2119084894657135\n",
"Starting epoch number 4\n",
"Loss for Training on Epoch 4 is 0.20369315147399902\n",
"Starting epoch number 5\n",
"Loss for Training on Epoch 5 is 0.20315158367156982\n",
"------- Class: aeroplane AP: 0.3626 -------\n",
"------- Class: bicycle AP: 0.2405 -------\n",
"------- Class: bird AP: 0.1528 -------\n",
"------- Class: boat AP: 0.1622 -------\n",
"------- Class: bottle AP: 0.0988 -------\n",
"------- Class: bus AP: 0.1112 -------\n",
"------- Class: car AP: 0.4536 -------\n",
"------- Class: cat AP: 0.2903 -------\n",
"------- Class: chair AP: 0.3473 -------\n",
"------- Class: cow AP: 0.1311 -------\n",
"------- Class: diningtable AP: 0.2157 -------\n",
"------- Class: dog AP: 0.2106 -------\n",
"------- Class: horse AP: 0.2748 -------\n",
"------- Class: motorbike AP: 0.2210 -------\n",
"------- Class: person AP: 0.6588 -------\n",
"------- Class: pottedplant AP: 0.1325 -------\n",
"------- Class: sheep AP: 0.1200 -------\n",
"------- Class: sofa AP: 0.2328 -------\n",
"------- Class: train AP: 0.2848 -------\n",
"------- Class: tvmonitor AP: 0.1616 -------\n",
"mAP: 0.2431\n",
"Avg loss: 0.21260209567844868\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 5 is 0.24314458486617122\n",
"Starting epoch number 6\n",
"Loss for Training on Epoch 6 is 0.19945751130580902\n",
"Starting epoch number 7\n",
"Loss for Training on Epoch 7 is 0.19383274018764496\n",
"Starting epoch number 8\n",
"Loss for Training on Epoch 8 is 0.1899273693561554\n",
"Starting epoch number 9\n",
"Loss for Training on Epoch 9 is 0.18524472415447235\n",
"Starting epoch number 10\n",
"Loss for Training on Epoch 10 is 0.18443499505519867\n",
"------- Class: aeroplane AP: 0.5229 -------\n",
"------- Class: bicycle AP: 0.3424 -------\n",
"------- Class: bird AP: 0.2069 -------\n",
"------- Class: boat AP: 0.2263 -------\n",
"------- Class: bottle AP: 0.1476 -------\n",
"------- Class: bus AP: 0.1413 -------\n",
"------- Class: car AP: 0.5335 -------\n",
"------- Class: cat AP: 0.3131 -------\n",
"------- Class: chair AP: 0.4004 -------\n",
"------- Class: cow AP: 0.2006 -------\n",
"------- Class: diningtable AP: 0.3209 -------\n",
"------- Class: dog AP: 0.2489 -------\n",
"------- Class: horse AP: 0.4278 -------\n",
"------- Class: motorbike AP: 0.2891 -------\n",
"------- Class: person AP: 0.6952 -------\n",
"------- Class: pottedplant AP: 0.1709 -------\n",
"------- Class: sheep AP: 0.1527 -------\n",
"------- Class: sofa AP: 0.2310 -------\n",
"------- Class: train AP: 0.3768 -------\n",
"------- Class: tvmonitor AP: 0.2199 -------\n",
"mAP: 0.3084\n",
"Avg loss: 0.19116417095065116\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 10 is 0.30841289285977125\n",
"Starting epoch number 11\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loss for Training on Epoch 11 is 0.18256309628486633\n",
"Starting epoch number 12\n",
"Loss for Training on Epoch 12 is 0.17831477522850037\n",
"Starting epoch number 13\n",
"Loss for Training on Epoch 13 is 0.1789664328098297\n",
"Starting epoch number 14\n",
"Loss for Training on Epoch 14 is 0.17364290356636047\n",
"Starting epoch number 15\n",
"Loss for Training on Epoch 15 is 0.16964001953601837\n",
"------- Class: aeroplane AP: 0.5740 -------\n",
"------- Class: bicycle AP: 0.3439 -------\n",
"------- Class: bird AP: 0.2256 -------\n",
"------- Class: boat AP: 0.2604 -------\n",
"------- Class: bottle AP: 0.1784 -------\n",
"------- Class: bus AP: 0.2091 -------\n",
"------- Class: car AP: 0.5984 -------\n",
"------- Class: cat AP: 0.3271 -------\n",
"------- Class: chair AP: 0.4666 -------\n",
"------- Class: cow AP: 0.1665 -------\n",
"------- Class: diningtable AP: 0.3468 -------\n",
"------- Class: dog AP: 0.2556 -------\n",
"------- Class: horse AP: 0.4742 -------\n",
"------- Class: motorbike AP: 0.4842 -------\n",
"------- Class: person AP: 0.7216 -------\n",
"------- Class: pottedplant AP: 0.1541 -------\n",
"------- Class: sheep AP: 0.1395 -------\n",
"------- Class: sofa AP: 0.2850 -------\n",
"------- Class: train AP: 0.4901 -------\n",
"------- Class: tvmonitor AP: 0.2508 -------\n",
"mAP: 0.3476\n",
"Avg loss: 0.18170567639172078\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 15 is 0.34759194415442923\n",
"Starting epoch number 16\n",
"Loss for Training on Epoch 16 is 0.16874007880687714\n",
"Starting epoch number 17\n",
"Loss for Training on Epoch 17 is 0.16736312210559845\n",
"Starting epoch number 18\n",
"Loss for Training on Epoch 18 is 0.163020059466362\n",
"Starting epoch number 19\n",
"Loss for Training on Epoch 19 is 0.16554924845695496\n",
"Starting epoch number 20\n",
"Loss for Training on Epoch 20 is 0.17070019245147705\n",
"------- Class: aeroplane AP: 0.5576 -------\n",
"------- Class: bicycle AP: 0.4350 -------\n",
"------- Class: bird AP: 0.2916 -------\n",
"------- Class: boat AP: 0.2841 -------\n",
"------- Class: bottle AP: 0.1602 -------\n",
"------- Class: bus AP: 0.2333 -------\n",
"------- Class: car AP: 0.6102 -------\n",
"------- Class: cat AP: 0.3825 -------\n",
"------- Class: chair AP: 0.4620 -------\n",
"------- Class: cow AP: 0.1699 -------\n",
"------- Class: diningtable AP: 0.3665 -------\n",
"------- Class: dog AP: 0.2665 -------\n",
"------- Class: horse AP: 0.5044 -------\n",
"------- Class: motorbike AP: 0.4951 -------\n",
"------- Class: person AP: 0.7533 -------\n",
"------- Class: pottedplant AP: 0.1846 -------\n",
"------- Class: sheep AP: 0.1917 -------\n",
"------- Class: sofa AP: 0.2975 -------\n",
"------- Class: train AP: 0.5083 -------\n",
"------- Class: tvmonitor AP: 0.2874 -------\n",
"mAP: 0.3721\n",
"Avg loss: 0.1762672808021307\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 20 is 0.3720877654347446\n",
"Starting epoch number 21\n",
"Loss for Training on Epoch 21 is 0.16286873817443848\n",
"Starting epoch number 22\n",
"Loss for Training on Epoch 22 is 0.1630406379699707\n",
"Starting epoch number 23\n",
"Loss for Training on Epoch 23 is 0.16148406267166138\n",
"Starting epoch number 24\n",
"Loss for Training on Epoch 24 is 0.1570442169904709\n",
"Starting epoch number 25\n",
"Loss for Training on Epoch 25 is 0.1540795862674713\n",
"------- Class: aeroplane AP: 0.5951 -------\n",
"------- Class: bicycle AP: 0.4239 -------\n",
"------- Class: bird AP: 0.2968 -------\n",
"------- Class: boat AP: 0.3095 -------\n",
"------- Class: bottle AP: 0.1637 -------\n",
"------- Class: bus AP: 0.2550 -------\n",
"------- Class: car AP: 0.6035 -------\n",
"------- Class: cat AP: 0.3583 -------\n",
"------- Class: chair AP: 0.4111 -------\n",
"------- Class: cow AP: 0.1807 -------\n",
"------- Class: diningtable AP: 0.3922 -------\n",
"------- Class: dog AP: 0.2473 -------\n",
"------- Class: horse AP: 0.4994 -------\n",
"------- Class: motorbike AP: 0.4774 -------\n",
"------- Class: person AP: 0.7476 -------\n",
"------- Class: pottedplant AP: 0.1771 -------\n",
"------- Class: sheep AP: 0.2043 -------\n",
"------- Class: sofa AP: 0.2684 -------\n",
"------- Class: train AP: 0.5100 -------\n",
"------- Class: tvmonitor AP: 0.2682 -------\n",
"mAP: 0.3695\n",
"Avg loss: 0.18335324190557004\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 25 is 0.3694674591641289\n",
"Starting epoch number 26\n",
"Loss for Training on Epoch 26 is 0.15583045780658722\n",
"Starting epoch number 27\n",
"Loss for Training on Epoch 27 is 0.15162751078605652\n",
"Starting epoch number 28\n",
"Loss for Training on Epoch 28 is 0.15160952508449554\n",
"Starting epoch number 29\n",
"Loss for Training on Epoch 29 is 0.14582422375679016\n",
"Starting epoch number 30\n",
"Loss for Training on Epoch 30 is 0.14083173871040344\n",
"------- Class: aeroplane AP: 0.6004 -------\n",
"------- Class: bicycle AP: 0.4131 -------\n",
"------- Class: bird AP: 0.3028 -------\n",
"------- Class: boat AP: 0.3291 -------\n",
"------- Class: bottle AP: 0.1763 -------\n",
"------- Class: bus AP: 0.2273 -------\n",
"------- Class: car AP: 0.5795 -------\n",
"------- Class: cat AP: 0.3793 -------\n",
"------- Class: chair AP: 0.4499 -------\n",
"------- Class: cow AP: 0.2396 -------\n",
"------- Class: diningtable AP: 0.3557 -------\n",
"------- Class: dog AP: 0.2421 -------\n",
"------- Class: horse AP: 0.5700 -------\n",
"------- Class: motorbike AP: 0.5042 -------\n",
"------- Class: person AP: 0.7694 -------\n",
"------- Class: pottedplant AP: 0.1763 -------\n",
"------- Class: sheep AP: 0.1624 -------\n",
"------- Class: sofa AP: 0.3135 -------\n",
"------- Class: train AP: 0.5423 -------\n",
"------- Class: tvmonitor AP: 0.2982 -------\n",
"mAP: 0.3816\n",
"Avg loss: 0.1869926393032074\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 30 is 0.3815685719174271\n",
"Starting epoch number 31\n",
"Loss for Training on Epoch 31 is 0.13976247608661652\n",
"Starting epoch number 32\n",
"Loss for Training on Epoch 32 is 0.1488606035709381\n",
"Starting epoch number 33\n",
"Loss for Training on Epoch 33 is 0.14547854661941528\n",
"Starting epoch number 34\n",
"Loss for Training on Epoch 34 is 0.14075598120689392\n",
"Starting epoch number 35\n",
"Loss for Training on Epoch 35 is 0.14369454979896545\n",
"------- Class: aeroplane AP: 0.6105 -------\n",
"------- Class: bicycle AP: 0.4361 -------\n",
"------- Class: bird AP: 0.3511 -------\n",
"------- Class: boat AP: 0.3084 -------\n",
"------- Class: bottle AP: 0.1770 -------\n",
"------- Class: bus AP: 0.2426 -------\n",
"------- Class: car AP: 0.6160 -------\n",
"------- Class: cat AP: 0.3557 -------\n",
"------- Class: chair AP: 0.4610 -------\n",
"------- Class: cow AP: 0.2404 -------\n",
"------- Class: diningtable AP: 0.3689 -------\n",
"------- Class: dog AP: 0.2722 -------\n",
"------- Class: horse AP: 0.5078 -------\n",
"------- Class: motorbike AP: 0.4901 -------\n",
"------- Class: person AP: 0.7772 -------\n",
"------- Class: pottedplant AP: 0.1706 -------\n",
"------- Class: sheep AP: 0.2236 -------\n",
"------- Class: sofa AP: 0.3197 -------\n",
"------- Class: train AP: 0.5837 -------\n",
"------- Class: tvmonitor AP: 0.3058 -------\n",
"mAP: 0.3909\n",
"Avg loss: 0.17877154015004634\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 35 is 0.3909212617946651\n",
"Starting epoch number 36\n",
"Loss for Training on Epoch 36 is 0.13688017427921295\n",
"Starting epoch number 37\n",
"Loss for Training on Epoch 37 is 0.13824954628944397\n",
"Starting epoch number 38\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loss for Training on Epoch 38 is 0.13404829800128937\n",
"Starting epoch number 39\n",
"Loss for Training on Epoch 39 is 0.13584311306476593\n",
"Starting epoch number 40\n",
"Loss for Training on Epoch 40 is 0.135951966047287\n",
"------- Class: aeroplane AP: 0.6141 -------\n",
"------- Class: bicycle AP: 0.4321 -------\n",
"------- Class: bird AP: 0.3064 -------\n",
"------- Class: boat AP: 0.3494 -------\n",
"------- Class: bottle AP: 0.1876 -------\n",
"------- Class: bus AP: 0.2403 -------\n",
"------- Class: car AP: 0.6061 -------\n",
"------- Class: cat AP: 0.3440 -------\n",
"------- Class: chair AP: 0.4232 -------\n",
"------- Class: cow AP: 0.2019 -------\n",
"------- Class: diningtable AP: 0.4184 -------\n",
"------- Class: dog AP: 0.2904 -------\n",
"------- Class: horse AP: 0.5515 -------\n",
"------- Class: motorbike AP: 0.5291 -------\n",
"------- Class: person AP: 0.7694 -------\n",
"------- Class: pottedplant AP: 0.1822 -------\n",
"------- Class: sheep AP: 0.2053 -------\n",
"------- Class: sofa AP: 0.2436 -------\n",
"------- Class: train AP: 0.5481 -------\n",
"------- Class: tvmonitor AP: 0.2632 -------\n",
"mAP: 0.3853\n",
"Avg loss: 0.1889136206358671\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 40 is 0.3853112696500623\n",
"Starting epoch number 41\n",
"Loss for Training on Epoch 41 is 0.1400957852602005\n",
"Starting epoch number 42\n",
"Loss for Training on Epoch 42 is 0.1302216351032257\n",
"Starting epoch number 43\n",
"Loss for Training on Epoch 43 is 0.13306787610054016\n",
"Starting epoch number 44\n",
"Loss for Training on Epoch 44 is 0.13404273986816406\n",
"Starting epoch number 45\n",
"Loss for Training on Epoch 45 is 0.12778177857398987\n",
"------- Class: aeroplane AP: 0.6184 -------\n",
"------- Class: bicycle AP: 0.4657 -------\n",
"------- Class: bird AP: 0.3285 -------\n",
"------- Class: boat AP: 0.3630 -------\n",
"------- Class: bottle AP: 0.1900 -------\n",
"------- Class: bus AP: 0.2338 -------\n",
"------- Class: car AP: 0.6241 -------\n",
"------- Class: cat AP: 0.3551 -------\n",
"------- Class: chair AP: 0.4284 -------\n",
"------- Class: cow AP: 0.2170 -------\n",
"------- Class: diningtable AP: 0.4105 -------\n",
"------- Class: dog AP: 0.2787 -------\n",
"------- Class: horse AP: 0.5110 -------\n",
"------- Class: motorbike AP: 0.5163 -------\n",
"------- Class: person AP: 0.7713 -------\n",
"------- Class: pottedplant AP: 0.1633 -------\n",
"------- Class: sheep AP: 0.1915 -------\n",
"------- Class: sofa AP: 0.2591 -------\n",
"------- Class: train AP: 0.6189 -------\n",
"------- Class: tvmonitor AP: 0.2731 -------\n",
"mAP: 0.3909\n",
"Avg loss: 0.1838364016264677\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 45 is 0.3908794211060601\n",
"Starting epoch number 46\n",
"Loss for Training on Epoch 46 is 0.12238850444555283\n",
"Starting epoch number 47\n",
"Loss for Training on Epoch 47 is 0.1206757202744484\n",
"Starting epoch number 48\n",
"Loss for Training on Epoch 48 is 0.12219202518463135\n",
"Starting epoch number 49\n",
"Loss for Training on Epoch 49 is 0.12049726396799088\n",
"Starting epoch number 50\n",
"Loss for Training on Epoch 50 is 0.12036202847957611\n",
"------- Class: aeroplane AP: 0.6360 -------\n",
"------- Class: bicycle AP: 0.4731 -------\n",
"------- Class: bird AP: 0.3395 -------\n",
"------- Class: boat AP: 0.3744 -------\n",
"------- Class: bottle AP: 0.2077 -------\n",
"------- Class: bus AP: 0.3183 -------\n",
"------- Class: car AP: 0.6134 -------\n",
"------- Class: cat AP: 0.3800 -------\n",
"------- Class: chair AP: 0.4364 -------\n",
"------- Class: cow AP: 0.2128 -------\n",
"------- Class: diningtable AP: 0.4355 -------\n",
"------- Class: dog AP: 0.2853 -------\n",
"------- Class: horse AP: 0.5255 -------\n",
"------- Class: motorbike AP: 0.5186 -------\n",
"------- Class: person AP: 0.7916 -------\n",
"------- Class: pottedplant AP: 0.1698 -------\n",
"------- Class: sheep AP: 0.2146 -------\n",
"------- Class: sofa AP: 0.2966 -------\n",
"------- Class: train AP: 0.6332 -------\n",
"------- Class: tvmonitor AP: 0.2891 -------\n",
"mAP: 0.4076\n",
"Avg loss: 0.1809161150828004\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 50 is 0.40757003383421775\n",
"Starting epoch number 51\n",
"Loss for Training on Epoch 51 is 0.11942806094884872\n",
"Starting epoch number 52\n",
"Loss for Training on Epoch 52 is 0.11680026352405548\n",
"Starting epoch number 53\n",
"Loss for Training on Epoch 53 is 0.12001264095306396\n",
"Starting epoch number 54\n",
"Loss for Training on Epoch 54 is 0.11999066919088364\n",
"Starting epoch number 55\n",
"Loss for Training on Epoch 55 is 0.1144702285528183\n",
"------- Class: aeroplane AP: 0.6527 -------\n",
"------- Class: bicycle AP: 0.4721 -------\n",
"------- Class: bird AP: 0.3809 -------\n",
"------- Class: boat AP: 0.3692 -------\n",
"------- Class: bottle AP: 0.1757 -------\n",
"------- Class: bus AP: 0.2506 -------\n",
"------- Class: car AP: 0.6335 -------\n",
"------- Class: cat AP: 0.3820 -------\n",
"------- Class: chair AP: 0.4432 -------\n",
"------- Class: cow AP: 0.2186 -------\n",
"------- Class: diningtable AP: 0.3530 -------\n",
"------- Class: dog AP: 0.3070 -------\n",
"------- Class: horse AP: 0.5898 -------\n",
"------- Class: motorbike AP: 0.5679 -------\n",
"------- Class: person AP: 0.7870 -------\n",
"------- Class: pottedplant AP: 0.1699 -------\n",
"------- Class: sheep AP: 0.2169 -------\n",
"------- Class: sofa AP: 0.2912 -------\n",
"------- Class: train AP: 0.6380 -------\n",
"------- Class: tvmonitor AP: 0.2623 -------\n",
"mAP: 0.4081\n",
"Avg loss: 0.18189681880176067\n",
"Evaluating classifier\n",
"Mean Precision Score for Testing on Epoch 55 is 0.40807357723365695\n"
]
}
],
"source": [
"###############################\n",
"#Delete this\n",
"################################\n",
"#classifier = SimpleClassifier().to(device)\n",
"classifier_lol = Classifier().to(device)\n",
"# You can can use this function to reload a network you have already saved previously\n",
"#classifier.load_state_dict(torch.load('voc_classifier.pth'))\n",
"print(classifier_lol)\n",
"criterion_lol = nn.MultiLabelSoftMarginLoss()\n",
"optimizer_lol = torch.optim.Adam(classifier_lol.parameters(), lr=1e-4,betas=(0.9, 0.999), eps=1e-08, weight_decay=0.001, amsgrad=False)\n",
"num_epochs = 55\n",
"test_frequency = 5\n",
"from torchsummary import summary\n",
"summary(classifier_lol, input_size=(3, 227, 227))\n",
"\n",
"classifier_lol, train_losses_lol, val_losses_lol, train_mAPs_lol, val_mAPs_lol = train(classifier_lol, num_epochs, train_loader, val_loader, criterion_lol, optimizer_lol, test_frequency)\n"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3yV9fn/8deVnZAEQoAwQgbIHoKEJUipExdYQQTBgQpW62xta9e3VuuvVmtrrVpFRURBRFCLC0GLWJSVCDLCHoEwkjAChJB9/f64TyDAScg4Jyfjej4eeSTnHudcNyTnfT73574/H1FVjDHGmLP5+boAY4wxdZMFhDHGGLcsIIwxxrhlAWGMMcYtCwhjjDFuWUAYY4xxywLCmDpGRIaLSLqv6zDGAsI0aiKyS0Qu98Hr3iEixSKSIyLHRGSNiFxXjeeZLiJ/9kaNxlhAGOM7y1Q1HGgGvAHMEZHmPq7JmFMsIIwph4hMFpFtInJYROaLSFvXchGRf4hIpogcFZG1ItLTte4aEUkVkeMisldEHj3f66hqCTANCAU6uKmjm4h8LSLZIrJBREa6lk8BJgC/crVEPvbg4RtjAWGMOyJyKfAXYCzQBkgDZrtWXwkMAzrjfPq/GTjkWvcGcI+qRgA9gf9W4rUCgLuBHGDrWesCgY+BhUAr4AFgpoh0UdWpwEzgGVUNV9Xrq33AxrhhAWGMexOAaar6varmA78BBotIAlAIRABdAVHVjaq637VfIdBdRCJV9Yiqfl/BawwSkWzgADAe+ImqHj17GyAceFpVC1T1v8Anru2N8SoLCGPca4vTagBAVXNwWgntXG/SLwIvARkiMlVEIl2bjgauAdJEZImIDK7gNZarajNVbaGqg1T1y3Lq2OM6DVUqDWhX/UMzpnIsIIxxbx8QX/pARJoA0cBeAFV9QVX7AT1wTjX90rV8laqOwjkd9BEwxwN1tBeRsn+rcaV1ADYcs/EaCwhjIFBEQsp8BQCzgEki0kdEgoH/B6xQ1V0i0l9EBrr6B04AeUCxiASJyAQRaaqqhcAxoLiGta1wvcavRCRQRIYD13O6PyQDNx3bxniCBYQx8BlwsszX46r6FfAHYB6wH+gIjHNtHwm8BhzBOd1zCPiba92twC4ROQb8FJhYk8JUtQAYCVwNHAReBm5T1U2uTd7A6fPIFpGPavJaxpxNbMIgY4wx7lgLwhhjjFsWEMYYY9yygDDGGOOWBYQxxhi3AnxdgKe0aNFCExISfF2GMcbUKykpKQdVtaW7dQ0mIBISEkhOTvZ1GcYYU6+ISFp56+wUkzHGGLcsIIwxxrjl1YAQkREistk1pv5jbtb/3DV2/loR+UpEyo59EyciC0Vko2ubBG/Waowx5kxe64MQEX+c0S6vANKBVSIyX1VTy2y2GkhS1VwRuRd4BmdsfYAZwFOqukhEwoGyo1kaY4xHFBYWkp6eTl5enq9L8aqQkBBiY2MJDAys9D7e7KQeAGxT1R0AIjIbGAWcCghVXVxm++W4xq0Rke5AgKoucm2X48U6jTGNWHp6OhERESQkJCAivi7HK1SVQ4cOkZ6eTmJiYqX38+YppnbAnjKP06l4DPu7gM9dP3cGskXkAxFZLSLPulokZxCRKSKSLCLJWVlZHivcGNN45OXlER0d3WDDAUBEiI6OrnIryZsB4e5f2+3IgCIyEUgCnnUtCgAuAR4F+uMMZ3zHOU+mOlVVk1Q1qWVLt5fxGmPMeTXkcChVnWP0ZkCkA+3LPI7FmfzkDCJyOfA7YKRrasfSfVer6g5VLcKZeOUibxR59GQhz3+5hR/2ZHvj6Y0xpt7yZkCsAjqJSKKIBOGMpT+/7AYi0hd4FSccMs/aN0pESpsFl1Km78LTnv9yK6t2HfbW0xtjTLmys7N5+eWXq7zfNddcQ3a2dz/Yei0gXJ/87we+ADYCc1R1g4g8ISIjXZs9izMh+/siskZE5rv2LcY5vfSViKzDOV31mjfqjAwJICTQjwNHG/YVDMaYuqm8gCgurngyws8++4xmzZp5qyzAy0NtqOpnOLN1lV32f2V+vryCfRcBvb1XnUNEaB0ZQsbx/PNvbIwxHvbYY4+xfft2+vTpQ2BgIOHh4bRp04Y1a9aQmprKDTfcwJ49e8jLy+Ohhx5iypQpwOnhhXJycrj66qsZOnQo3333He3ateM///kPoaGhNa6twYzFVBOtIkPIOGYtCGMauz99vIHUfcc8+pzd20byx+t7lLv+6aefZv369axZs4avv/6aa6+9lvXr15+6HHXatGk0b96ckydP0r9/f0aPHk10dPQZz7F161beffddXnvtNcaOHcu8efOYOLFGs90CNtQGADEWEMaYOmLAgAFn3KvwwgsvcOGFFzJo0CD27NnD1q1bz9knMTGRPn36ANCvXz927drlkVqsBQG0jgxm0bE8VLVRXO5mjHGvok/6taVJkyanfv7666/58ssvWbZsGWFhYQwfPtztvQzBwcGnfvb39+fkyZMeqcVaEDgtiLzCEo6dLPJ1KcaYRiYiIoLjx4+7XXf06FGioqIICwtj06ZNLF++vFZrsxYETkAAZBzPo2lY5ccpMcaYmoqOjmbIkCH07NmT0NBQYmJiTq0bMWIEr7zyCr1796ZLly4MGjSoVmuzgKBMQBzLo3NMhI+rMcY0NrNmzXK7PDg4mM8//9ztutJ+hhYtWrB+/fpTyx999FGP1WWnmICYSOf8nd0LYYwxp1lAcLoFkWn3QhhjzCkWEEBIoD9NQwPtUldjjCnDAsKldWSInWIyxpgyLCBcWkUG23AbxhhThgWES0xkCBnWgjDGmFMsIFxaR4aQlZNPcYnbOY2MMaZOCA8Pr7XXsoBwiYkMprhEOXTCTjMZYwzYjXKnnLpZ7mg+rSJCfFyNMaax+PWvf018fDz33XcfAI8//jgiwjfffMORI0coLCzkz3/+M6NGjar12iwgXMreTd2Lpj6uxhjjE58/BgfWefY5W/eCq58ud/W4ceN4+OGHTwXEnDlzWLBgAY888giRkZEcPHiQQYMGMXLkyFofTNQCwqU0IA7YvRDGmFrUt29fMjMz2bdvH1lZWURFRdGmTRseeeQRvvnmG/z8/Ni7dy8ZGRm0bt26VmuzgHBpER6En0CmBYQxjVcFn/S9acyYMcydO5cDBw4wbtw4Zs6cSVZWFikpKQQGBpKQkOB2mG9vs4BwCfD3o0V4MBnHrJPaGFO7xo0bx+TJkzl48CBLlixhzpw5tGrVisDAQBYvXkxaWppP6vLqVUwiMkJENovINhF5zM36n4tIqoisFZGvRCT+rPWRIrJXRF70Zp2lWjcNsVNMxpha16NHD44fP067du1o06YNEyZMIDk5maSkJGbOnEnXrl19UpfXWhAi4g+8BFwBpAOrRGS+qqaW2Ww1kKSquSJyL/AMcHOZ9U8CS7xV49laRYSQfiS3tl7OGGNOWbfudOd4ixYtWLZsmdvtcnJyaqskr7YgBgDbVHWHqhYAs4EzrtNS1cWqWvqOvByILV0nIv2AGGChF2s8Q0xksA3YZ4wxLt4MiHbAnjKP013LynMX8DmAiPgBzwG/rOgFRGSKiCSLSHJWVlYNy3Xupj6SW0h+UXGNn8sYY+o7bwaEuwt23Y5jISITgSTgWdei+4DPVHWPu+1PPZnqVFVNUtWkli1b1qhYKDMvhHVUG9OoqDb8IXaqc4zevIopHWhf5nEssO/sjUTkcuB3wI9UtfSdeTBwiYjcB4QDQSKSo6rndHR7UkzT0zfLtW8e5s2XMsbUESEhIRw6dIjo6OhavxGttqgqhw4dIiSkaqNEeDMgVgGdRCQR2AuMA24pu4GI9AVeBUaoambpclWdUGabO3A6sr0aDnB66lG71NWYxiM2Npb09HQ8cZq6LgsJCSE2Nvb8G5bhtYBQ1SIRuR/4AvAHpqnqBhF5AkhW1fk4p5TCgfddyb1bVUd6q6bziYmwu6mNaWwCAwNJTEz0dRl1kldvlFPVz4DPzlr2f2V+vrwSzzEdmO7p2txpFhZIUICf3U1tjDHYcN9nEBG71NUYY1wsIM7SOtLupjbGGLCAOEeryBC7zNUYY7CAOEdMhNOCaAzXRRtjTEUsIM7SumkwuQXF5OQX+boUY4zxKQuIs5yeWc5OMxljGjcLiLOUnXrUGGMaMwuIs1hAGGOMwwLiLKXDbdilrsaYxs4C4ixhQQFEhATYpa7GmEbPAgJg+2IoLjz1MCYyxE4xGWMaPQuIrC3wzo3w7jjId6bys7upjTHGAgJadobrnndaEW9dBzlZtIoMtlNMxphGzwICoN/tMG4WZG6CN66gS9BBMo7lUVJid1MbYxovC4hSXUbA7R9D3lFuS51MV93O4dwCX1dljDE+YwFRVvv+cNdCCAjhvaAnydnwha8rMsYYn7GAOFuLTmwb+RFp2pq4BZPgh9m+rsgYY3zCAsKN6NZxjC34A5nN+8GH98DS58FGdzXGNDJeDQgRGSEim0Vkm4g85mb9z0UkVUTWishXIhLvWt5HRJaJyAbXupu9WefZWkYEc0LCmNPlH9BzDHz5R1jwGJSU1GYZxhjjU16bk1pE/IGXgCuAdGCViMxX1dQym60GklQ1V0TuBZ4BbgZygdtUdauItAVSROQLVc32Vr1lBfr7Ed0kmP05xXDjaxDRGpa9CMcPwE9ehcCQ2ijDGGN8ypstiAHANlXdoaoFwGxgVNkNVHWxqua6Hi4HYl3Lt6jqVtfP+4BMoKUXaz2HMzd1Pvj5wVVPwZV/htSP4J3RcLJWcsoYY3zKmwHRDthT5nG6a1l57gI+P3uhiAwAgoDtHq3uPFpHhnDgaJm7qS9+AG58HfasgDevgWP7arMcY4ypdd4MCHGzzG1Pr4hMBJKAZ89a3gZ4G5ikqud0AIjIFBFJFpHkrKwsD5R8WqvIEDKPnzXcRu+bYOJcyN4Nb1wJWZs9+prGGFOXeDMg0oH2ZR7HAud87BaRy4HfASNVNb/M8kjgU+D3qrrc3Quo6lRVTVLVpJYtPXsGKiYymIM5BRzPKzxzRYfhMOlTKMp3QmL3Co++rjHG1BXeDIhVQCcRSRSRIGAcML/sBiLSF3gVJxwyyywPAj4EZqjq+16ssVzDu7RCBJ5buOXclW0uhLsXQVg0zBgJmz6t/QKNMcbLvBYQqloE3A98AWwE5qjqBhF5QkRGujZ7FggH3heRNSJSGiBjgWHAHa7la0Skj7dqdadP+2bcNiiet5btIiXtyLkbRCXAXYsgpie8NxGSp9VmecYY43WiDeQGsKSkJE1OTvboc+bkF3Hl35cQFhzApw8OJTjA/9yNCk7A+5Ng6xfwo1/D8N+AuOt+McaYukdEUlQ1yd06u5O6AuHBATx1Yy+2Zebw8uJyLqIKauKMBNt3Iiz5K3z8IBQX1W6hxhjjBRYQ5/HjLq24oU9bXv56G1syjrvfyD8ARr4Iw34F38+A9yZAQa77bY0xpp6wgKiEP1zXnfDgAH41dy3F5c0RIQKX/g6u/TtsXeh0Xp84VLuFGmOMB1lAVEJ0eDB/vL4Ha/ZkM2PZroo37n8XjH0bDqyDaVfBkbTaKNEYYzzOAqKSRvVpy/AuLXn2i82kHznP6aNu18Ft/4ETWU5IHN5RO0UaY4wHWUBUkojw5xt6AvDEx6nn2RqIGwSTPofiAnhrFBxN93KFxhjjWRYQVRAbFcY9wzqyMDWD9XuPnn+HmO4w8QPIOwpvjYTjGd4v0hhjPMQCooruGJJAZEgAL3y1tXI7tO0DE953hgp/+wbIPezdAo0xxkMsIKqoaWggdw5NZGFqBhv2VaIVARA3EMa/C4e2w9s/cVoUxhhTx1lAVMOkIYlEVKUVAdDhR3DzO5CxAWaOde7ANsaYOswCohqahgZy55BEvtiQQeq+Y5XfsfOVMPp1SF8Js2+Bwrzz72OMMT5iAVFNdw6tRisCoMcNMOpl2PE1vH8HFBeebw9jjPEJC4hqahoayKQhiSzYcICN+6vQigDoMx6ufQ62fA4fTIGSYu8UaYwxNWABUQN3DUkkIrgarQiA/nc781xv+ADmPwAl50yYZ4wxPmUBUQNNwwKZNCSBz9cfYNOBKrYiwJnnevhvYM1MWPBraCBDrxtjGgYLiBq6c2gNWhHgzCFx8QOwcip8+biFhDGmzgjwdQH1XbOwIO4YksC//ruNh2evJi66Ce2jQolrHkZcdBgxESH4+VUwgZAIXPGkc9nrt89DUDj86Je1dwDGGFMOCwgPuPuSDmzJOM6qXUeY/8M+yo4I3q5ZKNPu6E+X1hHlP4EIXPMcFJ6ExX+GoDAY/DPvF26MMRWwgPCApqGBvHqrM2NfQVEJ+7JPsvtwLmmHTvDi4m3c9Mp3vH57fwYkNi//Sfz8nEmHCnPhi99CYBgkTaqlIzDGmHN5tQ9CREaIyGYR2SYij7lZ/3MRSRWRtSLylYjEl1l3u4hsdX3d7s06PSkowI+EFk0Y1rkltw5OYN69F9MiIphb31jBFxsOVLyzfwDc+Dp0uhI+eQR+eK92ijbGGDe8FhAi4g+8BFwNdAfGi0j3szZbDSSpam9gLvCMa9/mwB+BgcAA4I8iEuWtWr0pNiqMuT+9mG5tIrn3nRRmrdhd8Q4BQTB2BiQMhY/uhdT5tVOoMcacxZstiAHANlXdoaoFwGxgVNkNVHWxqpbOvrMciHX9fBWwSFUPq+oRYBEwwou1elXzJkHMmjyQH3VuyW8/XMc/v9yKVnS1UmAojJ8N7frB3Dth65e1V6wxxrh4MyDaAXvKPE53LSvPXcDnVdlXRKaISLKIJGdlZdWwXO8KCwpg6m1JjL4oln98uYXnFm6peIfgcGeY8Fbd4L0JsPN/tVOoMca4eDMg3F3b6fZjs4hMBJKAZ6uyr6pOVdUkVU1q2bJltQutLYH+fvztpt6MTYrlxcXb+GbLeUIttBnc+hFEJcCsm2HPqlqp0xhjwLsBkQ60L/M4Fth39kYicjnwO2CkquZXZd/6SET408iedGoVzs/n/MDBnPyKd2gS7YREeCuYORr2r62dQo0xjZ43A2IV0ElEEkUkCBgHnNHjKiJ9gVdxwiGzzKovgCtFJMrVOX2la1mDEBrkz79u6cuxvEJ+MecHSkrOc/d0ZBu4fT4ERTiz0mVtrp1CjTGNmtcCQlWLgPtx3tg3AnNUdYOIPCEiI12bPQuEA++LyBoRme/a9zDwJE7IrAKecC1rMLq2juQP13ZjyZYspn278/w7NItzQkL8nfmtD+/wfpHGmEZNKryaph5JSkrS5ORkX5dRJarKPW+nsHhzJh/cO4ResU3Pv1NGKky/xmlN3Pk5NI09/z7GGFMOEUlR1SR362ywPh8SEZ4Z05sW4cE88O735OQXnX+nmO5w64eQlw0zRkFO5vn3McaYarCA8LFmYUH84+Y+7D6cyx//s6FyO7Xt61wCe2yfExK5DersmzGmjrCAqAMGdYjm/ks7Me/7dF746jw30ZWKGwTj34VD2+GdGyGvGvNRGGNMBSwg6oiHLuvEjRe14++LtvDXBZsrFxIdhsPYt+DAOpg11hky3BhjPMQCoo7w9xP+NuZCJg6K45Ul23l8/obzX/4K0OVquHEq7FkBsydAYZ73izXGNAqVCggR6Sgiwa6fh4vIgyLSzLulNT5+fsKTo3oyZVgH3lqWxq/nraW4MiHRc7QzVPiOxfD+7XA8w/vFGmMavMq2IOYBxSJyAfAGkAjM8lpVjZiI8Juru/Lw5Z14PyWdh2avprC45Pw79p0A1/wNtnwBz/eCjx9y+ieMMaaaKjthUImqFonIT4DnVfVfIrLam4U1ZiLCw5d3pklQAE99tpGCohJennARAf7nyfMBk6HjpfDdv2DNLEh5C7pdD0Mehth+tVO8MabBqGwLolBExgO3A5+4lgV6pyRTavKwDvxpZA8Wpmbw+McbKtdxHd0Rrn8eHlkPl/wcdi6B1y+F6dfB1kXQQG6MNPVU3lHY8BFkbrLfxXqgsi2IScBPgadUdaeIJALveK8sU+r2ixPYfzSPV5ZsJzYqjJ/+qGPldgxvBZf9Hwx9BL6fActegpljoFUPGPIQ9LwR/C3jTS05mQ3L/+185R91ljVpCfEXQ8IlED8EWnZ1pt41dUaVh9pwDZ7XXlXr1LCi9XGojcoqKVEeem8NH/+wjxfG92XkhW2r/iRFBbB+Hnz7T8jaCE3bw+CfQd9bnbknjPGGk0dcwfCKEwxdr4MBUyB7N6R9C7uWwlHX1C9h0acDI2EotOxmgVELKhpqo1IBISJfAyNxWhxrgCxgiar+3IN11khDDgiA/KJibn1jJWt2Z/P2XQMY2CG6ek9UUgLbFjlBkfYthDRz/mAHTIHwuj+nRoOjCkfTITMVMtY7Y235B0KfW5w3SnE3NUo9kHvYabWueBUKjjt9YcN+BW16n7vtkTQnKNK+hV3/c8IDIDTKaVkkDHW+WvWwwPACTwTEalXtKyJ347Qe/igia11zSdcJDT0gALJzC7jx399xKKeAefdezAWtavjJf88q+PZ52PQpBARDnwlw8f3QvINnCjZnyjsGmRshcwNklH6lnj7lAtA0znmcdxSiL4B+d8CFtzjzgtQHJw7Bshdh5VQoyIHuo5xgaN2z8s+RvRt2uVoXaUvhyC5neUgzV2C4QiOmJ/j5e+UwGhNPBMQ6nDkZ3gJ+p6qrLCB8Y8/hXH7y8reEBPrzwvi+dG8TSUhgDf9IDm6F716AH2ZDSZHzRz3kIWfMJ1N1xUVwePvpEChtHZR+MgYIjoRW3Z3BF2N6OG92rbpBSFMoPOl05Ka86dwA6R/k/J/0m+ScgqmLrYoTB52r51a+BoW50OMGJxhiutf8uY+muwLjf04ro3So++CmrlNSrsBo3dsCoxo8ERA3AX8AvlXVe0WkA/Csqo72bKnV11gCAmBtejbjpi4nt6CYAD+hU0wEvdpF0rNdU/q2j6Jnu0ikOm8ixw8454uTp0H+MUj8kRMUHS+tm29KdUFO5ulTQxkbnNZB5iYods0UKP7QopMrDHqc/mravnL/phmpTlD88J7TsmjR2QmKC8dBWHPvHltl5GQ5Hy5WveEEQ88bnWBo1dV7r3l07+n+i11LnTAGJ3TjBpcJjAvBv7LX4TReNQ6I+qAxBQRA5vE8UnYdYf2+o6zbe4z1e49y+EQBAL1jmzJlWAdG9Gh9/nsn3Mk7BinTYfnLcHw/tO7l3EvR/YbG+wdXeBKyNpU5NeT6yj14epvw1qdbBK1cQdCiMwSG1Pz1C3Jhw4dOWKSvAv9g6PET5xRU3KDaD/CcTKcfK3kaFOVBzzEw7FFo2aV26wA4tv/MwDi01VkeFOH82yQMcf4/ojs6E2/Z1Xtn8EQLIhb4FzAEUGAp8JCqpnuy0JpobAFxNlVl/9E8/rspkzeW7mTnwRO0bx7K5Es6cFO/9oQGVaPpXZQP69533ggObnH+uAY/4Ny1HdTE8wdRlxQXwc6vYf2Hzmmew9tBXXe0B4Q6p4NiurtODblCoUmL2qntwDonwNfOcVp6Lbu5+ipudjp2vel4xulgKM6HXmOdYGjRybuvWxXHM5y+i9J+jINlpugVf+f3OLqj09fW3PW9EYeHJwJiEc7QGm+7Fk0EJqjqFR6rsoYae0CUVVyiLErNYOo32/l+dzZRYYE8fHlnbr84oXpPWFICWxY4bwx7lkNo89NXPtWXztPKKClxjm/dXEj9CHIPOee5Ey9xgqA0EKIS6sa57oITzqXLyW/Cvu8hIAR63AhJkyC2v2dbFcf2O///KW9CcSH0vtkJhuhK3pfjSycOOa2KQ9ud/ovDru+HdjhXWJUqDY/SwGgk4eGJgFijqn3Ot8zNfiOAfwL+wOuq+vRZ64cBzwO9gXGqOrfMumeAa3Hu9l6E02Ipt1gLCPeSdx3mn19t5X9bD/Lba7oyZVgN/6B3L3feKDZ/5nyS7jvROdUR2x8CgjxTdG1Shf1rnFDY8CEc2+scV5erodcYuOBy5wqvum7/D05QrHvfuXqoVQ8nKHqPdTq+q+vYPlj6vNNiKSmCC8c7d+jXh2A4H1Wnc/3w9mqER5nWR1R8zcND1WmxF5xw/v9OfS/9+QTkHz/989nrohLg6r9W66U9ERBfAtOBd12LxgOTVPWyCvbxB7YAVwDpwCpgvKqmltkmAYgEHgXmlwaEiFwMPAsMc226FPiNqn5d3utZQJSvqLiEh95bw6dr9/PEqB7cNjih5k+atRm+fQHWvgclhRAY5lyC2GG48xXTo253bGdtdkJh/TznTcEv0AmDXmOg84j6e/Ng/nHnuFLedEIjINQZ7TdpErTrV/n/k6PpsPQfzl34WuIKhl9A80Tv1l9XlA2PwztcAVJReLR3AqM0PPyD3LzZn4D8nHOXlX4vqcSUw84LQlC48zsa1MT5at0bRr1YrUP1REDEAS8Cg3H6IL4DHlTV3RXsMxh4XFWvcj3+DYCq/sXNttOBT8oExGDX6w0FBPgGuFVVN5b3ehYQFSssLuG+md+zKDWDZ0b3Zmz/9p554pPZznneHV87X6UdhE1aOldBdRjufDXz0OvVxJE0JxDWz3OuPBI/52a0XmOcO3zrwlVBnrRvtatVMRcKT0BML0i6w+k3CIl0v0/2Hlj6d1j9jhMMfSY4wRAVX6ul12luw8PV+jg7PMB5My99Iw8KP/Nx8FmPgyLOvy4w1KMfvrxyFZOIPKyqz1ewfgwwQlXvdj2+FRioqve72XY6ZQLCtexvwN04AfGiqv6uonosIM4vv6iYyTNS+N/WLJ6/uQ+j+rTz/IscTYcdS04HxolMZ3nzjqfDIvES73emljqe4Zw6Wj/XufoHIHaAEwrdb4CImNqpw5fyjjmnnlLedDq4A5tAr9HO5bLtLnK2yd4N/3sOVs90Hl90qzOOV7M439VdH6k6fVclxa4387A6f/e3twJit6qW+9vjunfiqrMCYoCqPuBm2+mc2YK4AKfv4mbXJouAX6vqN2ftNwWYAhAXF9cvLS2tWsfSmJwsKGbS9JWs2nWEl27py1U9WpOVk8+2zBy2Z+awLTOH4EB/fnFlZ4IDatgRq+rcOVwaFruWOp9kxc+5Ca/DcOcrdoBnLgUtlXsYNhE3iyIAAB0xSURBVH7shMKupc4n4Zhezptijxsb76dhVdj7PaRMg/UfOPcttLnQGSRv/Tzn/+Wi25xgaBrr62pNLfFWQOxR1XLPG9TwFNMvgRBVfdL1+P+APFV9przXsxZE5Z3IL+LWN1awNv0oYUH+HMs7fe6zSZA/JwqKmTgojj/f0MuzL1xUAHtTXIGxGNKTQYud8+Txg8v0X/Sq+qeu/BzY/LkTCtu+cvpFmnd0Wgo9R/vm+vy6LO+oc5lsynTnTvp+tzv3ujT1QqvS1Gm+akEE4HRSXwbsxemkvkVVN7jZdjpnBsTNwGRgBM4ppgU4ExV9XN7rWUBUzbG8Qv76+SYALmgVfuqrdWQIf/l8E1O/2cHzN/fhhr5efMPIO+bc4FTawshy6iG0OXQo038RleB+/6J8Z46L9XNh8wIoOgmR7Zy7eXuOhjZ96nZHeV2g6rSw6sJlu8Ynqh0QInIcp1P6nFVAqKpWeFutiFyDcxmrPzBNVZ8SkSeAZFWdLyL9gQ+BKCAPOKCqPVxXQL2McxWTAgvON3KsBYTnFBWXcMtrK1i39yj/uX8InWMiaueFj+13JjgqDYzj+53lUQmnwyJ+iHMeff082PiJM/xEWAtn7J+eY6D9wDp/zteYusSG2jBVlnksj2teWEpkaADz7x9KeHAtD7Gh6ty9XRoWO/935tUhwZHOENI9b4TE4Y13CBBjasgCwlTLsu2HmPD6cq7u1YYXx/et3gCAnlJc5NwtnPadc635BVd4tmPbmEaqooCwj12mXIM7RvPoVV14ZsFmkuKjmDTEhzdJ+QdA+wHOlzGmVtjJWlOhnw7ryOXdWvHUpxtJSTvi63KMMbXIAsJUyM9PeO6mPrRtFspdb61i/d6j59/JGNMgWECY82oaFsg7dw2kSVAAt7y2nB/2ZPu6JGNMLbCAMJUSFx3Ge/cMomlYIBNfX2Gnm4xpBCwgTKXFRoUx557BtIgI5rY3VrBy5+Far2HnwRPkFxXX+usa0xhZQJgqadM0lNlTBtG6aQi3T1vJd9sPnn8nD/lu20Eue+5rfvPBulp7TWMaMwsIU2UxkSHMnjKY9s1DmfTmKu6f9T3Tlu5k9e4jXvt0vzf7JPe/uxo/ET5cvZctGcfPv5MxpkbsPghTLS0jgnl38iD+/OlGVuw4xCdrnWExgvz96NEukgGJzbmmZxt6xzat8Q12eYXF/PTtFAqLSnjvnsHcPm0lf1+4hVdu7eeJQzHGlMMCwlRbdHgw/7jZmXX2wNE81uw5wve7s1m9+wjTlu7k1SU7iI0K5drebbiuV1t6tousclioKr/7cD3r9h7l9duS6Bcfxd2XJPL8l1tZm55N79hm3jg0Yww21IbxkqO5hSxMPcCn6/azdOtBikqUuOZh/GpEF67r3bbSzzNj2S7+7z8beOiyTjxyRWcAjucVMuyZxfSKbcaMO+3OamNqwobaMLWuaVggNyW156ak9mTnFrBwQwZvL0/jwXdXE+AnjOjZ5rzPsXLnYZ74OJXLurbiocs6nVoeERLIvcM78v8+28TyHYcY1CHam4diTKNlndTG65qFBTG2f3veu2cQfdo344F3V/PNlqwK9zlwNI/7Zn5P++Zh/P3mPvj5nXlq6rbBCcREBvO3LzbTUFrBxtQ1FhCm1oQFBfDmpAFc0CqCKW8ns2rXufdRqCqfrN3HyBeXkltQxKu39qNpaOA524UE+vPApZ1ITjvC15srDhtjTPVYQJha1TQ0kLfvGkDbpqHc+eaZYzulHTrBHW+u4v5Zq2kVGcx7UwZXOFnR2KT2xDUP428LN1NSYq0IYzzNAsLUuhbhwbxz90AiQwO5bdpKUvcd48X/buXKf3xDStoRHr++O//52VB6xTat8HmCAvx4+PJObNh3jM/XH6il6o1pPOwqJuMzOw+e4KZXlnEwJx+Aa3u14f+u705MZOUnAiouUUY8/w0nC4u5onsMJwuKOVlYTG5BMXmFxbRpGsLoi2IZkNjctxMeGVNH2Yxyps7adOAYf/tiMxMGxfPjLq2q9RyLN2Vy78wUAv38CAnyJyzIn9BAf0KD/NmakUNOfhHx0WGMuSiW0f1iadss1MNHYUz95bOAEJERwD8Bf+B1VX36rPXDgOeB3sA4VZ1bZl0c8DrQHlDgGlXdVd5rWUAYd3ILiliw/gDvJ6ezbMchRGDoBS344/U9uKBVuK/LM8bnfBIQIuIPbAGuANKBVcB4VU0ts00CEAk8Csw/KyC+Bp5S1UUiEg6UqGpuea9nAWHOZ/ehXOZ+n847y9MI8vdj7r2DiY0K83VZxvhURQHhzU7qAcA2Vd2hqgXAbGBU2Q1UdZeqrgVKyi4Xke5AgKoucm2XU1E4GFMZcdFh/PyKzsyaPJDcgiJue2Plqf4PY8y5vBkQ7YA9ZR6nu5ZVRmcgW0Q+EJHVIvKsq0VyBhGZIiLJIpKclWXXwpvK6do6kjcn9Wff0ZPc8eZKjucV+rokY+okbwaEu0tGKns+KwC4BOfUU3+gA3DHOU+mOlVVk1Q1qWXLltWt0zRC/eKb8++J/di0/ziTZySTV2iTEBlzNm8GRDpOB3OpWGBfFfZd7To9VQR8BFzk4fpMI/fjLq14buyFLN9xmAffXU1Rccn5dzKmEfFmQKwCOolIoogEAeOA+VXYN0pESpsFlwKpFWxvTLWM6tOOx6/vzsLUDH49bx2FFhLGnOK1gHB98r8f+ALYCMxR1Q0i8oSIjAQQkf4ikg7cBLwqIhtc+xbjnF76SkTW4Zyues1btZrG7Y4hiTxyeWfmfZ/OTa8sY/chux7CGLAb5Yw55ZO1+/jNB+tQhad+0pNRfc69piInv4h5KeksWH+AR67ozIDE5j6o1BjPsTupjamkPYdzeWj2ar7fnc2YfrH8aWQPmgQHsPPgCd76bhdzU9LJyS8iLMgfAWbcNZB+8VG+LtuYarOAMKYKiopLeOGrrby4eBvx0U2Ijw7j681ZBPoL1/Zqw+0XJ9CuWSg3T13OweP5vHP3QC5sb1OfmvrJAsKYali+4xA/f28NhSXKhIFx3DIwjlYRpwcS3H/0JGNfXcbR3EJmTR5Ez3YVjz5rTF1kAWFMNakqqpwzo12pPYdzGTd1ObkFRbw7ZRBdW0fWcoXG1Iyvhtowpt4TkXLDAaB98zBmTR5IUIAfE19fweYDxykoKqGwuISi4hJKStSmRDX1lrUgjPGA7Vk53Pzq8nLHdrrnRx34zdXdarkqY86vohZEQG0XY0xD1LFlOPPuHcyn6/ZTUqKUKKiCoqTuO8arS3bQP745l3eP8XWpxlSaBYQxHhIf3YT7hl9wzvL8omJ+8tJ3/HreWha0H0bLiGAfVGdM1VkfhDFeFhzgzz/H9SEnv4hfzf3BY30SqkrGsTyPPJcx7lhAGFMLOsVE8NtrurF4cxZvL0/zyHO+t2oPA//fVzyzYBMlJQ2jL9HULRYQxtSS2wbHM7xLS576dCNbM47X6LmKS5RXlmwnPDiAl7/ezj3vpJCTX+ShSo1xWEAYU0tEhGfG9KZJcAAPzV5DflH156D4cmMGuw7l8vToXjx+fXf+uymTMf/+jj2HbaBB4zkWEMbUolYRITwzujep+4/x94Vbqv08r/9vB+2ahTKiR2vuGJLI9En92Zd9klEvfcvKnYc9WLFpzCwgjKlll3ePYcLAOKb+bweLN2VWef81e7JZtesIdw1NJMDf+RO+pFNLPvrZEJqFBTLh9eXMS0n3dNmmEbKAMMYHfn9td7q1juTBd1ezLTOnSvu+9r8dRIQEMLZ/+zOWd2gZzof3DWFAYnN+OfcHvtqY4cmSTSNkAWGMD4QG+fPa7UkEBfgxZUYyR3MLK7XfnsO5fL5uP7cMjCM8+NzbmJqGBvLabUn0aNuU+2etZl360fM+p82iZ8pjAWGMj7RrFsort/Zjz5FcHphduTmxp327Ez8R7rg4odxtwoICeOOOJJo3CeLOt1axN/uk2+0Onyhg8oxk+j25yDq3jVsWEMb4UP+E5vz5hp58syWLpz/fVOG2R08WMmfVHq6/sC1tmoZWuG2riBDenNSfvMJiJr25kmN5Z7ZQlm49yIjnv2HJ5iwKikt44hOb8t2cywLCGB+7uX8cd1ycwOtLd/J+8p5yt3t35W5OFBRz9yWJlXrezjERvDqxHzuyTnDfO99TWFxCQVEJf/lsIxPfWEFkaCAf/WwID13WmUWpGdXqMDcNm1cDQkRGiMhmEdkmIo+5WT9MRL4XkSIRGeNmfaSI7BWRF71ZpzG+9vtruzHkgmh+9+F6knede5lqQVEJ07/dxZALounRtvITE118QQueHt2bpdsO8vB7a7jx39/y6jc7mDAwjo/vH0r3tpHcNTSRDi2b8PjHG8grrP69Gabh8VpAiIg/8BJwNdAdGC8i3c/abDdwBzCrnKd5EljirRqNqSsC/P14cfxFtGkWwphXljHi+W944uNUvtqYQU5+EZ+u28eBY3ncfUmHKj/3mH6xPHhZJz5du5/0Iyd59dZ+PPWTXoQG+QMQFODHEyN7knYol6nf7PD0oVWKqlo41UHeHM11ALBNVXcAiMhsYBRw6mSnqu5yrTund05E+gExwALA7VjlxjQkUU2CeG/KYOZ9n8532w8yc0Ua077dib+fEBroT6dW4Qzv3LJaz/3I5Z3oHBNOUnxzWjcNOWf90E4tuLZXG15avI2f9G1H++ZhNT2cSsvJL+KOaSvJPlnIFw8Pw7+CCZpM7fJmQLQDyp5QTQcGVmZHEfEDngNuBS6rYLspwBSAuLi4ahdqTF3RumkIP/vxBfzsxxeQV1jM97uP8N22QySnHWbyJR0Qqd6bp4hwXe+2FW7z++u6sXhzJk98ksprt9XOZ7ITrnBITjsCwOJNmTZnRh3izT4Id7/JlR1y8j7gM1Utv8cOUNWpqpqkqkktW1bvk5UxdVVIoD8Xd2zBo1d1YfaUwVzWzbtvnG2ahvLApZ3cdlgXFpewZEsWf12wiX3lXDZbVbkFRUyavorVe7L557g+xEQGe2ykW+MZ3mxBpANlb/WMBfZVct/BwCUich8QDgSJSI6qntPRbYzxnLuGJvJ+yh4e/3gDAxKbk5J2hE/X7ueL1ANku27m+2zdfubcM5iYyHNPVVXWyYJi7py+iuRdh/nnuL5cf2Fbdh48wfNfbiXt0Anio5t46pBMDXizBbEK6CQiiSISBIwD5ldmR1WdoKpxqpoAPArMsHAwxvvKdlhf9OQibpu2kk/X7Wd455a8dlsSs6cM4uDxfG55bTlZx93Pv30+JwuKueutVazceZh/3NyH6y90Tn2NHxCHv58wc8VuTx6SqQGvtSBUtUhE7ge+APyBaaq6QUSeAJJVdb6I9Ac+BKKA60XkT6raw1s1GWPOb2inFtw3vCP7sk9yTa82DOvckpBA/1Prp985gNveWMmE15fz7uRBRIe7n0J184HjpB/JpaCohILiEvKLSigsLuHTtftZtuMQfx97IaP6tDu1fUxkCFf1iGFO8h5+fkXnM17T+IZ4avpDX0tKStLk5GRfl2FMo/DdtoNMmr6KDi3DeXfyQJqFBQHOREaLUg8wbekuVrq5nwMg0F/4y429GdMv9px1y7YfYvxry/nbTRe6XW88T0RSVNXtVQkWEMaYalmyJYvJbyXTtU0E/57Yj8/X7Wf6d7tIP3KSds1CmTQkgf4JzQkK8HO+/P0IDvCjSXAATdwMNAjO/RBX/OMbmgQH8J+fDanlI2qcKgoIb3ZSG2MasB91bsm/J17ET99JYcjT/wWgf0IUv7+2G5d3izk1V0VViAi3Dornj/M3sDY9m96xzSrcvqCohPX7jrJq52FW7TpC19YRPHpVl2odjzmXBYQxptou6xbD1FuTWLQxg3H925/3Db0yfnJRO/66YBPvLE/jmTHnPt/BnHzeWZ7Gih2HWb3nCHmFzn22LcKD+HJjBv3io/hx11Y1rsPYKSZjTB302w/XMS8lnZW/vZymYYGnli9KzeCxeWs5kltAtzaR9E9ozoDE5iQlRNE0NJDrXlhKTn4RCx8ZRkRIYAWvYEpVdIrJRnM1xtQ5EwfGk19Uwvspzr2yOflFPDZvLZNnJBMTGcKCh4fx6YOX8PjIHlzTqw2tIkIIDvDnmTG9yTiWx1/OM3S6qRw7xWSMqXO6t40kKT6KmSt2c2H7Zvxizg/sOZLLvcM78sjlnQkKcP/Ztm9cFHcOSeT1pTu5vndbBneMruXKGxZrQRhj6qRbB8ez8+AJbnplGSWqzLlnML8e0bXccCj1iyu7EB8dxmMfrOVkgY0QWxMWEMaYOmlEz9Zc2L4Z4/q35/OHLqF/QvNK7Rca5M/TN/Ym7VAuzy3c7OUqGzY7xWSMqZOCA/yrfS/E4I7RTBgYxxvf7uSa3m24KC7Kw9U1DtaCMMY0SI9d3ZU2kSH8au5a8ovsVFN1WEAYYxqkiJBAnrqxF9syc3hmgZ1qqg4LCGNMg/XjLq24bXA8byzdyYL1B3xdTr1jAWGMadB+d203esc25Zdzf2D3oVxfl1OvWEAYYxq04AB/XrrlIgS4b1YKeYXWH1FZFhDGmAavffMwnhvbh/V7j/HUpxt9XU69YQFhjGkUrugew5RhHXh7eRrzf6js7MeNmwWEMabR+OVVXUiKj+KxeWvZlpnj63LqPAsIY0yjEejvx79u6UtIoD9TZiSzcMMBiksqHtF6z+FcZq5Ia5SB4tU7qUVkBPBPnDmpX1fVp89aPwx4HugNjFPVua7lfYB/A5FAMfCUqr7nzVqNMY1Dm6ahvHTLRTzy3hqmvJ1Cu2ahTBgUx81J7U/Nr308r5DP1u1nXsreM6ZOvbhjNLcOiufy7jEEVmNCpPrGa/NBiIg/sAW4AkgHVgHjVTW1zDYJOCHwKDC/TEB0BlRVt4pIWyAF6Kaq2eW9ns0HYYypiqLiEr7cmMGMZWl8t/0QQf5+XNe7DcWqfLHhAHmFJXRo0YTR/WK5tGsrFm/OZOby3ezNPklMZDDjB8RxXe+2RIYEEBzgT3CgM6WqiPj60KrEJ3NSi8hg4HFVvcr1+DcAqvoXN9tOBz4pDQg3638Axqjq1vJezwLCGFNdWzOO8/byND74fi9+Atdf2JbR/WLp277ZGW/4xSXK15szmbEsjSVbstw+V3CAHwnRTbgoPoqk+Cj6xUcRHx1W6eBQVeb/sI+92SfxE0HA+S7QMiKY63u3xc/PcyHkqzmp2wF7yjxOBwZW9UlEZAAQBGx3s24KMAUgLi6uelUaYxq9TjERPDGqJ7+7thuClDukuL+fcFm3GC7rFkPaoROs2nWEvMJi8otKTn0/WVDElowcPlm7j3dX7gac6VAHJDbnwcs60bV1ZLl1HMsr5NE5P7AwNaPcbfZl53Hv8I41O+BK8mZAuIu4KjVXRKQN8DZwu6qWnPNkqlOBqeC0IKpTpDHGlAoO8K/0tvHRTYiPblLu+pISZWtmDslph0lJO8LiTZks3JDBvcM78rMfX0BI4JmvtSXjOD99O4W0w7n84bruTBgYhyooSok6LYvHPljH3xZuZkBiFP3iKzf8eU14MyDSgfZlHscClb74WEQigU+B36vqcg/XZowxXuXnJ3RpHUGX1hFMGBjPkRMFPPlpKv/67zY+W7efp0f3PjXHxSdr9/GruWsJCwpg1t0DGdjB/Ux4f7mxF+vSj/LArNV89tAlNAsL8u4xePG5VwGdRCRRRIKAccD8yuzo2v5DYIaqvu/FGo0xplZENQni72P7MOPOAeQXlXDTK8v4/UfrePKTVO6ftZqurSP49MGh5YYDQGRIIC/e0pesnHwefX8t3upDLuW1gFDVIuB+4AtgIzBHVTeIyBMiMhJARPqLSDpwE/CqiGxw7T4WGAbcISJrXF99vFWrMcbUlmGdW7LwkWHcNTSRWSt288bSndw2OJ7ZUwYTExly3v17xzbjsau78eXGDKZ/t8urtXrtKqbaZlcxGWPqm/V7j5J1PJ8fd21Vpf1UlckzUliyJZN5915M79hm1a6hoquYGv6dHsYYU0f1bNe0yuEAICL87abetAwP5v5ZqzmWV+iF6iwgjDGmXmoWFsQL4/uyN/skv/1gnVf6I7w61IYxxhjvSUpozqNXduFkYTGq4OmbuC0gjDGmHvPmTXN2iskYY4xbFhDGGGPcsoAwxhjjlgWEMcYYtywgjDHGuGUBYYwxxi0LCGOMMW5ZQBhjjHGrwQzWJyJZQFoNnqIFcNBD5dRFdnz1X0M/Rjs+34hX1ZbuVjSYgKgpEUkub0TDhsCOr/5r6Mdox1f32CkmY4wxbllAGGOMccsC4rSpvi7Ay+z46r+Gfox2fHWM9UEYY4xxy1oQxhhj3LKAMMYY41ajDwgRGSEim0Vkm4g85ut6PEFEpolIpoisL7OsuYgsEpGtru9RvqyxJkSkvYgsFpGNIrJBRB5yLW8QxygiISKyUkR+cB3fn1zLE0Vkhev43hORIF/XWhMi4i8iq0XkE9fjhnZ8u0RknYisEZFk17J69TvaqANCRPyBl4Crge7AeBHp7tuqPGI6MOKsZY8BX6lqJ+Ar1+P6qgj4hap2AwYBP3P9vzWUY8wHLlXVC4E+wAgRGQT8FfiH6/iOAHf5sEZPeAjYWOZxQzs+gB+rap8y9z/Uq9/RRh0QwABgm6ruUNUCYDYwysc11ZiqfgMcPmvxKOAt189vATfUalEepKr7VfV718/Hcd5k2tFAjlEdOa6Hga4vBS4F5rqW19vjAxCRWOBa4HXXY6EBHV8F6tXvaGMPiHbAnjKP013LGqIYVd0Pzhss0MrH9XiEiCQAfYEVNKBjdJ1+WQNkAouA7UC2qha5Nqnvv6vPA78CSlyPo2lYxwdOqC8UkRQRmeJaVq9+RwN8XYCPiZtldt1vPSEi4cA84GFVPeZ8CG0YVLUY6CMizYAPgW7uNqvdqjxDRK4DMlU1RUSGly52s2m9PL4yhqjqPhFpBSwSkU2+LqiqGnsLIh1oX+ZxLLDPR7V4W4aItAFwfc/0cT01IiKBOOEwU1U/cC1uUMcIoKrZwNc4fS3NRKT0Q119/l0dAowUkV04p3UvxWlRNJTjA0BV97m+Z+KE/ADq2e9oYw+IVUAn19UTQcA4YL6Pa/KW+cDtrp9vB/7jw1pqxHW++g1go6r+vcyqBnGMItLS1XJAREKBy3H6WRYDY1yb1dvjU9XfqGqsqibg/M39V1Un0ECOD0BEmohIROnPwJXAeurZ72ijv5NaRK7B+fTiD0xT1ad8XFKNici7wHCc4YUzgD8CHwFzgDhgN3CTqp7dkV0viMhQ4H/AOk6fw/4tTj9EvT9GEemN04Hpj/Mhbo6qPiEiHXA+cTcHVgMTVTXfd5XWnOsU06Oqel1DOj7XsXzoehgAzFLVp0Qkmnr0O9roA8IYY4x7jf0UkzHGmHJYQBhjjHHLAsIYY4xbFhDGGGPcsoAwxhjjlgWEMVUgIsWu0TlLvzw22JqIJJQdgdcYX2vsQ20YU1UnVbWPr4swpjZYC8IYD3CN/f9X1zwOK0XkAtfyeBH5SkTWur7HuZbHiMiHrjkffhCRi11P5S8ir7nmgVjoupPaGJ+wgDCmakLPOsV0c5l1x1R1APAizt35uH6eoaq9gZnAC67lLwBLXHM+XARscC3vBLykqj2AbGC0l4/HmHLZndTGVIGI5KhquJvlu3Am+dnhGkjwgKpGi8hBoI2qFrqW71fVFiKSBcSWHUrCNXT5ItdkMojIr4FAVf2z94/MmHNZC8IYz9Fyfi5vG3fKjj1UjPUTGh+ygDDGc24u832Z6+fvcEYsBZgALHX9/BVwL5yaHCiytoo0prLs04kxVRPqmumt1AJVLb3UNVhEVuB88BrvWvYgME1EfglkAZNcyx8CporIXTgthXuB/V6v3pgqsD4IYzzA1QeRpKoHfV2LMZ5ip5iMMca4ZS0IY4wxblkLwhhjjFsWEMYYY9yygDDGGOOWBYQxxhi3LCCMMca49f8B6SiBrxLw7FsAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3yV9fn/8deVQQIkrJCEkBAS9l6Goag4+wWRoaLi3rS11tHWOn5WraO19mtb26+tdeBoQYugiJMKVdCCSJBNmGFkQUIgIQEyz/X74z7IMSQQIHdOcs71fDzOI+ceOee6NXze9/x8RFUxxhgTvEL8XYAxxhj/siAwxpggZ0FgjDFBzoLAGGOCnAWBMcYEOQsCY4wJchYExviRiHwhIrf7uw4T3CwIjKmDiLQWkVIR+biWZTtE5LB3+R4ReU1Eour4nHqve5xaUkRERSTsVLfHmLpYEBhTtylAOfADEUmoZfkEVY0ChgHDgUeO81kns64xjcqCwAQF7175/SKyRkQOisirIhIvIp+ISImILBCR9jV+7SbgRWANcF1dn62qOcAnwIAT1XG8dUUkREQeEZGdIpIvIm+KSFvv4sXen0XeI4szT7zVxtSPBYEJJlcAFwO9gAk4DfLDQEecfwt3H1lRRJKB84AZ3teNdX2oiHQBLgFWnqiAE6x7s/d1PtANiAL+z7vsXO/PdqoapapLT/RdxtSXnW80weQvqroHQES+BPJVdaV3+j3gQp91bwTWqOoGESkCnhWRoUfW95orIlVAMfAR8JvjfHd91r0O+IOqZnpreghYJyK3nMrGGlNfFgQmmOzxeX+4lmnfC7g3Ai8DqGquiCzCOVXkGwSTVXVBPb+7Put2Bnb6TO/E+TcaX8/vMOaU2KkhY2oQkbOAnsBDIrJbRHYDI4FrXL5rJxfo6jOdDFThBJZ1E2xcY0FgzLFuAj4D+gFDvK8BQCtgnIvf+xZwn4ikem8v/Q3wL1WtAgoAD861A2MalAWBMT5EJBK4Cud6wm6f13bgHzgh4Zbp3u9YDGwHyoCfAqjqIeBp4L8iUiQio1yswwQZsYFpjDEmuNkRgTHGBDkLAmOMCXIWBMYYE+QsCIwxJsi5+kCZiIwFngdCgVdU9Zkay5OBN4B23nUeVNVjenr01bFjR01JSXGnYGOMCVArVqzYq6qxtS1zLQhEJBR4Aadvl2xguYjMU9UNPqs9AsxS1b+JSD/gYyDleJ+bkpJCenq6S1UbY0xgEpGddS1z89TQCGCrqmaqagXwNjCpxjoKtPG+b4vzZKUxxphG5GYQJAJZPtPZ3nm+HgeuF5FsnKOBn9b2QSIyTUTSRSS9oKDAjVqNMSZouRkEUsu8mk+vXQO8rqpJOF3z/kNEjqlJVV9S1TRVTYuNrfUUlzHGmFPk5sXibKCLz3QSx576uQ0YC6CqS72P93cE8k/miyorK8nOzqasrOw0ym36IiMjSUpKIjw83N+lGGMCiJtBsBzoKSKpQA4wFbi2xjq7cPqAf11E+gKROJ1rnZTs7Gyio6NJSUlBpLYDkeZPVSksLCQ7O5vU1FR/l2OMCSCunRry9ph4FzAfyMC5O2i9iDwhIhO9q/0cuENEVuP0vHiznkLnR2VlZcTExARsCACICDExMQF/1GOMaXyuPkfgfSbg4xrzHvV5vwEY3RDfFcghcEQwbKMxpvHZk8XGGNOEVXuUlbv28/yCLWTkHXDlO2yoygZQVFTEzJkzufPOO0/q9y655BJmzpxJu3btXKrMGNMc5R8oY9HmAhZv2cuXWwooOlSJCHSIakHfhDYn/oCTZEHQAIqKivjrX/96TBBUV1cTGhpa5+99/PFxe9MwxgSJ8qpqVuzYz6ItBSzaVMDG3SUAdIyK4MI+8ZzbqyPn9IylQ+sWrny/BUEDePDBB9m2bRtDhgwhPDycqKgoEhISWLVqFRs2bGDy5MlkZWVRVlbGPffcw7Rp04Cj3WWUlpYybtw4zj77bJYsWUJiYiLvv/8+LVu29POWGWPcsrPwoLPXv7mAJdsKOVRRTXiocEbX9jwwtg/n9upI305tCAlx/9pgwAXBrz9Yz4bchj2P1q9zGx6b0L/O5c888wzr1q1j1apVfPHFF4wfP55169Z9d5vn9OnT6dChA4cPH2b48OFcccUVxMTEfO8ztmzZwltvvcXLL7/MVVddxZw5c7j++usbdDuMMf5zsLyKrzMLv2v8dxQeAqBLh5ZcPiyRMb3iOLN7DFERjd8sB1wQNAUjRoz43r3+f/7zn3nvvfcAyMrKYsuWLccEQWpqKkOGDAHgjDPOYMeOHY1WrzGm4akqG3eXsHhzAYs2F7B8xz4qq5WW4aGc2T2Gm89KYUzvOFJiWvn9jsCAC4Lj7bk3ltatW3/3/osvvmDBggUsXbqUVq1acd5559X6LEBERMR370NDQzl8+HCj1GqMaThFhyr4csteFm8uYPGWAvYcKAegd3w0t4xOZUyvWNJS2hMRVve1Q38IuCDwh+joaEpKSmpdVlxcTPv27WnVqhUbN27k66+/buTqjDFuqfYoq7OLWLTJ2etfk12ER6FNZBjn9IxlTK9YzunVkYS2Tft6nwVBA4iJiWH06NEMGDCAli1bEh8f/92ysWPH8uKLLzJo0CB69+7NqFGj/FipMaYhbM0v5fmFW1i8uYDiw86tnYOT2vHTC3pybq9YBie1JSy0+TymJafQo4NfpaWlac2BaTIyMujbt6+fKmpcwbStxjQ11R7l1a8y+d9/byYyLIQf9O/EmF6xnN2jI+1durWzoYjIClVNq22ZHREYY0w9ZBaU8ot3VvPtriIu7hfP05cNIC460t9lNQgLAmOMOY5qj/Laf7fz+/mbiAwP5U9XD2HSkM5+v9OnIVkQGGNMHbbvPcj976wmfed+Luobx28uG0hcm8A4CvBlQWCMMTV4PMrrS3bw7PyNtAgN4bkrB3P5sMSAOgrwZUFgjDE+dhYe5P7Za/hm+z7O7x3Lby8fRKe2gXcU4MuCwBhjcI4C/vH1Tp75ZCNhocLvpwxiyhlJAXsU4MuCwA+ioqIoLS31dxnGGK+sfYe4f/Zqvs7cx5hesTxzxcAm/xBYQ7IgMMYELY9HmfHNLn77cQahIjx7xSCuTAuOowBfFgQN4IEHHqBr167fjUfw+OOPIyIsXryY/fv3U1lZyVNPPcWkSZP8XKkx5oisfYd4YM4almwr5JyeHfndFYPo3C54jgJ8BV4QfPIg7F7bsJ/ZaSCMe6bOxVOnTuXee+/9LghmzZrFp59+yn333UebNm3Yu3cvo0aNYuLEiUG3p2FMU6OqzPxmF7/5KAMR4beXD2Tq8C5B/W8z8ILAD4YOHUp+fj65ubkUFBTQvn17EhISuO+++1i8eDEhISHk5OSwZ88eOnXq5O9yjQlaOUWHeXDOGr7cspfRPWL43RWDSGrfyt9l+V3gBcFx9tzdNGXKFGbPns3u3buZOnUqM2bMoKCggBUrVhAeHk5KSkqt3U8bY9ynqvxreRZPfZSBR5WnJg/gupHJQX0U4CvwgsBPpk6dyh133MHevXtZtGgRs2bNIi4ujvDwcD7//HN27tzp7xKNCUp5xYd5YM5aFm8u4MxuMTw7ZRBdOthRgC8LggbSv39/SkpKSExMJCEhgeuuu44JEyaQlpbGkCFD6NOnj79LNCaoqCrvrMjmyQ82UOVRnpzUn+tGdm2UMYCbG1eDQETGAs8DocArqvpMjeV/BM73TrYC4lS1nZs1uWnt2qMXqTt27MjSpUtrXc+eITDGXbuLy3jo3TV8vqmAkakd+P2UwSTH2FFAXVwLAhEJBV4ALgaygeUiMk9VNxxZR1Xv81n/p8BQt+oxxgQ+VWXOtzn8+oP1VFZ7eHxCP248M8WOAk7AzSOCEcBWVc0EEJG3gUnAhjrWvwZ4zMV6jDEBbM+BMh56dy3/2ZjP8JT2/H7KYFI6tj7xLxpXgyARyPKZzgZG1raiiHQFUoH/1LF8GjANIDk5udYvU9WAvwOguY0mZ0xj2L73IHNX5vDaf7dTUe3hV5f245az7CjgZLgZBLX9X6irJZsKzFbV6toWqupLwEvgDFVZc3lkZCSFhYXExMQEbBioKoWFhURGBnYviMbUR2FpOR+uyeO9lTmsyipCBM7tGctjE/rRLTbK3+U1O24GQTbQxWc6CcitY92pwE9O9YuSkpLIzs6moKDgVD+iWYiMjCQpKcnfZRjjF2WV1Xy2YQ9zV+awaHMBVR6lb0IbHr6kDxMHJwZ8V9FucjMIlgM9RSQVyMFp7K+tuZKI9AbaA7XfYlMP4eHhpKamnuqvG2OaqGqPsiyzkHdX5vDput2UllfRqU0kt52TymVDE+nTqY2/SwwIrgWBqlaJyF3AfJzbR6er6noReQJIV9V53lWvAd5WOwFujPHKyDvA3JU5vL8ql90HyoiOCOOSgZ2YPDSRkakxhNr5/wYlza39TUtL0/T0dH+XYYxpYHnFh3l/VS5zV+awcXcJYSHCeb1jmTw0kYv6xhMZHurvEps1EVmhqmm1LbMni40xflNSVskn63Yzd2UOSzMLUYWhye14YlJ/Lh3UmQ6tW/i7xKBgQWCMaVSV1R4Wby7g3ZU5LNiwh/IqDykxrbjnwp5MHpJo9/77gQWBMcZ1qsrKrCLmrszhwzV57DtYQftW4Vw9vAuXDU1kSJd2AXvrd3NgQWCMcc2OvQeZuyqHuStz2FF4iIiwEC7uF89lQxM5t1cs4aEh/i7RYEFgjGlgew6U8cnaPN5fncvKXc7DXmd2i+HO83swbkAnoiPD/V2iqcGCwBhz2vIPlPHJut18tCaP5Tv3oQp9OkXz0Lg+TBzSmYS2wTkWcHNhQWCMOSX5JWV8um43H67JY/kOp/HvHR/NvRf2YvygTvSIi/Z3iaaeLAiMMfWWX1LGfG/j/4238e8VH8U9F/Zk/MAEesZb498cWRAYY46roKScT9fv5qM1uSzb7jT+PeKiuPuCnowflEAva/ybPQsCY8wx9paW86n3nP+y7YV4FLrHtuanF/TkUmv8A44FgTEGcLp2dvb88/g602n8u8W25q7zezB+UGd6xUfZvf4ByoLAmCBWWFrO/PV7+GhtLku3eRv/jq35yfk9GD8ogd7x0db4BwELAmOCzL6DFcz37vkvzSyk2qOkdmzNnec5jX+fTtb4BxsLAmOCwL6DFfx7/W4+WpvHkm1O458S04ofjenG+IGd6ZtgjX8wsyAwJkBtKyhlwYY9LMjYw4qd+/EodI1pxQ/P7cb4QQn0S2hjjb8BLAiMCRhV1R7Sd+5nYcYeFmTks33vQQD6JbThrvN78IP+nejf2Rp/cywLAmOasQNllSzeXMCCDXv4fFMBxYcraREawqjuMdw6OoUL+saT2M66dzDHZ0FgTDOTte/Qd3v9X2cWUuVR2rcK56K+8VzUN45zesUSFWH/tE392V+LMU2cx6Oszi5iQcYeFmbks3F3CeA84HXbOalc3DeeocntbRxfc8osCIxpgg5VVPHVlr0szMhn4cZ89paWExoiDE9pzyPj+3Jh33hSbSQv00AsCIxpIvYcKHMa/ow9fLV1L+VVHqIjwhjTO5aL+8VzXq842rayvvxNw7MgMMZPVJUNeQdYmJHPgow9rMkuBqBLh5ZcOzKZi/rGMzylAy3CbBQv4y4LAmMa2c7Cg7y+ZAfz1+0mt7gMERjapR33/09vLu4XT88469PHNC4LAmMaydrsYl5cvI1P1uYRFhLCmN6x3HtRL87vE0dsdIS/yzNBzNUgEJGxwPNAKPCKqj5TyzpXAY8DCqxW1WvdrMmYxqSqfLllLy8u2saSbYVER4Qx7dzu3Do6hbg2kf4uzxjAxSAQkVDgBeBiIBtYLiLzVHWDzzo9gYeA0aq6X0Ti3KrHmMZUVe3ho7V5vLgok4y8A8S3ieDhS/pwzYhkG7zdNDluHhGMALaqaiaAiLwNTAI2+KxzB/CCqu4HUNV8F+sxxnWHKqqYtTyLl7/cTk7RYXrERfHslEFMHpJoF31Nk+VmECQCWT7T2cDIGuv0AhCR/+KcPnpcVT+t+UEiMg2YBpCcnOxKscacjsLSct5YupM3l+6g6FAlaV3b8+uJ/bmgTxwh9qCXaeLcDILa/vq1lu/vCZwHJAFfisgAVS363i+pvgS8BJCWllbzM4zxm12Fh3jlq0xmpWdRVunh4n7x/PDcbqSldPB3acbUm5tBkA108ZlOAnJrWedrVa0EtovIJpxgWO5iXcactnU5xby4aBsfr80jNES4fGgSd5zbjR5xUf4uzZiT5mYQLAd6ikgqkANMBWreETQXuAZ4XUQ64pwqynSxJmNOmary1da9/H1RJl9t3Ut0RBh3nNuNW0enEm93AJlmzLUgUNUqEbkLmI9z/n+6qq4XkSeAdFWd5132AxHZAFQD96tqoVs1GXMqqqo9fLxuN39ftI31uQeIi47goXF9uGZkMm3sDiATAES1eZ1yT0tL0/T0dH+XYYLA4YpqZqVn8fKXmWTvP0z32Nb88NzuTBramYiwUH+XZ8xJEZEVqppW2zJ7stiYGvYdrODNpTt4Y8kO9h+q5Iyu7XlsQn8utDuATICyIDDGK2vfIV75MpN/ee8AuqhvPD8aY3cAmcBnQWCCmqqSvnM/by7dycdr8wgRmDwkkR+O6UaPuGh/l2dMo7AgMEGp+HAl732bzcxvdrF5TynREWHcfnYqt4xOpVNbuwPIBBcLAhM0VJVVWUXMWLaLD9fkUlbpYXBSW569YhCXDk6gVQv752CCk/3lm4BXUlbJ3FW5zFy2i4y8A7RuEcrlw5K4dkQyAxLb+rs8Y/zOgsAErLXZxcz8Zifvr8rlUEU1/Tu34enLBjBpSCJREfanb8wR9q/BBJSD5VXMW+3s/a/NKaZleCgTBidw7ciuDE5qayN/GVMLCwITEDbkHmDmNzuZuzKX0vIq+nSK5olJ/Zk8NNGe/jXmBCwITLN1uKKaD9fkMmPZLlZlFRERFsL4QQlcNzKZYcntbe/fmHqyIDDNzuY9Jcxctos532ZTUlZF99jW/OrSflwxLJF2rVr4uzxjmh0LAtMslFVW88m6PGYu28XyHftpERrCuIGduHZEMiNSO9jevzGnwYLANGnbCkq/2/svOlRJasfWPHxJH6ac0YUOrW3v35iGYEFgmpyqag+frNvNjGU7+TpzH2Ehwv/078R1I5MZ1S3GOn4zpoFZEJgmZcXO/Twydx0ZeQfo0qElvxzbmyvP6EJsdIS/SzMmYFkQmCah6FAFv/t0I299k0WnNpG8cO0wxg3oZHv/xjQCCwLjV6rK7BXZ/PaTjRQfruSOc1K556Je9uSvMY3I/rUZv9m0u4RH5q5l+Y79nNG1PU9NHkDfhDb+LsuYoGNBYBrdwfIq/rxwC69+tZ3oyDCevWIQU85IstNAxviJBYFpNKrK/PV7eOKD9eQWl3F1WhceGNfHbgM1xs8sCEyjyNp3iMfmrec/G/Pp0ymaP18z1IaANKaJsCAwrqqo8vDyl5n85T9bCBHh/13Sl5tHpxAeGuLv0owxXhYExjVLtu3lV3PXsa3gIGP7d+LRCf3o3K6lv8syxtTg6m6ZiIwVkU0islVEHqxl+c0iUiAiq7yv292sxzSOgpJy7n17Jde+vIyKag+v3TycF284w0LAmCbKtSMCEQkFXgAuBrKB5SIyT1U31Fj1X6p6l1t1mMZT7VFmLtvJs/M3UVZZzU8v6MFPzu9BZHiov0szxhxHvYJARDqq6t6T/OwRwFZVzfR+xtvAJKBmEJgAsDa7mEfmrmV1djFndY/hyckD6B4b5e+yjDH1cNwgEJEJwHSgSkSqgatUdUk9PzsRyPKZzgZG1rLeFSJyLrAZuE9Vs2pZxzRRB8oqeW7+Jv7x9U5ioiJ4fuoQJg7ubN1CG9OMnOiI4GngHFXdKCIjgWeBMfX87NpaAq0x/QHwlqqWi8iPgDeAC475IJFpwDSA5OTken69cZOqMm91Lk9+mMG+g+XcMKorP/+f3jYspDHN0ImCoEpVNwKo6jIRiT6Jz84GuvhMJwG5viuoaqHP5MvA72r7IFV9CXgJIC0trWaYmEa2raCUR99fx3+3FjIoqS2v3TycgUlt/V2WMeYUnSgI4kTkZ3VNq+ofjvO7y4GeIpIK5ABTgWt9VxCRBFXN805OBDLqXblpdGWV1bzw+Vb+viiTiPAQnpw8gGtHJBNqXUMY06ydKAheBqLrmD7unrmqVonIXcB8IBSYrqrrReQJIF1V5wF3i8hEoArYB9x88ptgGsPnG/N5dN46svYd5rKhiTx8SV8bI8CYACGqp3amRUSGq+ryBq7nhNLS0jQ9Pb2xvzZoVVV7eGzeemYs20X32NY8OXkAZ3Xv6O+yjDEnSURWqGpabctO6jkCEemHc4rnGqAYqPVDTWAoq6zm7rdW8u8Ne/jhud34+Q960yLMuoYwJtCcMAhEpCtOw38NzimcrkCaqu5wtzTjT8WHK7njzXSW79jH4xP6cfPoVH+XZIxxyYmeI1gCtAXeBqao6hYR2W4hENj2HCjjpunfsK2glOenDmXi4M7+LskY46ITHREU4Nz2GQ/EAls4wUVi07xlFpRyw6vfUHSogtduHsHZPe16gDGB7rgnfFV1EjAQ+Bb4tYhsB9qLyIjGKM40rtVZRUx5cSllldW8Pe1MCwFjgsQJrxGoajFONxPTRSQOuBr4o4h0UVV7zDdALN5cwI/+uYKYqBa8eetIUju29ndJxphGUt9O59KAh4EU4EgfAu1dqsk0svdX5fDzWavpGR/NG7cMJ65NpL9LMsY0ovrePjoDuB9YC3jcK8c0tle/2s6TH25gVLcOvHRjmvUVZEwQqm8QFHifBDYBQlX53aebeHHRNsYN6MQfrx5i4wYYE6TqGwSPicgrwEKg/MhMVX3XlaqMq6qqPTz47lpmr8jmupHJPDFpgPUXZEwQq28Q3AL0wbk+cOTUkAIWBM3M4Ypq7pr5LQs35nPvRT2558KeNnaAMUGuvkEwWFUHulqJcV3RoQpufX05q7KKeGryAK4f1dXfJRljmoD6BsHXItKvlvGGTTORW3SYG6d/w67CQ/z1umGMHZDg75KMMU1EfYPgbOAm7wNl5Tijj6mqDnKtMtNgtuaXcMOr31BaVsUbt47gzO4x/i7JGNOE1DcIxrpahXHNip37ue2N5YSHhvCvH55Jv85t/F2SMaaJqVcQqOpOtwsxDe8/G/dw54xv6dQmkn/cNpIuHVr5uyRjTBN0UuMRmOZj9opsHpizhn4JbXjtluF0jLLRxIwxtbMgCDCqyt8XZ/LMJxs5u0dHXrzhDKIi7H+zMaZu1kIEEI9HefrjDF79ajsTBnfmuSsH24hixpgTsiAIEBVVHu6fvZr3V+Vy81kpPHppP0LsaWFjTD1YEASAg+VV/HjGtyzeXMAvx/bmx2O629PCxph6syBo5gpLy7n19eWszSnm2SsGcdXwLv4uyRjTzFgQNGNZ+w5x0/RvyCk6zN9vSOPifvH+LskY0wxZEDRTGXkHuGn6N5RVVjPj9pGkpXTwd0nGmNOh6rxQUI932vP96dAWENaiwb/a1SAQkbHA80Ao8IqqPlPHelOAd4DhqpruZk2BYPmOfdz6+nJatwhj9o/Pold8tL9LMiawqULRLshb7bx2r4GCTeCpqrvR/m76OA2773R9jP8DDL+twTfPtSAQkVDgBeBiIBtYLiLzanZcJyLRwN3AMrdqCSRZ+w5x+xvpxEZF8I/bR5LYrqW/SzImsHiqoXCbt9Ff5TT6eWugrMhZLqEQ2weShkN4SxABCQG8P+ucppblx/vdWpZ3GeHKJrt5RDAC2KqqmQAi8jYwCajZg+mTwLPAL1ysJSCUVzljCXg8ymu3DLcQMOZ0VVVAwcaje/l5q2H3Wqg85CwPjYD4/tD/MkgYBAmDIa6fEwABxM0gSASyfKazgZG+K4jIUKCLqn4oInUGgYhMA6YBJCcnu1Bq8/CbjzJYnV3Mi9cPo2tMa3+XY0zzUnEI9qyH3auPnuLJz4DqCmd5iyjoNAiG3eg0+AmDoWMvCA38cbzdDILabmTX7xaKhAB/BG4+0Qep6kvASwBpaWl6gtUD0odrcnlj6U5uOzvVxhIw5kTKip3TOUf28vPWwN5NR8/Ft+zg7OGP+rHT4HcaDB26QUhwPonvZhBkA743tScBuT7T0cAA4Avvw0+dgHkiMtEuGH9fZkEpD85Zy7Dkdjw4ro+/yzHNgarTGB7I9b5yAIXIdhDZFlq2O/o+si2EhPq74vpRhYqDzraVFTk/D3t/luQ6DX7eati//ejvRCc4jX2/ic4ef8JgaJvknH83gLtBsBzoKSKpQA4wFbj2yEJVLQY6HpkWkS+AX1gIfN/himrunPEt4aHC/107jPDQ4NxjMT48HjhU6DTuJXnOT98G/8j7I+e56yPCGwgt29YSFu2+Hxw134edZM+21VVQfgAO76+9QS8r+v77mss8VXV/dvsUp6EfdoOzl58wCKLiTq6+IORaEKhqlYjcBczHuX10uqquF5EngHRVnefWdweSR99fx6Y9Jbx+ywg628XhwOephtI9NRr1HDiQd/R9Sd7R89pHSKiz59umM8QPgJ7/47xvkwBtEp1lIWE1Gtaiuhvfwm1H358oUMJa1h4QIWG1N+gVJcf/vJCwGuHTzmngI2sLqSPv20KrjhBpAy+dClefI1DVj4GPa8x7tI51z3OzluZoVnoW76zI5u4LejCmV6y/yzENoawY9mU6r+KcY/fiS/eAVn//d0IjvI16InQZebRxb9P56PzWsfU7vdM28eRrrqqoPSx89+h955fkOXfieKqONujtun6/0T5egx7eyk7bNDJ7sriJysg7wK/mrmN0jxjuuaiXv8sx9aXqNJBHGvuar0OF31+/RdTRBr37+d9v3KO9DX6rDv5tGMNaQFSs8zIByYKgCSopq+TOGd/StmU4f7p6KKHWnXTTogoHC+pu7MuKfVYW58Jkh1ToO8G5M6VDN2ifCu2S7VSGaRIsCJoYVeXBOWvZte8Qb90xitjoIBxisqzYeagnJMy5EBkW6e1jJdL7OjLPxT9fjwdKd9fR2G+Hiodm/dkAABJ6SURBVNKj60qI06h36AYDrzza2Hfo5pwSCY90r05jGoAFQRPzxpIdfLQ2jwfH9WFEapB0JKfqNPxbP4MtCyBr2bHnyWsjod5QiPh+QIRFOOfVa5tf17ohoVCc5TTyRxr7qsNHvyskHNp3dRr3rqO/39i37eJKR2DGNBYLgiZk5a79PP1xBhf1jWPaOd38XY67Du+HbZ/D1oWwdYGz9w3Ofd5n3wvJZznnxasroKoMqsp9fpb7THvnVZfXWKfMuch5qPD7833Xq3nnTWiEcwqnQzfofsHR9x26QZskd49AjPEj+8tuIvYfrOCumSuJbxPJc1cOCbxhJj0e5ynPI3v92cudvf7Itk6j2+Ni6HERRDfimAoez9Fg8FQ5T5sG6ZOlJrhZEDQBHo/ys1mrKCgpZ/aPz6RtqwDp2+TQPtj2n6N7/QfznfkJQ+CcnzmNf+IZ/tvTDgmBkJYB14GYMSfLgqAJ+NuibXy+qYAnJ/VnUFI7f5dz6jwep9verQtgy2eQk+707dKyvc9e/4X2pKcxTYwFgZ8t3VbIc//exITBnbl+VFd/l3PyDhZ69/oXwLaFzm2VCHQeCuf8Anp69/qbS182xgQhCwI/yi8p46dvrSSlY2t+e/lApDk8TemphtxV3nP9n0HOCkCd8+s9LnT2+rtfYA8fGdOMWBD4SVW1h7vfWklpeSUzbh9JVEQT/l9RdgA2feI0/tv+4306Vpw9/TEPOHv9nYfaXr8xzVQTbn0C2x8XbObrzH3875WD6d2piY45XJwDy/4GK95weots1dG5s+fIXn/rGH9XaIxpABYEfvD5xnxe+HwbV6d1YcoZSf4u51i718GSv8C62c7F3n6TYeSPnDFa7fZKYwKOBUEjyyk6zH2zVtE3oQ2/ntTf3+UcpQqZXzgBsG0hhLeG4bfDqDudJ2qNMQHLgqARVVR5+MmMb6mqVv563TAiw5vAOfXqSlj/Hiz5s9PNQ+s4uOBXkHar0+ulMSbgWRA0ot98nMGqrCL+dt0wUjv6efD58hLn3P/Xf4MD2c4g3RP/AgOvsk7SjAkyFgSN5KM1eby+ZAe3jE5h3EA/Dj5/IM+5AJz+OpQXQ9ezYfxz0PMHdv7fmCBlQdAIMgtKeWDOGoYmt+OhcX39U8SeDbD0/2DNLKePn74T4ay7IekM/9RjjGkyLAhcVlbpDD4f5h18vkVYI+51q8L2xc4F4K2fOUMApt3iXADukNp4dRhjmjQLApc9+v46Nu4u4bVbhpPYWIPPV1fBhrnOBeC81c54tuc/AsNvswvAxphjWBC46J30LGalZ3PX+T04v3cjdLRWXgor/wFL/wrFuyCmJ0x4HgZNtQvAxpg6WRC4ZOPuA/zq/XWc2S2G+y52efD5kt2w7O+Q/qozzGPymTDud9BrrF0ANsackAWBC0rKKrnzn98SHRnO89cMcW/w+fyNsPQvzgXg6kpncPSz7oYuw935PmNMQLIgaGCqyoPvrmVH4UFm3jGKuGgXTsnkfAtfPANb5kNYSxh2o3MBOKZ7w3+XMSbguRoEIjIWeB4IBV5R1WdqLP8R8BOgGigFpqnqBjdrctubS3fy0Zo8fjm2N6O6NXCnbKrOLaALHneGeDzvYacbCOv8zRhzGlwLAhEJBV4ALgaygeUiMq9GQz9TVV/0rj8R+AMw1q2a3LYqq4inPtrAhX3i+NG5Dbx3fng/zP0JbPrIOQU06QUnDIwx5jS5eUQwAtiqqpkAIvI2MAn4LghU9YDP+q0BdbEeVxUdquAnM74lLjqS564a3LCDz+d8C+/cDAdyYOwzTk+gzWEQG2NMs+BmECQCWT7T2cDImiuJyE+AnwEtgAtq+yARmQZMA0hOTm7wQk+XM/j8avJLynjnR2fRrlWLhvlgVVj+Csx/2OkM7pZP7UKwMabBuXlvYW27rMfs8avqC6raHXgAeKS2D1LVl1Q1TVXTYmOb3hCIf1+cyX825vPI+H4M6dJAg8+Xl8DsW+HjX0C38+BHX1oIGGNc4eYRQTbQxWc6Ccg9zvpvA39zsR5XrM8t5rl/b2L8wARuPLOB+u3fvQ7euQn2ZcKFj8Hoe+15AGOMa9xsXZYDPUUkVURaAFOBeb4riEhPn8nxwBYX62lwldUefjl7De1ateDpywY0zODzK/8Jr1zoPCV80wdwzs8sBIwxrnLtiEBVq0TkLmA+zu2j01V1vYg8AaSr6jzgLhG5CKgE9gM3uVWPG15anMn63AO8eP2w078uUHHIOQ20agakjoErXoGoRuiWwhgT9Fx9jkBVPwY+rjHvUZ/397j5/W7aml/C8wu2MH5gAmMHnOb4AgWbnVNB+Rkw5gHnFdIERi8zxgQFe7L4FFR7lPtnr6FVRCiPTzzNcYfXzoZ5dzudwl0/B3pc2DBFGmNMPVkQnILX/rudlbuK+NPVQ4iNjji1D6ksg/kPQfp06DIKrnwN2nRu2EKNMaYeLAhO0s7Cg/zvvzdxYZ84Jg05xYZ733bnVFDeahh9jzNYfGh4wxZqjDH1ZEFwEjwe5YE5awgPCeHpywae2l1CGR84XUUIMPUt6HNJg9dpjDEnw4LgJMz8ZhdfZ+7jmcsH0qntSfYqWlXhdBb39QvQeRhc+Tq0b6DnDowx5jRYENRTTtFhnvlkI6N7xHD18C4n/gVfRVkw+xbIXg4jfgg/eBLCTvHagjHGNDALgnpQVR5+dy3VHuWZywed3CmhLZ/Bu3c44whf+Tr0v8y1Oo0x5lRYENTDnG9zWLS5gMcn9KNLh1b1+6XqKvjiN/DlcxA/EK56wwaOMcY0SRYEJ5B/oIwnPlhPWtf23HhmSv1+qWQ3zL4Ndn7ljB427lkIb+lqncYYc6osCI5DVXlk7jrKqjz8bsqg+o0xkLkI5twOFaUw+UUYco37hRpjzGmwIDiOj9bm8e8Ne3hwXB+6x0Ydf2WPB778X/jitxDTE26aB3F9G6dQY4w5DRYEddh3sILH3l/PoKS23H526vFXPrgX3p0G2xbCwKvg0j9CxAmCwxhjmggLgjr8+oP1HCirZMaUkYSFHqcb6H2Z8MYkKN0Dl/4JzrjZhpE0xjQrFgS1WLBhD++vyuXei3rSp1Obulcs3AZvTIDKQ3Drp5A4rPGKNMaYBmJBUEPx4Ur+39y19OkUzZ3n9ah7xb1bnBCornAGkOk0sPGKNMaYBmRBUMNvPsqgoKScl29Mo0VYHaeECjY5IeCphps+hPh+jVukMcY0IBsD0ceXWwr4V3oW087tzqCkOgahz8+A18eDKtz8kYWAMabZsyDwOlhexYNz1tIttjX3XtSz9pV2r3NCQEKdEIjr07hFGmOMC+zUkNezn24kt/gw7/zwTCLDaxkmMm8NvDkJwiLh5g+tuwhjTMCwIwLgm+37eGPpTm46M4W0lA7HrpC7yrkmEN4KbvnIQsAYE1CCPgjKKqt5YM4aunRoyS/H9j52hZwV8OZEiGjjhECHbo1fpDHGuCjoTw398bPNbN97kBm3j6RVixr/ObKWwz8vh5btnVtEbSAZY0wACuojgtVZRbz8ZSbXjOjC6B4dv79w1zL4x2XQKsa5MGwhYIwJUEEbBOVV1dw/ezVx0ZE8dEmNzuF2LnGOBKLinBBod5IjkhljTDPiahCIyFgR2SQiW0XkwVqW/0xENojIGhFZKCKNttv9wufb2LynlN9cPoA2keFHF2z/Ev55BUQnOCHQNrGxSjLGGL9wLQhEJBR4ARgH9AOuEZGaT1+tBNJUdRAwG3jWrXp8bcg9wF8/38plQxO5oE/80QWZX8CMK6FtFycE2iQ0RjnGGONXbh4RjAC2qmqmqlYAbwOTfFdQ1c9V9ZB38msgycV6AKiq9vDLOatp1yqcRy/1yaWtC2Hm1dAh1QmB6Pi6P8QYYwKIm0GQCGT5TGd759XlNuCT2haIyDQRSReR9IKCgtMq6qUvM1mXc4AnJw2gfesWzswtC+Cta7wDynwIUbGn9R3GGNOcuBkEtXXKr7WuKHI9kAb8vrblqvqSqqapalps7Kk30lvzS/nTgi2MG9CJcQO9p302z4e3r4HY3s6oYq1jTvnzjTGmOXLzOYJswPd2myQgt+ZKInIR8P+AMapa7lYx1R7ll7NX06pFKL+e1N+ZufFjmHUjdBoAN7znPC9gjDFBxs0jguVATxFJFZEWwFRgnu8KIjIU+DswUVXzXayFN5bs4NtdRTw2oR9x0ZGQ8QHMugESBsENcy0EjDFBy7UgUNUq4C5gPpABzFLV9SLyhIhM9K72eyAKeEdEVonIvDo+7rSN7tGRH47pxuQhibD+PZh1E3Qe5j0SqKPLaWOMCQKiWutp+yYrLS1N09PTT/0D1s52BprvMgKuewciohuuOGOMaaJEZIWqptW2LLieLF4zC969A5JHwXWzLQSMMYZgCoLV/3KOBLqO9h4JRPm7ImOMaRKCJwjaJUOf8XDtLGjR2t/VGGNMkxE83VB3PdN5GWOM+Z7gOSIwxhhTKwsCY4wJchYExhgT5CwIjDEmyFkQGGNMkLMgMMaYIGdBYIwxQc6CwBhjglyz63RORAqAnaf46x2BvQ1YTlMU6Nto29f8Bfo2NtXt66qqtY7s1eyC4HSISHpdve8FikDfRtu+5i/Qt7E5bp+dGjLGmCBnQWCMMUEu2ILgJX8X0AgCfRtt+5q/QN/GZrd9QXWNwBhjzLGC7YjAGGNMDRYExhgT5IImCERkrIhsEpGtIvKgv+s5XSIyXUTyRWSdz7wOIvKZiGzx/mzvzxpPh4h0EZHPRSRDRNaLyD3e+YG0jZEi8o2IrPZu46+981NFZJl3G/8lIi38XevpEJFQEVkpIh96pwNt+3aIyFoRWSUi6d55zervNCiCQERCgReAcUA/4BoR6effqk7b68DYGvMeBBaqak9goXe6uaoCfq6qfYFRwE+8/88CaRvLgQtUdTAwBBgrIqOA3wF/9G7jfuA2P9bYEO4BMnymA237AM5X1SE+zw80q7/ToAgCYASwVVUzVbUCeBuY5OeaTouqLgb21Zg9CXjD+/4NYHKjFtWAVDVPVb/1vi/BaUgSCaxtVFUt9U6Ge18KXADM9s5v1tsoIknAeOAV77QQQNt3HM3q7zRYgiARyPKZzvbOCzTxqpoHTkMKxPm5ngYhIinAUGAZAbaN3tMmq4B84DNgG1CkqlXeVZr73+qfgF8CHu90DIG1feCE979FZIWITPPOa1Z/p8EyeL3UMs/um20GRCQKmAPcq6oHnB3KwKGq1cAQEWkHvAf0rW21xq2qYYjIpUC+qq4QkfOOzK5l1Wa5fT5Gq2quiMQBn4nIRn8XdLKC5YggG+jiM50E5PqpFjftEZEEAO/PfD/Xc1pEJBwnBGao6rve2QG1jUeoahHwBc71kHYicmQnrTn/rY4GJorIDpzTsRfgHCEEyvYBoKq53p/5OGE+gmb2dxosQbAc6Om9W6EFMBWY5+ea3DAPuMn7/ibgfT/Wclq855JfBTJU9Q8+iwJpG2O9RwKISEvgIpxrIZ8DU7yrNdttVNWHVDVJVVNw/s39R1WvI0C2D0BEWotI9JH3wA+AdTSzv9OgebJYRC7B2RsJBaar6tN+Lum0iMhbwHk4Xd7uAR4D5gKzgGRgF3Clqta8oNwsiMjZwJfAWo6eX34Y5zpBoGzjIJwLiaE4O2WzVPUJEemGswfdAVgJXK+q5f6r9PR5Tw39QlUvDaTt827Le97JMGCmqj4tIjE0o7/ToAkCY4wxtQuWU0PGGGPqYEFgjDFBzoLAGGOCnAWBMcYEOQsCY4wJchYExtQgItXeniSPvBqswzARSfHtMdaYpiBYupgw5mQcVtUh/i7CmMZiRwTG1JO33/nfeccQ+EZEenjndxWRhSKyxvsz2Ts/XkTe8443sFpEzvJ+VKiIvOwdg+Df3qeKjfEbCwJjjtWyxqmhq32WHVDVEcD/4Typjvf9m6o6CJgB/Nk7/8/AIu94A8OA9d75PYEXVLU/UARc4fL2GHNc9mSxMTWISKmqRtUyfwfOQDKZ3g7xdqtqjIjsBRJUtdI7P09VO4pIAZDk232Ct0vtz7wDliAiDwDhqvqU+1tmTO3siMCYk6N1vK9rndr49qtTjV2rM35mQWDMybna5+dS7/slOL1rAlwHfOV9vxD4MXw3AE2bxirSmJNheyLGHKuld9SwIz5V1SO3kEaIyDKcnahrvPPuBqaLyP1AAXCLd/49wEsichvOnv+PgTzXqzfmJNk1AmPqyXuNIE1V9/q7FmMakp0aMsaYIGdHBMYYE+TsiMAYY4KcBYExxgQ5CwJjjAlyFgTGGBPkLAiMMSbI/X/nxGYAzLN/cAAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plot_losses(train_losses_lol, val_losses_lol, test_frequency, num_epochs)\n",
"plot_mAP(train_mAPs_lol, val_mAPs_lol, test_frequency, num_epochs)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"------- Class: aeroplane AP: 0.6468 -------\n",
"------- Class: bicycle AP: 0.4247 -------\n",
"------- Class: bird AP: 0.3491 -------\n",
"------- Class: boat AP: 0.3989 -------\n",
"------- Class: bottle AP: 0.1596 -------\n",
"------- Class: bus AP: 0.2318 -------\n",
"------- Class: car AP: 0.6456 -------\n",
"------- Class: cat AP: 0.3552 -------\n",
"------- Class: chair AP: 0.4179 -------\n",
"------- Class: cow AP: 0.2235 -------\n",
"------- Class: diningtable AP: 0.3586 -------\n",
"------- Class: dog AP: 0.3028 -------\n",
"------- Class: horse AP: 0.6846 -------\n",
"------- Class: motorbike AP: 0.5332 -------\n",
"------- Class: person AP: 0.7901 -------\n",
"------- Class: pottedplant AP: 0.2159 -------\n",
"------- Class: sheep AP: 0.2858 -------\n",
"------- Class: sofa AP: 0.2924 -------\n",
"------- Class: train AP: 0.5996 -------\n",
"------- Class: tvmonitor AP: 0.2998 -------\n",
"mAP: 0.4108\n",
"Avg loss: 0.1801034240768506\n",
"0.41079506406923744\n"
]
}
],
"source": [
"mAP_test, test_loss, test_aps = test_classifier(test_loader, classifier_lol, criterion_lol)\n",
"print(mAP_test)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"torch.save(classifier_lol.state_dict(), './voc_my_best_classifier.pth')\n",
"output_submission_csv('my_solution.csv', test_aps)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 1
}