diff --git a/HW7/Code/FC_NN_practice.ipynb b/HW7/Code/FC_NN_practice.ipynb new file mode 100644 index 0000000..1fcef55 --- /dev/null +++ b/HW7/Code/FC_NN_practice.ipynb @@ -0,0 +1,1021 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "view-in-github" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-pSGPQS8Btmc" + }, + "source": [ + "Всем привет! Сегодня вы впервые попробуете написать свою собственную нейронную сеть и попробовать ее обучить. Мы будем работать с картинками, но пока что не совсем тем способом, которым лучше всего это делать, но должно получиться неплохо.\n", + "\n", + "Будем работать с [датасетом](https://github.com/rois-codh/kmnist) `Kuzushiji-MNIST` (`KMNIST`). Это рукописные буквы, изображения имеют размер (28, 28, 1) и разделены на 10 классов, по ссылке можно прочитать подробнее." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "75HVAP_RFU7r" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import torch\n", + "import matplotlib.pyplot as plt\n", + "from IPython.display import clear_output\n", + "from IPython.display import Image, display" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "46iQ8ixtEruP" + }, + "source": [ + "## Загрузка данных" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "R6h1jVreJlV-" + }, + "source": [ + "Сейчас мы будем использовать встроенные данные, но в реальности приходится писать свой класс для датасета (Dataset), у которого реализовывать несколько обязательных методов (напр, `__getitem__`), но это обсудим уже потом." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "s9L9Z02o_1bK" + }, + "outputs": [], + "source": [ + "import torchvision\n", + "from torchvision.datasets import KMNIST\n", + "\n", + "\n", + "# Превращает картинки в тензоры\n", + "transform = torchvision.transforms.Compose(\n", + " [torchvision.transforms.ToTensor()])\n", + "\n", + "# Загрузим данные (в переменных лежат объекты типа `Dataset`)\n", + "# В аргумент `transform` мы передаем необходимые трансформации (ToTensor)\n", + "trainset = KMNIST(root=\"./KMNIST\", train=True, download=True, transform=transform)\n", + "testset = KMNIST(root=\"./KMNIST\", train=False, download=True, transform=transform)\n", + "\n", + "clear_output()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "V83E2vDrO9CC" + }, + "source": [ + "Определим даталоадеры, они нужны, чтобы реализовывать стохастический градиентный спуск (то есть мы не хотим считывать в оперативную память все картинки сразу, а делать это батчами)." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "oqC8XO8pO8Px" + }, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader\n", + "\n", + "\n", + "# Можно оставить таким\n", + "batch_size = 256\n", + "\n", + "trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2)\n", + "testloader = DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-Ntp5sLoPyGx" + }, + "source": [ + "Подумайте, как может влиять на скорость обучения параметр `batch_size`, почему вы так считаете?\n", + "\n", + "**Ответ:** кажется, чем больше будет batch, тем быстрее будет обучаться модель, но оперативная память может не выдержать" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "t2vmM4KaHvrs" + }, + "source": [ + "Посмотрим на какую-нибудь картинку:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 266 + }, + "id": "N-b-kFCYAoOP", + "outputId": "d077a35b-f874-40c2-e1da-1f461c11172f" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGdCAYAAAD9pm++AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVEElEQVR4nO3de5Ce4/nA8Wtz2pwP1ZId1SSkRARFaBF+EZI6lJQijWokmKbM1ATjMDWlY2qmqKIRNcZIzKiatqNaSjGUNk7pUdE0bVJmMnUIEYc0JBt7//7ouEZsovs8dt+cPp8Zf9j3ud773hX55nnf3TtNpZQSABAR3Tb2BgDYdIgCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCneaBBx6IGTNmxM477xwDBw6M5ubmaGlpiYkTJ8bVV18dr7zyyjrXz5s3L5qammL69OkbZ8Od5Pnnn4+mpqYYPnx4l6/12GOPxZQpU2KHHXaIXr16Rb9+/WL33XePCy64IJYtW9bl67PlEwU+sldffTUmTpwYkyZNinnz5kVra2sccsgh8aUvfSl23XXXeOyxx+Kcc86JHXfcMZ588smNvd3N1vXXXx/jxo2Ln/zkJzF48OD44he/GP/3f/8X//73v+OKK66I3XffPf7+979v7G2ymeuxsTfA5u2NN96IcePGxaJFi2LUqFFx4403xkEHHbTONatXr45bbrklLrnkknjxxRc30k67zvbbbx8LFy6Mnj17dtkaL7/8cpx99tlRSol58+bFKaecko+99dZbcfzxx8f9998fs2bNil//+tddtg+2fKLAR/KNb3wjFi1aFMOHD49HH300Pvaxj7W7prm5Ob72ta/F5MmT4/XXX2/8JrtYz549Y9SoUV26xvz582PNmjUxevTodYIQETFgwIC45JJL4v7774/HH3+8S/fBls/LR9T2r3/9K2677baIiPj+97+/3iC833bbbRe77LJLh577jjvuiNNPPz3GjBkTQ4YMid69e8eIESPi1FNPjUWLFq13ZvXq1XHllVfGPvvsEwMGDIhevXrF0KFDY999943zzz8/XnvttXWu/+c//xmnnnpqjBgxIpqbm6N///4xbNiwOOqoo2Lu3Lkd2mfEh7+n0Flr9O7du0PXffzjH+/wc8J6Fajp2muvLRFRBg8eXNauXVt5fu7cuSUiyimnnNLuse7du5e+ffuWsWPHluOOO64cc8wxZccddywRUfr161ceffTRda5/9913y6GHHloiogwcOLAcccQRZerUqeWwww4rw4YNKxFR/vznP+f1Tz/9dBk4cGCJiLLLLruU4447rpxwwgll//33L/379y977rlnhz+P5557rkREGTZs2Dof78w1li9fXgYNGlQiosybN2+dx956660yadKkEhHl8ssv7/BzwvqIArV99atfLRFRJkyYUGv+w6Jw++23l5UrV67zsba2tjJnzpwSEWW33XYrbW1t+dgjjzxSIqLstdde5c0332z3fL///e/Lq6++mv8+Y8aMEhHlO9/5TrtrV61aVR555JEOfx4bikJnrlFKKXfeeWfp169fiYgyZsyYcsIJJ5QjjzyyDBkypAwZMqRcdtll63xNoA7vKVDbe99iuu2223b6c0+ZMqXdx5qamuLMM8+MW2+9NR5//PFYuHBhjB49OiL++0ZsRMRBBx0UAwYMaDc7duzYdf79veuPPPLIdtf26dMnDj744I/8OXT2GpMnT45HHnkkpkyZEs8880w888wz+dikSZPiwAMPjKampo+2abZ63lNgk7V48eK47rrrYtasWXHaaafF9OnTY/r06fmb7fvfW9h7772je/fucfPNN8ecOXP+53c57bfffhERccYZZ8R9990X77zzTqfvv7PX+OEPfxj7779/bLPNNvHwww/HG2+8EUuXLo05c+bEE088ERMmTIh58+Z1ws7Zqm3sWxU2X1318tHatWvL17/+9dLU1FQiYoP/fPC19R/84Aelubk5Hx82bFj58pe/XG699dayevXqda79z3/+Uw477LC8tmfPnmXs2LHlnHPOKQsWLKj0eWzo5aPOXGP+/PklIkpLS0t5/fXX2z3+4x//uEREGTRoUFmxYkWl54b3EwVqu+aaa7rkjearrrqqREQZOnRoue2228rzzz9f3n777Xx86tSpJSLK3Llz2z3nSy+9VG688cYybdq0fGM6IspOO+1UXnjhhXbXL1iwoFx66aXl8MMPzzeFI6KceeaZHf48NhSFzlzj9NNPLxFRTjvttPU+3traWnr16lUiotx///0dfl74IFGgtsWLF5du3bqViCh33HFH5fkNReGzn/1siYjyi1/8Yr1zY8eO3WAUPmjhwoVl//33LxFRpk2b9qHXtra2lp/+9KelT58+JSLKQw891KHP439FoTPWeO+7i84+++wNXjNkyJASEeX222/v0HPC+nhPgdp22mmnmDp1akREnHvuue1+DuCDli1btsGfMXi/955n2LBh7R579tln4y9/+UuH9zhq1Ki44IILIiL+51yPHj3i+OOPj89//vMdur6Oumtsv/32EREbPCZk0aJFsWLFioiIGDFixEffKFstUeAjmT17dowcOTKee+65GDduXMyfP7/dNWvWrImbb7459tprr1i4cOH/fM5dd901IiLmzJkTbW1t+fEXX3wxpk2bFmvXrm0389BDD8U999wTra2t63y8lBJ33313RKwbmeuvv369gXrppZfiD3/4Q7vr6+jMNY4//viI+O+BeFdeeWWUUvKxZcuWxWmnnRYRETvvvHO777SCSjb2rQqbv5dffrmMHz8+XysfMWJEmTx5cpk6dWqZMGFC6d+/f/5Q2ZNPPplzG3r56IknnsjXx0eOHFlOPPHEcvjhh5c+ffqU3XbbrRx77LHtXj66+uqrc43x48eXk046qRx77LH5g2uDBg1a54fX9txzz9zr0UcfXb7yla+USZMm5cs6EyZMKK2trR36/Df08lFnrlFKKTNnzsyv8ciRI8txxx1XJk6cmO9TDB48eJ2vL9QhCnSae++9t0ybNq2MHDmy9O/fv/Ts2bMMHTq0TJw4sVxzzTVl+fLl61z/YT+89te//rUcc8wxpaWlpfTu3bt8+tOfLueff3558803yymnnNIuCosXLy7f/va3y6GHHlo+9alPld69e5chQ4aUPfbYo1x44YVl6dKl6zz/3XffXc4444yy1157lU984hOlV69e5ZOf/GQZP358ueWWW8qaNWs6/HlvKAqducZ77rzzznL00UeXlpaW0rNnz9KnT58yevTocvbZZ7f7HKGOplLedx8KwFbNewoAJFEAIIkCAEkUAEiiAEASBQBSh/8+Bee081Gdf/75teYuv/zyTt7J+n3rW9+qPHPZZZdVnvFd4GwsHfm1504BgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgCpw39HswPxeL+RI0dWnlmwYEGttYYMGVJ5Zvbs2ZVnZs2aVXmmra2t8gxsLA7EA6ASUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASA7EI/r371955je/+U3lmbFjx1aeiYhYsmRJ5Zm999678sybb75ZeaaR+vXrV3nmwgsvrDyzePHiyjMnn3xy5Zlbb7218kxExC233FJrDgfiAVCRKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIDkllZgzZ07lmTPPPLPyzNq1ayvPRERMmjSp8kydU1wbpbm5udZcndNBn3/++cozl156aeWZZ599tvLMtttuW3kmIuKYY46pPPPggw/WWmtL45RUACoRBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGA1GNjb4DOtc8++1SeOfXUU7tgJ+3VOdAtIuLhhx/u3I1sZBdddFGtuREjRlSemTFjRuWZt99+u/JMnQMI6+wtImLmzJmVZxyI13HuFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkByIt4nq169frbl58+ZVnundu3flmaVLl1ae+eY3v1l5JiKilFJrrhEmT55ceeass86qtdZ+++1XeabO4XZ1vPHGGw1ZJyJi++23b9haWyN3CgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASA7E20R997vfrTU3ZsyYTt7J+l1++eWVZ5YtW9YFO+k8w4cPrzxz8803V56p87WLiPjHP/5Ra64RmpqaGrZWS0tLw9baGrlTACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAciDeJuqAAw5o2Fp33XVX5ZkbbrihC3bSeeoc0DZ79uzKM0uXLq08873vfa/yzKbutddea9haTzzxRMPW2hq5UwAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJJTUhugubm58syoUaO6YCfrd+ONN1aeeffdd7tgJ53noosuqjxz1FFHVZ4566yzKs+0trZWntnULVmypGFrrVq1qmFrbY3cKQCQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIDkQrwEGDx5ceaZv37611nr99dcrzzz88MO11mqUgw8+uPLMxRdfXHlm6dKllWduuummyjNborfeeqtha+26664NW2tr5E4BgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgDJgXgN0NLS0rC1nnrqqcozK1eu7IKdtFfnYMCIiGuvvbbyTLdu1f+8c+6551aeeeeddyrPbIm6d+/esLWWL1/esLW2Ru4UAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQHIjXAD16NO7L/PTTTzdsrarOO++8WnOf+cxnKs9cd911lWd+9rOfVZ7hvxp5MOArr7zSsLW2Ru4UAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGA5JTUBthtt90attbatWsbss4ZZ5xReeaCCy6otdaSJUsqz1x88cW11qKeXr16NWyt3/3udw1ba2vkTgGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAMmBeA2w3377NWytZcuWVZ454IADKs9cddVVlWe6dav3Z5Bzzz238syKFStqrbUpa25urjyz++67V5754x//WHlmu+22qzxTV48eftvqSu4UAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQnCzVAPvuu2/D1jr55JMrz8yaNavyTJ8+fSrPPPDAA5VnIiLuuuuuWnNbmtWrV1ee+dvf/lZ5Zo899qg8s8MOO1Seqevdd99t2FpbI3cKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIDsSraPDgwZVnxowZ0/kb2YDRo0c3ZJ0XX3yx8szUqVNrrdXW1lZrjohVq1ZVnnnqqacqz8yYMaPyTF1+PXQtdwoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEgOxKvoiCOOqDzTp0+fLthJ51m5cmXlmToHoC1fvrzyDJuHAQMGNGytvn37NmytrZE7BQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIDkltaKZM2du7C18qFJK5ZlDDz208syCBQsqz7DlGj58eMPWqvNrnI5zpwBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgLRVH4g3ZsyYyjPjxo3rgp10nra2tsozL7zwQhfshM1V9+7dK8/U+X+prv79+zdsra2ROwUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKAKSt+kC88847r/JMncPCGqmUUnmmtbW1C3bC5qrOoYqrVq3qgp2s38EHH1x5Zvbs2ZVnVq9eXXlmS+BOAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIAqal08AS1pqamrt7LR/K5z32u8sz8+fMrz2zqB+KtWbOm8szQoUMrz6xYsaLyDFuuuXPnVp6ZPn16529kA+65557KM1OmTKk8s3LlysozjdSR3+7dKQCQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIG0xB+LdddddlWe+8IUvdMFONq7W1tbKM3UOxHvttdcqz7Dl2mWXXSrP/OlPf6q1Vt++fWvNVfXLX/6y8szkyZO7YCedx4F4AFQiCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASJvcKamDBg2qNbd8+fLKM927d6+11qaszumldU5JrXMaK7zfjBkzas3ddNNNlWe6dav+59+XXnqp8kxLS0vlmUZySioAlYgCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEDqsbE38EF1Dq6qO7dy5crKM3UO3hs2bFjlmbrqfB0addghvN+9995ba67O/7cDBw6sPFPncMktgTsFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgCkTe5AvDqHXUVE3HDDDZVn7rzzzsozI0aMqDxTZ2913XfffZVn1qxZ0wU7YWuy0047VZ554IEHaq1V53C7n//855VnZs6cWXlmS+BOAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIAqamUUjp0YVNTV+9ls3DIIYdUnnnooYcqz3TwP0s748ePrzzz29/+ttZabJkOPPDAyjO333575Zltttmm8kxEvYPqfvSjH1WeaWtrqzyzqevI7yvuFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgNRjY29gc7NkyZLKM3VOW3zwwQcrz0REzJ8/v9YcW6ahQ4dWnvnVr37VBTtp78QTT6w1d/fdd3fyTng/dwoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEgOxKto2bJllWdmz55deeaKK66oPBNR7/A9tlwzZsyoPDNo0KDKMyeddFLlGQfbbZrcKQCQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIDWVUsrG3gQAmwZ3CgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgCk/wd9xedioWW7YwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.imshow(trainset[0][0].view(28, 28).numpy(), cmap=\"gray\")\n", + "plt.axis(\"off\")\n", + "plt.title(f\"Class is {trainset[0][1]}\", fontsize=16);" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8_MSY231Hzz9" + }, + "source": [ + "### Задание 1. Смотрим на картинки\n", + "\n", + "**2** балла\n", + "\n", + "Нарисуйте на одном графике изображения всех 10 классов:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "j0tNdHM5JS6l" + }, + "source": [ + "⣿⣿⣿⣿⣿⣿⠿⢋⣥⣴⣶⣶⣶⣬⣙⠻⠟⣋⣭⣭⣭⣭⡙⠻⣿⣿⣿⣿⣿\n", + "⣿⣿⣿⣿⡿⢋⣴⣿⣿⠿⢟⣛⣛⣛⠿⢷⡹⣿⣿⣿⣿⣿⣿⣆⠹⣿⣿⣿⣿\n", + "⣿⣿⣿⡿⢁⣾⣿⣿⣴⣿⣿⣿⣿⠿⠿⠷⠥⠱⣶⣶⣶⣶⡶⠮⠤⣌⡙⢿⣿\n", + "⣿⡿⢛⡁⣾⣿⣿⣿⡿⢟⡫⢕⣪⡭⠥⢭⣭⣉⡂⣉⡒⣤⡭⡉⠩⣥⣰⠂⠹\n", + "⡟⢠⣿⣱⣿⣿⣿⣏⣛⢲⣾⣿⠃⠄⠐⠈⣿⣿⣿⣿⣿⣿⠄⠁⠃⢸⣿⣿⡧\n", + "⢠⣿⣿⣿⣿⣿⣿⣿⣿⣇⣊⠙⠳⠤⠤⠾⣟⠛⠍⣹⣛⣛⣢⣀⣠⣛⡯⢉⣰\n", + "⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⡶⠶⢒⣠⣼⣿⣿⣛⠻⠛⢛⣛⠉⣴⣿⣿\n", + "⣿⣿⣿⣿⣿⣿⣿⡿⢛⡛⢿⣿⣿⣶⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⡈⢿⣿\n", + "⣿⣿⣿⣿⣿⣿⣿⠸⣿⡻⢷⣍⣛⠻⠿⠿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠿⢇⡘⣿\n", + "⣿⣿⣿⣿⣿⣿⣿⣷⣝⠻⠶⣬⣍⣛⣛⠓⠶⠶⠶⠤⠬⠭⠤⠶⠶⠞⠛⣡⣿\n", + "⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣶⣬⣭⣍⣙⣛⣛⣛⠛⠛⠛⠿⠿⠿⠛⣠⣿⣿\n", + "⣦⣈⠉⢛⠻⠿⠿⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⠿⠛⣁⣴⣾⣿⣿⣿⣿\n", + "⣿⣿⣿⣶⣮⣭⣁⣒⣒⣒⠂⠠⠬⠭⠭⠭⢀⣀⣠⣄⡘⠿⣿⣿⣿⣿⣿⣿⣿\n", + "⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣦⡈⢿⣿⣿⣿⣿⣿\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "x3j6Bsu9AoT6" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7YAAAHGCAYAAABARxdwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABZM0lEQVR4nO3deZzNdf//8dcYZjf2fc0WZqLFkiWjLpSElKVFtoSypCJJRQpJsreItFm6qCyVhOwkpWRfU5bsDMNsxuf3x/XNL31eb50zizPvcx732+3643p6zee8z/H5nHPenfE8QY7jOAIAAAAAgKVy+HoBAAAAAABkBBtbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYDU2tgAAAAAAq7GxBQAAAABYjY0tAAAAAMBqbGwBAAAAAFZjYwsAAAAAsBobWw+ULVtWOnXq5OtlAD7DNYBAxvmPQMc1gEDG+W+PgN7Y7t27V7p37y7lypWTsLAwiY6Olnr16sm4ceMkMTHR18v7V7t375YHHnhASpYsKREREVK5cmUZOnSoXLhwwddLgyVsvgY6deokQUFBxv8dOnTI10tENmfz+S8ikpycLAMGDJDixYtLeHi41K5dWxYvXuzrZcEiNl8DCQkJMnjwYLnrrrskf/78EhQUJB988IGvlwWL2Hz+/9OwYcMkKChIYmNjfb0Un8rp6wX4yldffSVt2rSR0NBQ6dChg8TGxkpKSoqsXr1a+vfvL1u3bpXJkyf7eplGBw4ckFq1akmePHmkV69ekj9/flm3bp0MHjxYfvrpJ5k3b56vl4hszvZroHv37tKoUaMrMsdxpEePHlK2bFkpUaKEj1YGG9h+/ov87z/uzJkzR/r27SsVK1aUDz74QO6++25ZtmyZ1K9f39fLQzZn+zVw4sQJGTp0qJQuXVqqV68uy5cv9/WSYBHbz/+/O3jwoAwfPlwiIyN9vRSfC8iN7W+//SYPPPCAlClTRr777jspVqzY5T/r2bOn7NmzR7766isfrvDfffzxx3LmzBlZvXq1xMTEiIhIt27d5NKlS/LRRx/J6dOnJV++fD5eJbIrf7gG6tSpI3Xq1LkiW716tVy4cEEefvhhH60KNvCH8/+HH36QWbNmyahRo6Rfv34iIpffnD377LOydu1aH68Q2Zk/XAPFihWTP//8U4oWLSo//vij1KxZ09dLgiX84fz/u379+smtt94qaWlpcuLECV8vx6cC8leRX3/9dUlISJCpU6decTL/pUKFCvLkk08af/7UqVPSr18/ueGGGyQqKkqio6OladOmsmnTJtfshAkTJCYmRiIiIiRfvnxSo0YNmTFjxuU/P3funPTt21fKli0roaGhUrhwYWncuLFs3Ljxqvfh7NmzIiJSpEiRK/JixYpJjhw5JCQk5Ko/j8DmD9eAZsaMGRIUFCQPPfSQ1z+LwOEP5/+cOXMkODhYunXrdjkLCwuTRx99VNatWycHDhzw5KFAgPKHayA0NFSKFi3qxb0G/scfzv+/rFy5UubMmSNjx471aN7fBeQntgsWLJBy5cpJ3bp10/Xz+/btk7lz50qbNm3kuuuuk6NHj8q7774rcXFxsm3bNilevLiIiLz33nvSp08fad26tTz55JOSlJQkv/76q6xfv/7yG+8ePXrInDlzpFevXlK1alU5efKkrF69WrZv3y4333yzcQ0NGzaUkSNHyqOPPiovv/yyFChQQNauXStvv/229OnTh19HwFX5wzXwT6mpqfLf//5X6tatK2XLlk3X/UJg8Ifz/+eff5ZKlSpJdHT0FXmtWrVEROSXX36RUqVKpev+wf/5wzUApJe/nP9paWnSu3dv6dq1q9xwww3pui9+xwkw8fHxjog4LVu29PhnypQp43Ts2PHy/09KSnLS0tKumPntt9+c0NBQZ+jQoZezli1bOjExMVc9dp48eZyePXt6vJa/e+WVV5zw8HBHRC7/b9CgQek6FgKHP10Df7dgwQJHRJy33norw8eC//KX8z8mJsa54447XPnWrVsdEXHeeecdr4+JwOAv18DfbdiwwRERZ9q0aRk6DvyfP53/EydOdPLkyeMcO3bMcRzHiYuL+9fb83cB96vIf/0Kb+7cudN9jNDQUMmR438PXVpampw8eVKioqLk+uuvv+JXB/LmzSsHDx6UDRs2GI+VN29eWb9+vRw+fNjrdZQtW1YaNGggkydPls8++0y6dOkiw4cPl4kTJ3p/pxAw/Oka+LsZM2ZIrly5pG3bthk6Dvybv5z/iYmJEhoa6srDwsIu/zmg8ZdrAEgPfzn/T548KS+99JK8+OKLUqhQofTdET8UcBvbv35t69y5c+k+xqVLl2TMmDFSsWJFCQ0NlYIFC0qhQoXk119/lfj4+MtzAwYMkKioKKlVq5ZUrFhRevbsKWvWrLniWK+//rps2bJFSpUqJbVq1ZIhQ4bIvn37/nUNs2bNkm7dusmUKVPksccek/vuu0+mTp0qHTt2lAEDBsjJkyfTff/g3/zlGvi7hIQEmTdvntx5551SoECBdN8v+D9/Of/Dw8MlOTnZlSclJV3+c0DjL9cAkB7+cv6/8MILkj9/fundu3e674c/CsiNbfHixWXLli3pPsbw4cPl6aeflgYNGsgnn3wiixYtksWLF0tMTIxcunTp8lyVKlVk586dMmvWLKlfv7589tlnUr9+fRk8ePDlmbZt28q+fftkwoQJUrx4cRk1apTExMTIwoULr7qGt956S2666SYpWbLkFXmLFi3kwoUL8vPPP6f7/sG/+cs18Hdz586lDRke8Zfz/69G2H/6K/vr33gB/+Qv1wCQHv5w/u/evVsmT54sffr0kcOHD8v+/ftl//79kpSUJKmpqbJ//345depUuu+f1Xz9u9C+0K1bN0dEnLVr13o0/8/fra9evbpz++23u+ZKlCjhxMXFGY+TnJzsNGvWzAkODnYSExPVmaNHjzolSpRw6tWrd9U1VapUyaldu7Yr//TTTx0RcRYuXHjVn0dg84dr4O/uuusuJyoqyjl//rzHP4PA5Q/nf79+/Zzg4GAnPj7+inzYsGGOiDh//PHHVX8egc0froG/49/Ywhu2n//Lli27ol9H+9+TTz7p0X3zNwH3ia2IyLPPPiuRkZHStWtXOXr0qOvP9+7dK+PGjTP+fHBwsDiOc0U2e/ZsOXTo0BXZP38dOCQkRKpWrSqO40hqaqqkpaVd8SsLIiKFCxeW4sWLq79i9neVKlWSn3/+WXbt2nVFPnPmTMmRI4dUq1btqj+PwOYP18Bfjh8/LkuWLJFWrVpJRESERz+DwOYP53/r1q0lLS1NJk+efDlLTk6WadOmSe3atWlExlX5wzUApJft539sbKx88cUXrv/FxMRI6dKl5YsvvpBHH33U+PP+LCC/7qd8+fIyY8YMadeunVSpUuXyl9qnpKTI2rVrZfbs2dKpUyfjz99zzz0ydOhQ6dy5s9StW1c2b94s06dPl3Llyl0x16RJEylatKjUq1dPihQpItu3b5eJEydKs2bNJHfu3HLmzBkpWbKktG7dWqpXry5RUVGyZMkS2bBhg4wePfqq96F///6ycOFCue2226RXr15SoEAB+fLLL2XhwoXStWtXfg0NV+UP18BfPv30U7l48SK/hgyP+cP5X7t2bWnTpo0MHDhQjh07JhUqVJAPP/xQ9u/fL1OnTs2Mhwl+zB+uARGRiRMnypkzZy4X7yxYsEAOHjwoIiK9e/eWPHnypP9Bgt+y/fwvWLCg3Hvvva78r++y1f4sYPjqo+LsYNeuXc5jjz3mlC1b1gkJCXFy587t1KtXz5kwYYKTlJR0eU6r+X7mmWecYsWKOeHh4U69evWcdevWOXFxcVf8CsK7777rNGjQwClQoIATGhrqlC9f3unfv//lXx1LTk52+vfv71SvXt3JnTu3ExkZ6VSvXt3jrytZv36907RpU6do0aJOrly5nEqVKjnDhg1zUlNTM+Xxgf+z/RpwHMe59dZbncKFCzsXL17M8OOBwGL7+Z+YmOj069fPKVq0qBMaGurUrFnT+eabbzLlsUFgsP0aKFOmjPFXMX/77bfMeIjgx2w///+Jr/txnCDH+cdn6QAAAAAAWCQg/40tAAAAAMB/sLEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDV2NgCAAAAAKyW09PBoKCgrFxHtvbss8+q+ciRIz0+xosvvqjmw4YNU3O+XtjNl49JIJ//yB58/ZzANQBf4zXAe2XLllXzV155xePZe++9V81PnjyZzlX9u6ZNm6r5119/7coOHjyozpYqVSpT1+RrvAbY49Zbb3VlTz31lDq7Z88eNX/ppZfUPC0tLf0Ls5wn1wCf2AIAAAAArMbGFgAAAABgNTa2AAAAAACrsbEFAAAAAFjN4/KoQFChQgU1f+6557w6zoQJE1zZ8OHD1VlflwEAAAD/dPz4cTWPiIhwZfXr11dnJ06cqOYPPvhg+hf2L9q1a+fxbEJCQpatA/YrUaKEx/kPP/ygzpYpU0bNX375ZTW/8847XVnhwoXV2UOHDqn58uXL1Xzx4sVqjv/hE1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNTa2AAAAAACrBTkethcFBQVl9VquqaioKFe2bNkydbZGjRpqvnfvXjW/+eabXdnZs2e9WJ1ZZGSkKzOVW+3Zs0fN27dvr+affPKJK/vwww+9WF3W8mXRlr+d/7CPr4vmsss1EBwcrOaXLl1Sc18/bullerzz5MnjynLkyD7/jfrChQtqnpSUlOFj8xqQecLCwlzZxo0b1dmKFSuqefXq1dV827ZtHq/DVNq5ZcsWj49hKr368ccfPT6GDXz9XGbrNdCnTx8179Spkyvr0qWLOjt9+nQ1N70eVapUyZUdOXJEnd26dauaf/PNN2o+evRoNQ8EnlwD2efVEAAAAACAdGBjCwAAAACwGhtbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYLWcvl6Ar4wcOdKVmdqPL168qOaPPfaYmmdGA3JoaKiaT5061ZXt379fnZ09e7aaDxkyRM3feustV3bw4EF1dunSpWoOAJlFa5x899131dn4+Hg1HzBggJqbntezC1PT8YMPPujK+vXrp86WLl1azXPmzLqX/s6dO6v5Bx98kGW3Ce9pLdUvv/yyOjtr1iw1f+mll9T8gQcecGWm9tjx48eruek90Lhx41yZv7UfI31M55ip1btKlSqu7IsvvlBnCxcurOampmgtNx3j559/VvP3339fzXF1fGILAAAAALAaG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNTa2AAAAAACrBTmO43g0aGj+yu5uueUWNV+9erUrCwsLU2e1JmIRcyuyhw/pVQ0dOlTN77zzTlfWsGFDdTYxMVHNTU1rWpulqVm5bdu2ap6VMuNxTS9bz3/4D1+e/yK+uQYKFSrkyrZt26bO5s+fX83r1Kmj5j/88EP6F5bNhISEqLnpvt9xxx1qrrUoP/LII+rs8ePH1dz07QKHDh1Sc2/wGpC1cuXKpea//vqrmleuXFnN77vvPldWrlw5dfaNN95Q8y1btqh5/fr1XZmpEd3fBOJrQGbQ3jeLiDRo0MCVmZrkT58+rea33367msfGxrqytLQ0dbZv375qPnHiRDUPZJ5cA3xiCwAAAACwGhtbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYDU2tgAAAAAAq+X09QIyS2RkpJp/8MEHaq41IB84cECdff7559U8MxrqWrZsqeZ9+vRR81q1arkyU/uxiTcNgiVKlPDq2ACQWaKjo11Z7ty51dkcOfT/Tmt6jvWnVuSUlBQ1X7VqlZrv3btXzatWrerKHnjgAXX27Nmzam5rcypEUlNT1fzZZ59V8/nz56v522+/7cq0a1lEZPv27WpuapsNlAZkZJ5FixZ5lWu++uorNdfaj00WLlyo5u+9957Hx8C/4xNbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYDU2tgAAAAAAq/lNedRrr72m5t78w+6RI0eq+bFjx9K1pr8rW7asmr///vterWXXrl0ZXos35R7FihXL8O3BP5nOo2HDhrmykiVLqrPJyclqPmnSJDX/5ZdfXFlcXJw6W7x4cTWfOXOmmiP7uXjxoiszlRYVKlRIzRs3bqzmgwYNSv/CLGG6NkylP6Ghoa4sV65c6mylSpXUfOPGjWr+/fffu7Lhw4d7PAvf+fLLL9V88eLFam665jQtWrRQ8xMnTnh8DCCztGvXTs3vvPNONTcV9507d86VmZ7XmjRpoua//fabmm/dutWVZUaZrb/gE1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNTa2AAAAAACrsbEFAAAAAFjNb1qR69at69X8ggULXNk777yTKWvR2mInTJigzh44cEDN33jjjUxZi+bUqVMez9JOCa0pVUTkqaeeUvMBAwa4srS0NHV2xIgRaq61H5usWLHC41mkX7ly5dQ8Ojrale3cuVOdTUxM9Oo2tedSb1rdRUSOHDni1Xx2FxYW5spMrZrTpk1T86ioKDXX/n5WrVrl1TEqVKig5s2bN3dl5cuXV2dvuukmNYdvVKxYUc2vv/76DB+7UaNGar5nz54MHxu4mtKlS7uyJ598Up01vYcxtRHnyOH+3PDll1/2eFZE5LPPPlPz//73vx7PXrp0Sc39GZ/YAgAAAACsxsYWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWI2NLQAAAADAata1IpsaWitXruzVcSZPnuzKTK1n3ho0aJAra9asmTrbp08fNU9NTc2UtWj27t3r8eyFCxeybB2wg6lx9dVXX1VzrYWvdevW6uz8+fPTvzBcU6NGjVLzVq1aubJ58+aps927d1fzY8eOqfn58+ddWUpKimmJqjFjxng1n13ky5dPzV966SVX1rt3b3X27Nmzam76BoCZM2e6MlMrskmlSpXU/P3333dlNWvWVGfbtWvn1W0ic9x7771qrr1fEhEpVKiQmmtNsaY28/Hjx6u5qVl92bJlag6Y5Mypb3X69evnyurUqZMpt6ntVUzfMlK9enU1b9mypZpfd911rszUUD5s2DA1N7U5+wM+sQUAAAAAWI2NLQAAAADAamxsAQAAAABWY2MLAAAAALAaG1sAAAAAgNWsa0XOmzevmkdERKj5mTNn1Hz58uUZXkuDBg3UXGutPHDggDo7ZcqUDK/DW+fOnfN4tkqVKlm4EviK1hJoaqw15VpjrYjIa6+95spq166tzq5cuVLNTdctfKd///5qHhcX58patGihztavX1/Nv/rqKzU/fPiwKytevLhpiao///zTq/lrLUcO/b8vf/DBB2qutSX36NFDnTU9rqYW6sz4ZgBTm63WjL5v3z51dvDgwRleB7z3008/qXnu3Lm9Oo6pAVmTK1cuNdcaukVEOnbs6MoWLVrk8e0h8Jheu5544glX5m1b8IkTJ9Rc2wd89tln6qzpdbFv375qHh4e7spMr4vR0dFqHh8fr+b+gE9sAQAAAABWY2MLAAAAALAaG1sAAAAAgNXY2AIAAAAArGZdeVSxYsW8mt+0aZOaJyQkeHwMU2HVuHHj1FwrA3nmmWfU2aSkJI/XkVmCg4M9nj158mQWrgRZTSuJEhEZM2aMK+vZs6c6ayqJMp27vXv3dmWm69ZUHDJq1Cg1P378uJoj65mKfkaPHu3KBg4cqM5qxUcieiGMt1JTU9XcVM50rcXGxqp5u3bt1HzixIlqrhUfmu57dhIaGurKTCVD5cuXz+rlQGEquZw2bZqa16pVS81DQkJcmenaL1GihJoXKVJEze+77z5XRnkURERKlSql5jfffLOaX7x40ZWZnkuPHj2q5l26dFFzUzGm5osvvlDzBQsWqLlWxFetWjV19tlnn1XzQYMGebg6+2SPV3wAAAAAANKJjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDVrGtFNrW8mmzevDnDt9m/f381v/HGG9Vca7OcM2dOhteRWbxpYqaF1g6tWrVSc1NjX7NmzVzZ1q1b1VlTo3HBggXVfPfu3WquMV1bWqumiEjfvn09PjauDa3Bes2aNersgAED1NzUZqm1ZleuXNnjWRGRtLQ0Nc8MFSpUUPPmzZu7MtN911peRUQefvhhNW/QoIErGzFihDp74cIFNfcF7TlKa0oWETl9+rSam5p1kbWeeOKJLDv2Rx99pOaPPPKImsfFxbky03lhOo9gtzJlyqj5hg0b1DwsLMzjY589e1bNmzRpouambwvIDFprs4jI559/7so6deqkztarV0/NZ8+erea//PKLR2vLzvjEFgAAAABgNTa2AAAAAACrsbEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVrOuFTkmJsareVOrmObxxx9Xc1Ob5d69e9X8pZde8vg2fcHUOKtZtWpVFq4E3jI1RU6ZMkXNTX/XmzZtcmVdu3ZVZ5cuXarmd955p5qvXr3alf3888/qbKVKldT8wQcfVPNZs2a5sjx58qiz1apVU/Ny5cqp+fbt212Z9jiJiPzwww9qnpiYqOb+THuOXblypTprakvOkUP/b6w9evRwZePHj/didSKFChVS8507d3p8jPr166v5iy++qOaLFy92ZZMmTVJnBw8erOaNGzdW86CgIFdmOqdNzcpZSWttFvHuddE0O2HChHStCVnDmybyS5cuqbNPP/20mrds2VLNr7/+eldmek4wvV7CHlrz/MKFC9VZ03O96ZtAvvnmG1fWvXt3dfbo0aOmJWYL8+bNU/MaNWqoedOmTdWcVmQAAAAAAHyMjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDVrGtFrlWrllfzx44dU/O6deu6stGjR6uzpsbOZ555Rs1Pnz7t4eq8Fxoa6spuuOEGdfann35S8yJFinh8ezlzWneKWCd37txq/sknn7iy2267TZ01naMDBw5U8/fee8+VtWjRQp01tQFGRkaqefv27V2Z1mx4NYULF1ZzrXE5ODjYq2N7w3EcNd+xY4eav/XWW2r+7rvvurLU1NT0L8xSWlvq1fICBQpk+DZNrdm33nqrK2vWrJk6W6dOHTXfuHGjmk+ePNmVnT17Vp0tVqyYmrdt21bNtdeudu3aqbOm9lDtucVbpgbS6dOnq3nevHld2f79+9XZjz/+WM1pRc5eTN8YoX3DhOk588cff1TzCxcuqHl0dLQra9OmjTpravpPTk5Wc2Q/pUqVcmWm12XT3+vIkSPVfNy4ca4sK9+/Z5aUlBRXdvLkSXVW2zOImNufx44d68ps+7YHPrEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwmnXNQDVr1vRqXiuyERHp27evKwsPD1dnFy9erOYLFizwai2ZQfvH8du2bVNnq1WrpubaP8Y3MZW6wHshISFqrhULiehlTqZCsBEjRqi5qWzt119/dWXFixdXZ00+++wzNdcKYS5evKjOmh4TE608atasWeqs6TyvUaOGmjdp0sSVBQUFqbNVqlRRc1O5zdNPP+3KnnvuOXV2zpw5ah6IvD0nNabzIyIiwpWZStjGjBmj5lOmTFFzU1GU5s8//1RzrdhERC9EbNq0qTr7/PPPq/ncuXPVPCEhwZXly5dPnTUVPJkKv7QSui5duqiz8fHxao7s5dVXX1XzqVOnurL+/furs08++aSam0pvvJm98cYb1Xz9+vUeHxu+pT0/litXTp01vV81/X1nZVGU9t4hLCxMnS1Tpoyat2rVSs1XrFjhykwFT7ly5VLzI0eOqLk/FKvxiS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDV2NgCAAAAAKyWrVuR8+bN68piY2O9OkbVqlU9njW1Uz744INqfunSJa/WklUuXLig5ps2bVLzzp07e3zs7HIfbXL77ber+eTJk9W8dOnSar5r1y5XZmozfeutt9S8UKFCam5q+9Xs3r1bzU33580333RlTzzxhDprahE2+fzzz13ZO++849UxTPe9cePGriw4OFid1VrVRcyNy9ddd50rmzFjhjp7/PhxNfdnt956q5rXr18/w8eOioryePaXX35Rc1Mr8oEDB9KzpAz54osvXNmAAQPU2fLly6t5wYIF1bxo0aKuzHSNai3iIuZG465du7qyZcuWqbOwm/ZeSmuGFzG/dg0cOFDNtSbtLVu2qLPaayjsou0DTO31L7zwgpp/8803GV5HyZIl1fw///mPmteuXduV1atXT509efKkmpv2O0OHDnVly5cvV2dNbclly5ZV88jISFd27tw5dTa74hNbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYDU2tgAAAAAAq7GxBQAAAABYLVu3Ijdt2tSVhYeHZ8qxExISXJmpLdjUWGar3LlzezwbERGRhSuxh6lJt1OnTq6sX79+6mxSUpKamxodb775Zlf2zDPPqLOm9mpTPnPmTFeWmpqqzhYoUEDNf/zxR49vM1euXOqsSVpamppv3rzZq+NoHMdR82+//dbjYyxcuFDNe/bsqeYTJ050ZabG5WbNmnm8juzK1ESsXS8iIr1791bzSpUqZdaSXLTXgIcfflid9UX7sYl2bcybN0+dNT0Xmf5+Xn31VVd21113qbPr1q1Tc9PfMQ210OzZs0fN33//fTXXWpEPHjyozp4+fTr9C0O2oL3PeOCBB9TZBQsWqLnpNT80NNSVmdreX3vtNTXPnz+/micnJ7uyUqVKqbOmNudJkyap+ezZs11ZgwYN1FnT+06tbVpEJCYmxpV9//336mx2xSe2AAAAAACrsbEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwWrZuRe7evXuGj2FqQ/vPf/7jyn744YcM354NypYt6/Gs6fELNKam2jfffNOVmRp9Te3C3khMTFTzpUuXqvmMGTPUfNGiRa6sYMGC6qzWwCdibiAcNGiQKytUqJA6a2JqUF2zZo1Xx8moChUqqPmAAQPU3NTW6I1bbrklw8e4lrSm+nHjxqmzbdu2VfPz589n6po88emnn7qy7du3X/N1eEtr0OzRo4c6a2q1N13TlStXdmWmJvJWrVqp+dGjR9Uc8EbhwoU9ni1ZsmQWrgS+dPHiRVf23//+16tjmL6B4JVXXnFlDz74oDprOsdM3zyhvQ/8448/1Nljx46pefPmzdVce0xSUlLU2Zw59W2eKY+NjXVltCIDAAAAAHANsbEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVssW5VHaP1YWEalfv36Gj236h92HDx/O8LGzO9M/mDc93pqoqKjMWo7VbrjhBjXPmzdvlt3m3r17Xdljjz2mzq5evVrNU1NTPb49U4HBI488ouaTJ09Wc61Q6/Tp0x6vQ0Rk7Nixam4qSPBGqVKl1PzJJ590ZZ07d1Zn8+fPn+F1mMyfP1/NGzZsmGW3mRGdOnVyZXfccYc6e/DgQTXPyueZH3/8Uc0HDhzoyrJTWV5QUJCaHzlyJMPH1kqiRESOHz/uyiZOnKjOUhJlN+357tlnn1Vnt27dqubTpk1T8y+++MKVJSUlebE6kbvvvtvj2d9++03NTddQdrrOkXlMhUjauS4i0rNnT1dm2jOYctPzca9evVyZ9vxqWoeIyP3336/muXLlcmU7d+5UZ8uVK6fmpv3BmDFjXNm3336rzprKsHyNT2wBAAAAAFZjYwsAAAAAsBobWwAAAACA1djYAgAAAACsxsYWAAAAAGC1IMfDejhTu1xm+PDDD9W8Q4cOGT72xYsX1bxkyZKuzN9aHk1/Z/v27VPzsmXLurIFCxaos23atFHz5ORkzxaXDr5sMtRa6ERE/vOf/7gyUyPsjTfeqOZ79uxR81GjRrmy/fv36wv0AVOT7fTp011ZixYtvDr2q6++qubff/+9Kytfvrw6azpHTX8P17oB/IcfflBz7ZwSETl37lxWLudfmVoUf/75Z1dmamI0NTdWq1ZNzU3Xncb0/GB6PJctW+bxsbNSjhz6f182XTPXXXedK2vXrp06W7t2bTXfvXu3mnfv3t2VadeciEhiYqKaZyVfvgZk5XugrGRq7tcaTU3Nr4sXL1bzm266Sc1LlCjhyn7//Xd11nQuNm7cWM21NXbr1k2d/e6779Tc1m/F8HWbc3a5Bkzr6N+/v5o//vjjaq69ppn2DKZvnjC9/4iIiHBlVapUUWdDQkLU/NChQ2qufTuEqXG5Vq1aam56DLV9kOk+nj9/Xs2zkifXAJ/YAgAAAACsxsYWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWI2NLQAAAADAate0FfnWW29Vc1PbmKmF0xtae5iISNGiRV3Z6dOnM3x7Npg2bZqad+rUyeNjfP3112puaudMSEjw+NgmNGLaIX/+/K7M1M5qauw8cuSImmvnV7NmzdRZrfk8q2kNwSJ68/uUKVPUWVPToK8bMWNjY9V8zpw5rqxy5cpZvRwXU6Pr+PHj1Tw0NNSVVapUSZ01tZGbGs3nzp3rygoWLKjONmrUSM03bdqk5q1atXJlDRs2VGfT0tLU/Pbbb1fz7NS6ruE1wHsNGjRQ8xUrVriysWPHqrNPPfWUmptay7WWYtOxc+bMqeYmHTt2dGUfffSRV8ewla9fA7LLNaB9g4eIuene1Ly9d+9eV9a1a1d11tSWfObMGTUvVKiQK/N2T7Nx40Y1115fHn74YXXW9Fhpr38iIh9//LEry4xvqMkstCIDAAAAAPweG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNTa2AAAAAACreVdHl0GDBg1S88xoPzYxtbhll3Y3X3jttdfUvG3btq4sIiJCnb377rvVfPr06WresmVLD1cH21WvXt2VLVy4MFOObWo9zAzlypVzZdHR0ers1q1b1dzUnOjrNsvMcPDgQTXXniOSkpLU2bCwMK9uUzvO8ePH1VlTQ2vfvn29us3MoD0mBQoUUGdNjZPa87GISL58+VyZqT3z6aefVvPs3n6MzLNt2zY1156TypQp49WxU1NT1XzSpEmubM2aNeqs6ZsU1q1bp+bz58/3cHXwV6Ym7SJFiqi5qRn4P//5jyvLkUP/vM/Uuh8eHq7m2nFM7wNM72ueeOIJNV++fLkrM91307pNvG0pz474xBYAAAAAYDU2tgAAAAAAq7GxBQAAAABYjY0tAAAAAMBqQY6HrSbelC3lyZNHzU+ePKnmWVkederUKTUvWrSoKzMVIQSKzp07u7IpU6aos6Z/YH/kyBE1L1asWPoX9n98WcATyGVjyB58XUBlugbuueceVzZ+/Hh1tnjx4mr+1VdfqblWOHj48GF11vQ60rBhQzUfNmyYK6tSpYo6m5aW5tVtzps3z5V98MEH6mxkZKSaN27cWM21+z9q1Ch19vTp02puK14DMo9WzlSzZk119sYbb1TzLVu2ZOaS8C+y62tAdjF58mQ1f+yxx7LsNk3lTF9//bUr++ijj9TZ3bt3q/nHH3+s5rGxsR6uzsy07latWrmy7FTY5sk1wCe2AAAAAACrsbEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwWpa0IufLl0/NTa3I58+f93i+TJkyHq9DROTMmTNqXqRIEVeWkpLi1bH9jdYUvXPnTnU2Ojpazbdt26bmMTEx6V/Y/6ERE4HMpkZM02tAeHi4mpuepy9cuODxbXqrUKFCrqxjx47qbJ8+fdS8VKlSaq618ZtmtTZ6EXPzZe/evV3ZxYsX1Vl/w2tA5tEax1999VV11tR+3Lx5czXfv39/utcFM5teA3xBe18vYv52D+152vTaZWo03rFjh5pv377dlZla9xctWqTmBQsWVPOQkBA115i+Gcb0mjZz5kxXZmpQ9gVakQEAAAAAfo+NLQAAAADAamxsAQAAAABWY2MLAAAAALAaG1sAAAAAgNWypBU5V65caj5u3Dg1nzt3rppfd911ruydd97xeB0iIp9++qmaP/DAA14dx5+UL19ezRcvXuzKtL8DEZEvvvhCzbt3767mx48f93B1ZjRiIpDRiOk7WmO8iMhXX32l5lq78tmzZ9VZU2Nn27Zt1dzUIB0IeA3IPMWLF3dl69evV2dLliyp5seOHVPzhx9+2JUtWbLEi9VBw2tA+uTIoX+G583jaZoNCwtT8yFDhrgyrdFeRCQiIkLN09LS1Fz7toBdu3aps+3bt1dzU5tzdkcrMgAAAADA77GxBQAAAABYjY0tAAAAAMBqbGwBAAAAAFZjYwsAAAAAsFqWtCJnlttvv92Vfffdd+qs6W40bNhQzVeuXJnuddmiXr16aj5r1iw1L1CggCsztRxPnz5dzS9duuTh6rxHIyYCGY2Y2Y+pzVJrrQwJCVFng4OD1TwxMTH9C/NTvAZkrapVq6r58uXL1bxQoUJqnpqa6sqmTp2qzg4cOFDNA7n924TXAN+JjIxU80mTJqn5I4884spM7cymc/3bb79V89mzZ7syU0O/v72O0IoMAAAAAPB7bGwBAAAAAFZjYwsAAAAAsBobWwAAAACA1bJ1eVTp0qVd2W+//abOLl26VM3vuusuNc/KkqNrrWjRomq+Y8cOr47Tvn17V/bll1+ma01ZgeIQBDKKQxDoeA3wjWrVqqn5qlWr1Dw6OtrjYw8ePFjNhw4dquZhYWGurFu3bursihUr1HzTpk0eri574TUg61WuXFnN33nnHTWPi4vz+Nh79+5V8x49eqj5kiVLPD52oKA8CgAAAADg99jYAgAAAACsxsYWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWC2nrxdwNceOHXNlEyZMUGdff/11Nfen9mOTzp07q3mePHnU/KGHHlLz7NSADAAAfO/XX39V86ZNm6r58OHDXVm5cuXU2a+//lrNS5YsqeaLFy92ZaYm29tvv13NARH9/DC1H1eqVEnNU1NT1Xzq1KmuzNQAru11kH58YgsAAAAAsBobWwAAAACA1djYAgAAAACsxsYWAAAAAGA1NrYAAAAAAKsFOY7j+HoRAAAAAACkF5/YAgAAAACsxsYWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWI2NLQAAAADAamxsAQAAAABWY2MLAAAAALAaG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNTa2AAAAAACrsbEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYDU2tgAAAAAAq7GxBQAAAABYjY0tAAAAAMBqbGwBAAAAAFZjYwsAAAAAsBobWwAAAACA1djYAgAAAACsxsYWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWI2NLQAAAADAamxsAQAAAABWY2MLAAAAALAaG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNTa2AAAAAACrsbEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYDU2tgAAAAAAq7GxBQAAAABYjY0tAAAAAMBqbGwBAAAAAFZjYwsAAAAAsBobWwAAAACA1djYAgAAAACsxsYWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWI2NLQAAAADAamxsAQAAAABWY2MLAAAAALAaG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNTa2AAAAAACrsbEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYDU2tgAAAAAAq7GxBQAAAABYjY0tAAAAAMBqbGw9ULZsWenUqZOvlwH4DNcAAhnnPwId1wACGee/PQJ6Y7t3717p3r27lCtXTsLCwiQ6Olrq1asn48aNk8TERF8v76o2bNggvXr1kpiYGImMjJTSpUtL27ZtZdeuXb5eGixi8zWwfPlyCQoKUv/3/fff+3p5sIDN5/9fNm7cKC1atJD8+fNLRESExMbGyvjx4329LFjC9mvgp59+krvuukuio6Mld+7c0qRJE/nll198vSxYwubzf+vWrdKmTRspV66cRERESMGCBaVBgwayYMECXy/Np3L6egG+8tVXX0mbNm0kNDRUOnToILGxsZKSkiKrV6+W/v37y9atW2Xy5Mm+XqbRyJEjZc2aNdKmTRupVq2aHDlyRCZOnCg333yzfP/99xIbG+vrJSKbs/0a+EufPn2kZs2aV2QVKlTw0WpgC384/7/99ltp3ry53HTTTfLiiy9KVFSU7N27Vw4ePOjrpcECtl8DGzdulPr160upUqVk8ODBcunSJXnrrbckLi5OfvjhB7n++ut9vURkY7af/7///rucO3dOOnbsKMWLF5cLFy7IZ599Ji1atJB3331XunXr5usl+oYTgPbt2+dERUU5lStXdg4fPuz68927dztjx469/P/LlCnjdOzY8Rqu8N+tWbPGSU5OviLbtWuXExoa6jz88MM+WhVs4Q/XwLJlyxwRcWbPnu3rpcAy/nD+x8fHO0WKFHFatWrlpKWl+Xo5sIw/XAN33323ky9fPufEiROXs8OHDztRUVHOfffd58OVIbvzh/Nfc/HiRad69erO9ddf7+ul+ExA/iry66+/LgkJCTJ16lQpVqyY688rVKggTz75pPHnT506Jf369ZMbbrhBoqKiJDo6Wpo2bSqbNm1yzU6YMEFiYmIkIiJC8uXLJzVq1JAZM2Zc/vNz585J3759pWzZshIaGiqFCxeWxo0by8aNG696H+rWrSshISFXZBUrVpSYmBjZvn37vz0ECHD+cA383blz5+TixYsezyOw+cP5P2PGDDl69KgMGzZMcuTIIefPn5dLly558SggkPnDNbBq1Spp1KiRFChQ4HJWrFgxiYuLky+//FISEhI8eSgQgPzh/NcEBwdLqVKl5MyZM17/rL8IyF9FXrBggZQrV07q1q2brp/ft2+fzJ07V9q0aSPXXXedHD16VN59912Ji4uTbdu2SfHixUVE5L333pM+ffpI69at5cknn5SkpCT59ddfZf369fLQQw+JiEiPHj1kzpw50qtXL6lataqcPHlSVq9eLdu3b5ebb77Zq3U5jiNHjx6VmJiYdN0vBA5/ugY6d+4sCQkJEhwcLLfddpuMGjVKatSoka77hcDgD+f/kiVLJDo6Wg4dOiT33nuv7Nq1SyIjI+WRRx6RMWPGSFhYWLruGwKDP1wDycnJEh4e7sojIiIkJSVFtmzZIrfeemu67h/8mz+c/385f/68JCYmSnx8vMyfP18WLlwo7dq1S9f98gu+/sj4WouPj3dExGnZsqXHP/PPX0FISkpy/erXb7/95oSGhjpDhw69nLVs2dKJiYm56rHz5Mnj9OzZ0+O1XM3HH3/siIgzderUTDke/JO/XANr1qxx7r//fmfq1KnOvHnznBEjRjgFChRwwsLCnI0bN3p9PAQGfzn/q1Wr5kRERDgRERFO7969nc8++8zp3bu3IyLOAw884PXxEDj85Rq44YYbnEqVKjkXL168nCUnJzulS5d2RMSZM2eO18eE//OX8/8v3bt3d0TEEREnR44cTuvWrZ1Tp06l+3i2C7hfRT579qyIiOTOnTvdxwgNDZUcOf730KWlpcnJkyclKipKrr/++it+dSBv3rxy8OBB2bBhg/FYefPmlfXr18vhw4fTvR4RkR07dkjPnj2lTp060rFjxwwdC/7NX66BunXrypw5c6RLly7SokULee655+T777+XoKAgGThwYPruGPyev5z/CQkJcuHCBenQoYOMHz9e7rvvPhk/frx0795dZs2aJbt3707fnYPf85dr4IknnpBdu3bJo48+Ktu2bZMtW7ZIhw4d5M8//xQRyfattvANfzn//9K3b19ZvHixfPjhh9K0aVNJS0uTlJSUdB3LHwTcxjY6OlpE/vc77el16dIlGTNmjFSsWFFCQ0OlYMGCUqhQIfn1118lPj7+8tyAAQMkKipKatWqJRUrVpSePXvKmjVrrjjW66+/Llu2bJFSpUpJrVq1ZMiQIbJv3z6v1nPkyBFp1qyZ5MmTR+bMmSPBwcHpvm/wf/54DfylQoUK0rJlS1m2bJmkpaWl+/7Bf/nL+f/Xr2A++OCDV+R//XrbunXr0n3/4N/85Rro0aOHPP/88zJjxgyJiYmRG264Qfbu3SvPPvusiIhERUWl+/7Bf/nL+f+XypUrS6NGjaRDhw6X/2158+bNxXGcdN8/q/n6I2NfKF68uFO+fHmP5//5KwivvPKKIyJOly5dnJkzZzqLFi1yFi9e7MTExDhxcXFX/GxCQoIza9Ysp1OnTk6RIkUcEXFeeumlK2YOHz7sTJo0yWnZsqUTERHhhIWFOV9//bVHaztz5oxz4403Ovnz53e2bt3q8X1CYPOna+Cf+vfv74iIEx8fn66fh//zh/O/cePGjog4O3bsuCLfvn27IyJXNHoC/+QP18BfTp065axatcr59ddfHcdxnIEDBzoiwnsiGPnT+f9P7777rvraECgCcmPbrVs3R0SctWvXejT/zxO6evXqzu233+6aK1GihOuE/rvk5GSnWbNmTnBwsJOYmKjOHD161ClRooRTr169f11XYmKic9tttzkREREe3xfAcfznGtDcf//9TlhYGF+BAiN/OP+fe+45R0ScpUuXXpEvXbrUERFn+vTpV/15BDZ/uAZMatas6ZQsWZLXABj58/k/duxYR0Sc9evXp+vnbRdwv4osIvLss89KZGSkdO3aVY4ePer6871798q4ceOMPx8cHOz6iH/27Nly6NChK7KTJ09e8f9DQkKkatWq4jiOpKamSlpa2hW/siAiUrhwYSlevLgkJydf9T6kpaVJu3btZN26dTJ79mypU6fOVeeBv/OHa+D48eOubNOmTTJ//nxp0qTJ5X//AvyTP5z/bdu2FRGRqVOnXpFPmTJFcubMKQ0bNrzqzyOw+cM1oPn0009lw4YN0rdvX14DYOQP5/+xY8dcWWpqqnz00UcSHh4uVatWverP+6uA/Lqf8uXLy4wZM6Rdu3ZSpUoV6dChg8TGxkpKSoqsXbtWZs+eLZ06dTL+/D333CNDhw6Vzp07S926dWXz5s0yffp0KVeu3BVzTZo0kaJFi0q9evWkSJEisn37dpk4caI0a9ZMcufOLWfOnJGSJUtK69atpXr16hIVFSVLliyRDRs2yOjRo696H5555hmZP3++NG/eXE6dOiWffPLJFX/evn37dD8+8H/+cA20a9dOwsPDpW7dulK4cGHZtm2bTJ48WSIiIuS1117LjIcJfsofzv+bbrpJunTpIu+//75cvHhR4uLiZPny5TJ79mwZOHDg5a+bADT+cA2sXLlShg4dKk2aNJECBQrI999/L9OmTZO77rrrqt9BCvjD+d+9e3c5e/asNGjQQEqUKCFHjhyR6dOny44dO2T06NGB+2/MffVRcXawa9cu57HHHnPKli3rhISEOLlz53bq1avnTJgwwUlKSro8p9V8P/PMM06xYsWc8PBwp169es66deucuLi4K34F4d1333UaNGjgFChQwAkNDXXKly/v9O/f//K//UtOTnb69+/vVK9e3cmdO7cTGRnpVK9e3Xnrrbf+de1xcXGX6721/wGesPkaGDdunFOrVi0nf/78Ts6cOZ1ixYo57du3d3bv3p1pjw/8m83nv+M4TkpKijNkyBCnTJkyTq5cuZwKFSo4Y8aMyYyHBgHC5mtgz549TpMmTZyCBQs6oaGhTuXKlZ0RI0Y4ycnJmfb4wL/ZfP7PnDnTadSokVOkSBEnZ86cTr58+ZxGjRo58+bNy7THx0ZBjhOotVkAAAAAAH/AP0AAAAAAAFiNjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYLWcng4GBQVl5TrgoejoaFdWu3ZtdXbJkiVqXqBAATVv1aqVKzty5Ig6u2DBAtMSs4wvv3KZ8x++5uuvHOca8FyZMmVc2e+//+7VMXLlyqXmXbt2dWURERHqbN26ddX88OHDav7aa6+5skOHDpmWeM3xGuAbpnOxffv2aj5y5EhXFhwcrM7269dPzWfMmKHmycnJah4IeA1AoPPkGuATWwAAAACA1djYAgAAAACsxsYWAAAAAGA1NrYAAAAAAKsFOR7+a3Rf/KNx7TZDQkLU2UApFHjjjTdcWa9evdTZSpUqqfmBAwfUvHz58q5s37596uylS5dMS8wyFIcgkFEcEli8KeypVauWOtu6dWs1P3nypJrnzZvXlcXExHh1jKzEa0DWypcvn5pPmjRJzevXr6/mv/zyiysbOnSoOvvjjz96trhMZCrQrFOnjiuLjIxUZ48dO6bmq1evVvPU1FQPV2fGawACHeVRAAAAAAC/x8YWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWC2nrxdwNdWqVXNlgwYNUme/++47NTcVE2ilSImJieqs6R/9mwqUvClWypFD/28LppIsrawhNDRUnX377bfVfMqUKWr+888/uzJflEQBQKAzve5MmzbNlZkKbnr06KHmBQsW9HgdpkIhX5RHIfPccsstruyzzz5TZ0uVKqXms2fPVnPtvDtz5ozni8skefLkUfP//ve/ar5mzRpXNmDAAHXWVO62detWNW/VqpUr27NnjzoLIP34xBYAAAAAYDU2tgAAAAAAq7GxBQAAAABYjY0tAAAAAMBqbGwBAAAAAFbL1q3ImzZtcmWPPPKIOrtkyRI1HzFihJovXLjQlZUoUUKd3b9/v5oXLlxYzSMiIlyZqf3OdAxTa2WBAgXUXNO0aVM1b9KkiZprrchaC7OISEpKisfrAABkHVNDKwJHWFiYmsfExKj5jBkzXFl0dLQ62717dzXXGrpFRNLS0tTcGxUqVFDz33//3ZWZ3rvNmjVLzQ8cOKDmr7zyiiszvRczvReNjY1V85UrV7oy7Zs/REROnDih5gD+HZ/YAgAAAACsxsYWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWI2NLQAAAADAatm6FVmTnJys5m3btlXzxx9/XM2joqJc2R9//KHObt++Xc07duyo5hcuXHBlhQoVUmdNbZam+6k1Bc6dO1edPXLkiJqb7ufp06ddWWa0GwImQUFBau44zjVeCQBkf6GhoWr+3nvvqfmdd96p5mvWrHFl/fr1U2f37t3r4eq8lyOH/vmKqXW4QYMGrqxcuXLq7OHDh9V89erVap6amurKevbsqc4mJSWp+VNPPaXmxYoVc2XPPfecOjto0CA1B/Dv+MQWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWI2NLQAAAADAamxsAQAAAABWC3I8rB81tZcGMm8aXYODg9VZU+twy5Yt1fzjjz92ZSVKlFBnz507p+a28mVTLud/xoSHh6t5165d1fy6665Tc63l8s8//1RnExIS1Hzt2rVqfuLECVeWndqZfb0WroHs64MPPlBzU3N/fHy8mi9btsyVde7cWZ09c+aMR2vLTIH2GqDd5k033aTOTp06Vc21bzsQEWnWrJkrS0xM9GJ1Weuee+5R85kzZ7oyU1P0b7/9puZ79uxRc+0xMSlQoICaf/TRR2retGlTV3bp0iV1tnv37mo+ZcoUD1eXNXgNsI+pddz0TS0a07e0ZNfXAD6xBQAAAABYjY0tAAAAAMBqbGwBAAAAAFZjYwsAAAAAsFpOXy/AZt4UWZhKokxy5tT/an7//XdXdv78ea+OjcyjlYJVrFhRnd2/f7+aJyUlZeaSsiVTKcmCBQvUfNasWWpeq1YtV2YqtDAVc+zbt0/NtWKO9957T509deqUmgO+YCqy2b17t5ofPXpUzYsVK+bK8ubNq876ojgk0OTJk8eVjRkzRp0tW7asmhcpUkTNo6KiXJkvyqNMz9+m19GwsDBXZnovZnpvNHbsWM8WdxUnT55U8wEDBqh5hQoVXJmp2CsQ3hPg/zNdA1WrVlXzu+66y5XlypVLnW3evLma161b18PVmc/TYcOGqfno0aM9PnZW4BNbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYDU2tgAAAAAAq7GxBQAAAABYLcjxsNrX1NqFrNGxY0c1r1Gjhivr3bt3Vi8nW/CmhTqzmc5/rYnu5ZdfVmerVKmi5q+99pqar1+/3sPV+Z+iRYuq+ciRI13ZI488os56+5ylnV87duxQZ++55x41NzUuZwZfnv8ivAZkF1pT7pYtW9RZUzP4xYsX1bxcuXKuzNSU+9Zbb6m5qbX30KFDau6N7PgakJW01v1nn31WnTXl0dHRaj506FBXZnrtygzafRER6datm5q/8MILar5582ZXpp23IiIbN25U806dOql5VrYRN2zY0JUdPnxYnd21a5ea8xpgN9P7mh49eqi5qWE7JCTEleXIce0/pzS9vsydO1fNO3To4Mq8/VYXT64BPrEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDVaEXOpkytyH/88YcrW7ZsWVYvJ1uwpREzZ86caj569Gg179y5s5r//vvvruybb75RZ4cMGaLm3jbOZXcRERGuzNTkaWoLDw0NzfA61q1bp+b333+/mv/5558Zvk0aMe1Rvnx5V1arVi11Vms5FjE3HZ84ccKVLVq0SJ395JNP1Pz5559Xc01aWpqamxpdV6xYoeba85ypndnElteArFS8eHE1P3DggJqbHuM5c+a4MlPDvKn91BumluOXXnpJzT/++GM17969uyu788471dnPP/9czbt06aLm06dPV/PsgteA7Kd+/fpq3r9/f1cWFxenzppeA7Zt26bm8+fPd2Vr1qxRZzds2KDmt912m5prr12ma1R7P3Y1q1atcmWtW7dWZ48dO6bmtCIDAAAAAPweG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNTa2AAAAAACr0YqcTdWoUUPNtSazMWPGZPVysgXbGzFNbcmmv7/HH3/c43UMHDhQzd944w01z4yWy+wiPDxczefOnavmTZo0ybK1mNpp27Rp48rOnTvn1bH9oRGzR48eav7ll1+q+cGDBzN8m9548MEH1fzFF19U85MnT6r5zTff7MpM52lKSoqaHz9+XM3Xr1/vyu699151VmtWFxG57rrr1Dwz/o5NTbSdOnVyZTZdA754DxQcHOzKTH93GzduVPN58+apea9evVxZfHy8F6sz05piTY3DpoZ57XwREUlKSnJlUVFR6uyMGTPU3NQsrb2/SkxMVGd9wR9eA7K7fPnyqfkXX3yh5vXq1VNz7do1nUum5+/vvvtOzU1N9VmlRIkSav7BBx+oecOGDdVcew/cr18/ddb0LSK0IgMAAAAA/B4bWwAAAACA1djYAgAAAACsxsYWAAAAAGA1yqOyqSFDhqj5+fPnXdmoUaOyeDXZg78Wh+TIof/3pTfffNOV9e7dW529ePGimo8YMULNhw8f7spMJTa2qlSpkpqbiooqVqyY4ds0naNfffWVK+vSpYs6ayoNsqk4JFeuXGquFR+JiLz++utqPmvWLI9vMzPkyZNHzd9++201N5VkrFq1ypUVLFhQnd2/f7+amwpuSpYs6cpiY2PV2fnz56u56TlnyZIlrqxZs2bqrKkMz1S0tWbNGjX3hr++BpjUrVvXlZkKkUzPG+3bt1fznTt3pn9h/8d0jn766aeuLCEhQZ1t0aKFmqempqZ/Yf+ncuXKar5p0yY1b9y4sStbuXJlhteRWWx6DfCFkJAQNTcVRrZu3dqVNW/eXJ3Nnz+/V2vRStsGDBigzmbGtegLkZGRam661jV79uxR85o1a6r56dOn//WYfGILAAAAALAaG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNTa2AAAAAACr0YrsY8HBwWo+Z84cNV+xYoUrGzt2bGYuKdsKtEbMQoUKuTJTQ6Op/dHUdNyyZUtX9s0333ixOns98sgjav7uu++6svDw8Ey5Te3cNTUBP/fccx4f41ry5hq4/vrr1dx0/i5atEjNO3To4PFt+kJUVJSaa42uoaGh6uzZs2fVfODAgWo+dOhQV7Z9+3Z1tnbt2mqeN29eNdeauidPnqzOmlp4s5K/vgbcfPPNaq499vHx8ersM888o+a//PJLutf1F9N9nz59upqXKlXKld1///3q7LFjx9K/sH9hOs9NTazDhg1zZWPGjMnMJWWITa8BmUVr2Dc1fXfr1k3Nb731Vo9v7+TJk2q+bNkyNV++fLmav/POO64sLS3N43XYwPTtB9u2bVPzChUqeHzsV199Vc1feOGFf/1ZPrEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDVcvp6AYEiLCxMzatVq6bmVapUUfP9+/e7soiICHU2MTFRzb1p1jO1Nvtbu1t2pLWOTp06VZ0dNWqUmoeEhKj5oEGDXNmSJUvU2YsXL5qWaKVZs2apeaVKlVyZJw18ntDaJO+991511tSWbJPcuXOrecGCBdU8X758WbmcLJOQkODxbHJyslfHNj2GOXO6X7arVq2qzk6ZMkXNX3zxRTXXnl9OnTplWiK8VKBAATX/8MMP1VxrZW/YsKE668256K2ePXuqeaNGjdRcawX3xXlUvnx5NTe9Z2rTpo0re++999TZrHy8s6vY2Fg11x4LU2O86f3ngAED1FxrNDa165pam03vYWbOnOnKnn76aXX2xIkTah7ITI/rl19+qeZ9+/b1+NiNGzdOz5JEhE9sAQAAAACWY2MLAAAAALAaG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNVqRs4DWzNavXz919qmnnlLz/Pnzq/mKFStcmam5ODPkyKH/tw9akX1Da0RNj1q1arky0zl37NixTLnN7EJr7BTRm6Xj4uLU2dtuuy3D69BamEVEOnfunOFj+1pkZKSam55PTG2boaGhrszbdmFfiImJcWWmNu59+/apuakxX2NqA33ooYfU/L777lPzuXPnurLRo0ersz/++KNniwtA119/vZqbWqpN5/+lS5dcmam1dfjw4WruTat9jRo11LxXr15q3r17dzUfMmSIK6tXr546q72nySyPPfaYmpsabkuWLOnKTNfh2rVr078wSw0ePFjNmzVr5spM70tNz1Wm9zZai7LpGEuXLlXziRMnqrn2fAfPmRquo6OjM3zsIkWKpPtn+cQWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWI2NLQAAAADAapRHZQGtbOCJJ55QZ8PDw7069h9//OHKzp0759UxvGEq2kHWK1y4sCvr2rVrphz7119/dWWmQo1AcfbsWVdmKk35/vvv1dyb69lUgGF6rrBJ7ty5vZrXSqJEzGVT2Z127ZoekxYtWmT49h599FE1j4qKUvMRI0aoedu2bV3ZPffco86aro2PP/5YzbUiJH9letwrVqyo5gcOHFDzokWLujJTeZSp0HHYsGFqHhIS4srefvttdfa9995T82+++UbNtdepJk2aqLPelkdpJUO33HKLOtuoUSM1HzNmjMe3Z7pWTMf25/dMkyZNUvOGDRu6srx586qz3pREiYgkJCS4MtP7oNmzZ6s5sobpPUx8fLyae1ME9vLLL6v5tGnT/nVddr5rAAAAAADg/7CxBQAAAABYjY0tAAAAAMBqbGwBAAAAAFZjYwsAAAAAsBqtyBkQHR2t5hcvXnRlW7duVWf379+v5qbWt1y5crkyU3NoILVQ2szUCqe1X5YvX96rY5uaMrXGOc4XN609WsTcvtihQ4cM32bBggUzfAxfCwsL82redJ4mJydnxnKuuWXLlrmy7777Tp1t06aNmpuadbXXF9M588Ybb6j5+++/r+ZaW2yFChXU2fHjx6u5qRFz7ty5au6PfvrpJzVv3769mmvtxyL6a0DlypXV2TJlyqi5qYV28ODBrsx03U6cOFHNTdfn5s2bXVmXLl3UWdO5uG/fPjXX1l23bl119s8//1Rz0/0pVKiQK3v88cfVWVMTs6kx3x8sX75czbXH7cYbb1RnTS3wpvfTWiP3zp079QXimtL+3kVEnnrqKTXXXudTUlLU2ZUrV6Z7XXxiCwAAAACwGhtbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYDU2tgAAAAAAq9GKnAGmNq/t27e7spMnT6qztWvX9uo24+LiXJmpyTAxMVHNHcfx6jaRtdq1a6fmvXv3zvCx165dq+ZLly7N8LED2euvv67mpobyiIgIV7ZhwwZ19ptvvlHzn3/+2cPV+V7u3Lm9mjc1cmeXpm7Tc6yp+TMyMtKVmRokw8PDvVrLiRMnXJmp/djkwoULav7MM8+4svXr16uzphZTUzP4vHnzXFmgvRYtWbJEzU3nhtY8bTpfzp8/r+amBmDtvUedOnXUWW/byadNm+bKTK3NI0aMUPPg4GA1v/fee12ZqVXddH9SU1PV/MiRI67sjz/+UGdfffVVNW/ZsqWam/5+/NUvv/ziVQ77ePseVbumTe36plZ0T/CJLQAAAADAamxsAQAAAABWY2MLAAAAALAaG1sAAAAAgNXY2AIAAAAArEYrcgYkJSV5PPv555+ruakR10Rr6LvhhhvUWVObJXzjlltuUXNTw67WpOstU+OcqTEbntm6dauaP/LII9d4JdlTzpzevbSEhISouXYNmBp9vaXd5kMPPaTO9u3bV82rV6+u5j/99JMrM13/pnZ9Uyvs5s2b1TwzJCQkuDLT341Jq1at1Lx06dKu7Pfff/fq2P7K1KSbJ08eV3b27Fl1tnv37mo+f/58NW/fvr0r0/7+02Pnzp2uLCoqSp2966671DwoKEjNtev/gw8+UGe9beDVWtgHDx6szq5atUrNtdZmEZHp06d7tRYgK5muL62pvnLlyupsz549vbrNw4cPu7Lhw4d7dQxP8IktAAAAAMBqbGwBAAAAAFZjYwsAAAAAsBobWwAAAACA1SiPuka+/vprNd+yZYuax8bGqvn+/ftdmalkxFQ+lFnlK9CVL19ezWfPnq3mpUqVyvBtrl69Ws0XLFiQ4WMD3kpOTvZqvmjRompetWpVV/bjjz+ma03/FBcX58qmTZuWKcc2FUVpTGVQpmNoj63pNSAtLc3jdZgcO3ZMzStUqODVcZo0aeLK3nvvvXStyVY5cuifJTz88MNqrr0/MJW+FCxYUM0//PBDNd+2bZuaZxVTkWHZsmXVPDIyUs0fffRRV7ZixQp1ViuD8pZWBCcicvr0aTV/88031VwrsjKVEAJZTSuJEtGfXz755BN11lQIFx8fr+b9+vVzZaYyvIzgE1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNTa2AAAAAACrsbEFAAAAAFiNVuRrJCEhQc1feOEFNZ87d66aa826pna+O++8U80XLVqk5rQle69IkSKubMqUKeqsqf3RG0ePHlXzvn37qrm37bTXmqnNNVeuXGqekpLiyjKj+RKZy/Qc462sbEVu2rRpho+xZ88eNfemMVi7j1dz9913uzKtcVhEZOHChV4dW7Nq1So1r1u3boaPHWhuuukmNa9Vq5aaa233Z86cUWf37t2r5qZr0dSKmlWWLl2q5j179lRz7RsgRET++OOPzFqSR5KSktS8S5cuam5qkG3evLkroxUZvqK9dxURGTlypCu7+eab1dn169eredeuXdX8Wp3vfGILAAAAALAaG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNTa2AAAAAACr0Yr8Nzly6Pv8rGxdNbXcetN8mJiYqM5u2LBBze+77z41nzdvnis7d+6cOhtoTC2nWgNyXFycOpuWlqbmpvNOM3r0aDX/6aefPD6GL+TNm1fN33zzTTU3PYbbtm1zZbt371ZnZ8yYoea//PKLml+8eFHN4b3U1NRMOY63jcHe0Fpan3rqKa+OYXpeOHv2rCuLjIxUZx944AE1//TTT9U8LCzMlZmuo2+++UbNvWnEjY6OVnNTi35ISIiab9++3ePb9FfDhg1T84iICDWvWLGiKzO9N3jsscfUPLu0xpvWsXLlymu8ksyxZMkSNT9+/Lia33777a7MdN1q7f9Aepjee40aNUrNGzdu7Mqef/55ddb0fjSzXv/Ti09sAQAAAABWY2MLAAAAALAaG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgNVqR/yZnTv3hyMqGujp16qj5xo0b1dzUgKw5fPiwmi9fvlzNg4KCPMpEvGvV9AcDBw5Uc1N7ryY4ONir29RaqidPnuzVMXxBO2cef/xxdbZz585eHbtcuXIez3bv3l3NTW3hjz76qCvTWsiR+UyN4fnz58+y26xUqZIrO3bsmDprei698cYb1fzgwYOuzNSgvGDBAjVv3ry5mi9evNjjY8fExKj5li1b1Pz+++93Zdp1ISJy5MgRNV+3bp2aZ/fm9szUvn17NdcaR69Ga9du0qSJx7PIOufPn1dz0/mvXc+ma/yzzz5L/8IQkEzvL1esWKHmpueLW2+91ZX9/vvv6V+YD/CJLQAAAADAamxsAQAAAABWY2MLAAAAALAaG1sAAAAAgNUCtjyqWLFirqxq1arq7NKlS7NsHTVr1lTz9evXZ/jYly5dUnOt2EREJCQkxJXlzZtXnT19+nS612WjCRMmqHmDBg1cmanIxWTRokVqrhWQJCQkeHVsX6hWrZor69OnzzVfR0REhJqbCr9Gjhzpytq2bavOmq4t/I+pWGXnzp1qbiqyaNmypSt7/vnn1dkTJ054uLr/efvtt12Zqawqd+7can78+HE110qCTOeMdh9FzMV9GlPxoenYPXv2VPN27dq5Mu11QUQkT548am4qz/Gm+NAmuXLlcmWmczRHDv2zBNO5ob02eHuee8N0nteuXVvNly1bpuamMrhA8OGHH6q59loyfPhwdXbu3LmZuSQEAFOh6+jRo9V85syZap6ampppa/IVPrEFAAAAAFiNjS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDVghxTldY/B71oaLRB2bJlXVmVKlXU2YULF2b49kxtiPv371fzSZMmqbnW3BooPDxVs4Tp/Nf+Xr29Vkz3y9bmXe3+m87/7C47tXv68vwXyZzXgJdfflnNX3rpJY+P0apVKzX3RZNoZGSkmv/222+u7OjRo+psxYoVM7yO0NDQDB/DJCUlRc0nT56s5r17986ytWTH1wDtvcTu3bvVWVN79Y4dO9T89ttvd2VHjhwxrDDjChUqpOabN29W8xUrVqj5iBEjXJl2TYiIxMfHe7g6O5iapbX7v3LlSnX2vvvuU3N/eA0AMsKTa8DOd5sAAAAAAPwfNrYAAAAAAKuxsQUAAAAAWI2NLQAAAADAamxsAQAAAABW0yv6AsCxY8dcmanN8dtvv1VzbxpTCxcurOYlSpRQ8507d6q51i5ra3uuP+Cxd9Na67JTuzB8Z+rUqWo+aNAgNQ8ODnZl9957rzrri1bk8+fPq/n111/vym644QZ19uOPP1ZzU7vqmTNnXNl1112nzpqenw4fPqzmixcvdmXz5s1TZxcsWKDm/srUCNujRw9XZnq+M7XDDxkyRM2zsgFZc/z4cTU3XZ+mZuy7777blZlawU3fDPH888+r+Q8//KDm2UVCQoKav/rqq65sz549Wb0cIODwiS0AAAAAwGpsbAEAAAAAVmNjCwAAAACwGhtbAAAAAIDV2NgCAAAAAKwWsK3IiYmJrmzbtm3qbGhoqJpfuHDB49srXbq0mptaEjdu3KjmWuMsANjg4MGDam5qnm/atKkrK168eKauKSucPn3alZnaXE1tu0lJSWq+atUqV7ZhwwaP1yEi8vnnn6v58uXLXVlKSoo6G2hq1aql5n369HFlpvcGmzdvVnPT30d28dFHH6l5bGysmnfo0MGVmRqXY2Ji1Fz75gobmN6jjR079touBAhQfGILAAAAALAaG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgtYAtj9L+gf/u3bvVWVPx044dOzK8DlNBwp9//qnmlEcBsNWlS5fUfN68eWqulUeZSv6yu7Zt26r5d999p+Y33XSTmvfs2dOVJSQkpH9h8Ijp70Mrl0xNTVVnBw4cqOam+ewuLCxMzfPnz+/Kzpw5o842btxYzffv35/eZQEIYHxiCwAAAACwGhtbAAAAAIDV2NgCAAAAAKzGxhYAAAAAYDU2tgAAAAAAqwVsK7Jm9erVah4XF6fmO3fuVHOtubh69erq7B9//KHm3rQk5smTR83j4+M9PgYA+MqMGTPUvFOnTq7s6NGjWbwaz+XMqb+ETps2zZXdcccd6qypLbZbt25qTgOyb6xbt07Ntb+Pjz76SJ1du3Ztpq7pWunatauaP/roo2o+d+5cV9arVy919tChQ+leFwD8E5/YAgAAAACsxsYWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWI2NLQAAAADAakGOVuELAAAAAIAl+MQWAAAAAGA1NrYAAAAAAKuxsQUAAAAAWI2NLQAAAADAamxsAQAAAABWY2MLAAAAALAaG1sAAAAAgNXY2AIAAAAArMbGFgAAAABgtf8H7LEtxd+GHNgAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, axs = plt.subplots(2, 5, figsize=(12, 6))\n", + "#fig.subplots_adjust(hspace = 0.5, wspace=0.1)\n", + "axs = axs.ravel()\n", + "\n", + "classes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]\n", + "k = 0\n", + "for i in range(len(trainset)):\n", + " if classes:\n", + " if trainset[i][1] in classes:\n", + " axs[k].imshow(trainset[i][0].view(28, 28).numpy(), cmap=\"gray\")\n", + " axs[k].axis(\"off\")\n", + " axs[k].set_title(f\"Class {trainset[i][1]}\", fontsize=12);\n", + " classes.remove(trainset[i][1])\n", + " k += 1\n", + " else:\n", + " break\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dLCfnFW-JtGx" + }, + "source": [ + "### Задание 2. Строим свой первый MLP\n", + "\n", + "**4** балла\n", + "\n", + "MLP (multilayer perceptron) или нейронная сеть из полносвязных (линейных) слоев, это мы уже знаем.\n", + "\n", + "Опишите структуру сети: 3 полносвязных слоя + функции активации на ваш выбор. **Подумайте** про активацию после последнего слоя!\n", + "\n", + "Сеть на выходе 1 слоя должна иметь 256 признаков, на выходе из 2 128 признаков, на выходе из последнего столько, сколько у вас классов.\n", + "\n", + "https://pytorch.org/docs/stable/nn.html?highlight=activation#non-linear-activations-weighted-sum-nonlinearity" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "dhYBvQIXJdSz" + }, + "outputs": [], + "source": [ + "import torch.nn as nn\n", + "#import torch.nn.functional as F\n", + "\n", + "class FCNet(nn.Module):\n", + " def __init__(self, activation = nn.ReLU(), last_activation = None):\n", + " super().__init__() # это надо помнить!\n", + " ## YOUR CODE HERE\n", + " self.fc1 = nn.Linear(28*28, 256) \n", + " self.fc2 = nn.Linear(256, 128) \n", + " self.fc3 = nn.Linear(128, 10) \n", + " \n", + " self.activation = activation # F.relu\n", + " self.last_activation = last_activation\n", + "\n", + "\n", + "\n", + " ## YOUR CODE HERE\n", + " def forward(self, x): # Forward вызывается внутри метода __call__ родительского класса\n", + " ## x -> тензор размерности (BATCH_SIZE, N_CHANNELS, WIDTH, HEIGHT)\n", + " ## надо подумать над тем, что у нас полносвязные слои принимают векторы\n", + "\n", + " ## YOUR CODE HERE\n", + " x = x.view(-1, 28*28)\n", + " x = self.fc1(x)\n", + " x = self.activation(x)\n", + " x = self.fc2(x)\n", + " x = self.activation(x)\n", + " logits = self.fc3(x)\n", + " if self.last_activation is not None:\n", + " logits = self.last_activation(logits)\n", + " \n", + " return logits\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uI0R77EQNKef" + }, + "source": [ + "Сколько обучаемых параметров у вашей модели (весов и смещений)?\n", + "\n", + "**Ответ:**" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MvGiuoykrzzJ" + }, + "source": [ + "n_весов = (28 * 28) * 256 + 256 * 128 + 128 * 10 = 234752\n", + "\n", + "n_смещений = 256 + 128 + 10 " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iwGllji2M4lp" + }, + "source": [ + "### Задание 3. Напишите код для обучения модели\n", + "\n", + "**5** баллов\n", + "\n", + "Можно (и нужно) подглядывать в код семинара по пайторчу. Вам нужно создать модель, определить функцию потерь и оптимизатор (начнем с `SGD`). Дальше нужно обучать модель, при помощи тренировочного `Dataloader'a` и считать лосс на тренировочном и тестовом `Dataloader'ах`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Grv9xcybRfCX" + }, + "source": [ + "Напишем функцию для рассчета `accuracy`:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "9D2QPFe5JdVc" + }, + "outputs": [], + "source": [ + "def get_accuracy(model, dataloader):\n", + " \"\"\"\n", + " model - обученная нейронная сеть\n", + " dataloader - даталоадер, на котором вы хотите посчитать accuracy\n", + " \"\"\"\n", + " correct = 0\n", + " total = 0\n", + " with torch.no_grad(): # Тензоры внутри этого блока будут иметь requires_grad=False\n", + " for images, labels in dataloader:\n", + " outputs = model(images)\n", + " _, predicted = torch.max(outputs.data, 1)\n", + " total += labels.size(0)\n", + " correct += (predicted == labels).sum().item()\n", + " accuracy = correct / total\n", + "\n", + " return accuracy" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "D3EmoWJyTBkE" + }, + "source": [ + "#### Основной цикл обучения\n", + "\n", + "Этот код можно (и зачастую нужно) выносить в отдельную функцию, но пока что можете это не делать, все по желанию)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "uIZKSOdgUi3e" + }, + "outputs": [], + "source": [ + "# Создадим объект модели\n", + "fc_net = FCNet()\n", + "# Определим функцию потерь\n", + "loss_function = nn.CrossEntropyLoss()\n", + "# Создадим оптимизатор для нашей сети\n", + "lr = 0.001 # скорость обучения\n", + "optimizer = torch.optim.Adam(fc_net.parameters(), lr=3e-4)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uKYzXFqoX_fd" + }, + "source": [ + "Напишите цикл обучения. Для начала хватит 10 эпох. Какое значение `accuracy` на тестовой выборке удалось получить?" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "Ma2bshC6MxI6" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch=1 loss=0.9258387318316926\n", + "Epoch=2 loss=0.43879760770087545\n", + "Epoch=3 loss=0.3453855958390743\n", + "Epoch=4 loss=0.28895044675532805\n", + "Epoch=5 loss=0.2470524107522153\n", + "Epoch=6 loss=0.21449826962136206\n", + "Epoch=7 loss=0.18603831558151449\n", + "Epoch=8 loss=0.16194980261807745\n", + "Epoch=9 loss=0.14370059221982956\n", + "Epoch=10 loss=0.12640472679062092\n" + ] + } + ], + "source": [ + "n_epochs = 10\n", + "loss_history = []\n", + "\n", + "\n", + "## YOUR CODE HERE\n", + "for epoch in range(n_epochs):\n", + " epoch_loss = 0\n", + " for images, labels in trainloader: \n", + " optimizer.zero_grad() \n", + " outputs = fc_net(images)\n", + " loss = loss_function(outputs, labels) \n", + " loss.backward() \n", + " optimizer.step() \n", + "\n", + " epoch_loss += loss.item()\n", + "\n", + " loss_history.append(epoch_loss/len(trainloader))\n", + "\n", + " print(f\"Epoch={epoch+1} loss={loss_history[epoch]}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "id": "2cB5LRbrS3BN" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0.8767" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_accuracy(fc_net, testloader)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4HTJzBM8Yk1R" + }, + "source": [ + "### Задание 4. Изучение влияния нормализации\n", + "\n", + "**3** балла\n", + "\n", + "Вы могли заметить, что мы забыли провести нормализацию наших данных, а для нейронных сетей это может быть очень критично.\n", + "\n", + "Нормализуйте данные.\n", + "\n", + "* Подсчитайте среднее значение и стандартное отклонение интенсивности пикселей для всех тренировочных данных\n", + "* Нормализуйте данные с использованием этих параметров (используйте трансформацию `Normalize`)\n", + "\n", + "\n", + "Оцените влияние нормировки данных." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "id": "FHlDaYWGR6YA" + }, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (2949313568.py, line 1)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m Cell \u001b[0;32mIn[11], line 1\u001b[0;36m\u001b[0m\n\u001b[0;31m mean = ## calculate mean\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + ] + } + ], + "source": [ + "mean = ## calculate mean\n", + "std = ## calculate std\n", + "print(mean, std)\n", + "\n", + "transform_with_norm = torchvision.transforms.Compose([\n", + " torchvision.transforms.ToTensor(),\n", + " torchvision.transforms.Normalize(mean, std)\n", + " ])\n", + "\n", + "trainset.transform = transform_with_norm\n", + "testset.transform = transform_with_norm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Qj93J3X_R6aa" + }, + "outputs": [], + "source": [ + "fc_net = FCNet()\n", + "loss_function = nn.CrossEntropyLoss()\n", + "lr = 0.001\n", + "optimizer = torch.optim.Adam(fc_net.parameters(), lr=3e-4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VWZtYBCvAoWQ" + }, + "outputs": [], + "source": [ + "n_epochs = 10\n", + "loss_history = []\n", + "\n", + "## YOUR CODE HERE\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Gfbv9OIAAoYT" + }, + "outputs": [], + "source": [ + "get_accuracy(fc_net, testloader)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RcIJvhWkcjlh" + }, + "source": [ + "Как изменилась `accuracy` после нормализации?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "G1LHKF2PsZ4U" + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "atcfzu4acxP2" + }, + "source": [ + "### Задание 5. Изучение влияния функции активации\n", + "\n", + "**3** балла\n", + "\n", + "Исследуйте влияние функций активации на скорость обучения и точность предсказаний модели.\n", + "\n", + "Используйте три функции:\n", + "\n", + "* [Sigmoid](https://pytorch.org/docs/stable/nn.functional.html#sigmoid)\n", + "* [GELU](https://pytorch.org/docs/stable/nn.functional.html#gelu)\n", + "* [Tanh](https://pytorch.org/docs/stable/generated/torch.nn.Tanh.html#torch.nn.Tanh)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "id": "bAESPpjGa3M1" + }, + "outputs": [], + "source": [ + "## YOUR CODE HERE\n", + "activations = [nn.ReLU(), nn.Sigmoid(), nn.GELU(), nn.Tanh()]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "id": "GdvHSFeKa2sW" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "======\n", + "activation = ReLU()\n", + "Epoch=1 loss=0.9121983143877476\n", + "Epoch=2 loss=0.4449225447279342\n", + "Epoch=3 loss=0.350468417494855\n", + "Epoch=4 loss=0.2908842179369419\n", + "Epoch=5 loss=0.24844031879242431\n", + "Epoch=6 loss=0.21375525118188654\n", + "Epoch=7 loss=0.18521909073312232\n", + "Epoch=8 loss=0.16225507262539357\n", + "Epoch=9 loss=0.14123910448335586\n", + "Epoch=10 loss=0.12444839244510265\n", + "accucary = 0.8805 \n", + "\n", + "======\n", + "activation = Sigmoid()\n", + "Epoch=1 loss=1.7704682050867284\n", + "Epoch=2 loss=0.9282372393506638\n", + "Epoch=3 loss=0.6783941922035623\n", + "Epoch=4 loss=0.5528541849014607\n", + "Epoch=5 loss=0.47401719410368737\n", + "Epoch=6 loss=0.4172642213232974\n", + "Epoch=7 loss=0.37606875763294545\n", + "Epoch=8 loss=0.34244891449492026\n", + "Epoch=9 loss=0.31413997595614573\n", + "Epoch=10 loss=0.2895667387449995\n", + "accucary = 0.8153 \n", + "\n", + "======\n", + "activation = GELU(approximate='none')\n", + "Epoch=1 loss=0.9035523103906753\n", + "Epoch=2 loss=0.45603306192032833\n", + "Epoch=3 loss=0.35554138713694633\n", + "Epoch=4 loss=0.29644079962943465\n", + "Epoch=5 loss=0.25280315828450184\n", + "Epoch=6 loss=0.21995192544257386\n", + "Epoch=7 loss=0.1920593207186841\n", + "Epoch=8 loss=0.16840226326851135\n", + "Epoch=9 loss=0.1483412579019019\n", + "Epoch=10 loss=0.13061735468341948\n", + "accucary = 0.8761 \n", + "\n", + "======\n", + "activation = Tanh()\n", + "Epoch=1 loss=0.9297982589995607\n", + "Epoch=2 loss=0.5073035159009568\n", + "Epoch=3 loss=0.3920398046361639\n", + "Epoch=4 loss=0.3150038076842085\n", + "Epoch=5 loss=0.2608283148166981\n", + "Epoch=6 loss=0.21882402842983287\n", + "Epoch=7 loss=0.1871667762703084\n", + "Epoch=8 loss=0.16000487157639037\n", + "Epoch=9 loss=0.13742000051635378\n", + "Epoch=10 loss=0.11895063246818299\n", + "accucary = 0.8827 \n", + "\n" + ] + } + ], + "source": [ + "n_epochs = 10\n", + "#loss_history = []\n", + "lr = 0.001\n", + "for activation in activations: #обЪединяю код из прошлых ячеек\n", + " print('======')\n", + " print(f'{activation = }')\n", + " fc_net = FCNet(activation = activation)\n", + " loss_function = nn.CrossEntropyLoss()\n", + " optimizer = torch.optim.Adam(fc_net.parameters(), lr=3e-4) \n", + " loss_history = []\n", + " for epoch in range(n_epochs):\n", + " epoch_loss = 0\n", + " for images, labels in trainloader: \n", + " optimizer.zero_grad() \n", + " outputs = fc_net(images) \n", + " loss = loss_function(outputs, labels) \n", + " loss.backward() \n", + " optimizer.step() \n", + "\n", + " epoch_loss += loss.item()\n", + "\n", + " loss_history.append(epoch_loss/len(trainloader))\n", + " print(f\"Epoch={epoch+1} loss={loss_history[epoch]}\")\n", + " accucary = get_accuracy(fc_net, testloader)\n", + " print(f'{accucary = } \\n')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aG2Oyxy2egVV" + }, + "source": [ + "С использованием какой функции активации удалось досчить наибольшей `accuracy`?" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5SaqWhlkjuO3" + }, + "source": [ + "Наилучший результат получился при использовании функции **Tanh**" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "20Ls3Bfsifqd" + }, + "source": [ + "### Задание 6. Другие оптимизаторы\n", + "\n", + "**4** балла\n", + "\n", + "Исследуйте влияние оптимизаторов на скорость обучения и точность предсказаний модели.\n", + "\n", + "Попробуйте следующие:\n", + "\n", + "* [Adam](https://pytorch.org/docs/stable/generated/torch.optim.Adam.html#torch.optim.Adam)\n", + "* [RMSprop](https://pytorch.org/docs/stable/generated/torch.optim.RMSprop.html#torch.optim.RMSprop)\n", + "* [Adagrad](https://pytorch.org/docs/stable/generated/torch.optim.Adagrad.html#torch.optim.Adagrad)\n", + "\n", + "Вам нужно снова обучить 3 модели и сравнить их перформанс (функцию активации используйте ту, которая показала себя лучше всего)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rzL2LdA-ifJh" + }, + "outputs": [], + "source": [ + "activation = nn.Tanh()\n", + "optimisators = [torch.optim.Adam, torch.optim.Adagrad, torch.optim.RMSprop]\n", + "\n", + "n_epochs = 10\n", + "#loss_history = []\n", + "lr = 0.001\n", + "for opt in optimisators: #обЪединяю код из прошлых ячеек\n", + " print('======')\n", + " print(f'{opt = }')\n", + " print('======')\n", + " fc_net = FCNet(activation = activation)\n", + " loss_function = nn.CrossEntropyLoss()\n", + " optimizer = opt(fc_net.parameters(), lr=3e-4) \n", + " loss_history = []\n", + " for epoch in range(n_epochs):\n", + " epoch_loss = 0\n", + " for images, labels in trainloader: \n", + " optimizer.zero_grad() \n", + " outputs = fc_net(images) \n", + " loss = loss_function(outputs, labels) \n", + " loss.backward() \n", + " optimizer.step() \n", + "\n", + " epoch_loss += loss.item()\n", + "\n", + " loss_history.append(epoch_loss/len(trainloader))\n", + " print(f\"Epoch={epoch+1} loss={loss_history[epoch]}\")\n", + " accucary = get_accuracy(fc_net, testloader)\n", + " print(f'{accucary = } \\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Лучше всех показала себя Adam, чуть лучше,чем RMSProp" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eHA48PsperxS" + }, + "source": [ + "### Задание 7. Реализация ReLU\n", + "\n", + "**4** балла\n", + "\n", + "Самостоятельно реализуйте функцию активации ReLU.\n", + "Замените в уже обученной модели функцию активации на вашу. Убедитесь что ничего не изменилась." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "id": "63uTkUp-a2xr" + }, + "outputs": [], + "source": [ + "class CustomReLU(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + "\n", + " def forward(self, x):\n", + " # YOUR CODE HERE\n", + " # если элемент x < 0, то 0, если >= 0, то x\n", + " x[x < 0] = 0\n", + " return x" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xsKzxa33fhbN" + }, + "source": [ + "Заново обучите модель и проверьте правильность реализации `CustomReLU`." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "id": "ePP55RBeecYh" + }, + "outputs": [], + "source": [ + "# Создадим объект модели\n", + "fc_net = FCNet(activation= CustomReLU())\n", + "# Определим функцию потерь\n", + "loss_function = nn.CrossEntropyLoss()\n", + "# Создадим оптимизатор для нашей сети\n", + "lr = 0.001 # скорость обучения\n", + "optimizer = torch.optim.Adam(fc_net.parameters(), lr=3e-4)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch=1 loss=0.9213035495991403\n", + "Epoch=2 loss=0.43675460333519794\n", + "Epoch=3 loss=0.3449437768535411\n", + "Epoch=4 loss=0.29073638605310564\n", + "Epoch=5 loss=0.25006262225673553\n", + "Epoch=6 loss=0.21877629065767248\n", + "Epoch=7 loss=0.19110677942950674\n", + "Epoch=8 loss=0.16773000343682917\n", + "Epoch=9 loss=0.14794787209718785\n", + "Epoch=10 loss=0.13121437915462128\n" + ] + } + ], + "source": [ + "n_epochs = 10\n", + "loss_history = []\n", + "\n", + "\n", + "## YOUR CODE HERE\n", + "for epoch in range(n_epochs):\n", + " epoch_loss = 0\n", + " for images, labels in trainloader: \n", + " optimizer.zero_grad() \n", + " outputs = fc_net(images)\n", + " loss = loss_function(outputs, labels) \n", + " loss.backward() \n", + " optimizer.step() \n", + "\n", + " epoch_loss += loss.item()\n", + "\n", + " loss_history.append(epoch_loss/len(trainloader))\n", + "\n", + " print(f\"Epoch={epoch+1} loss={loss_history[epoch]}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.8722" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_accuracy(fc_net, testloader)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Чуть отличается от того, что было, но работает" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vWBG1mMwgN17" + }, + "source": [ + "### Задание 8. Генерация картинок\n", + "\n", + "**3** балла\n", + "\n", + "Придумайте 3 предложения и сгенерируйте при помощи них 3 картинки, используя телеграм бота [ruDALLE](https://t.me/sber_rudalle_xl_bot). Прикрепите сюда ваши картины. (я пользовалась ботом @gigabot)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Итак. \"Ученье - свет, а неученье - тьма\". Оставляю две картины, обе шедевры" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hlqgll9qecdZ" + }, + "outputs": [], + "source": [ + "img_path1 = '../Data/svet1.jpg'\n", + "img_path2 = '../Data/svet2.jpg'\n", + "\n", + "\n", + "# Отображение изображений\n", + "display(Image(filename=img_path1))\n", + "display(Image(filename=img_path2))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Это был самый обычный день\n", + "(просто типичный мой день)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "img_path1 = '../Data/usual_day.jpg'\n", + "display(Image(filename=img_path1))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Все смешалось в доме Облонских\n", + "(спасибо, что без поезда)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "img_path1 = '../Data/home.jpg'\n", + "display(Image(filename=img_path1))" + ] + } + ], + "metadata": { + "colab": { + "authorship_tag": "ABX9TyN3ji5dthhFgQhP1CJ7JjiU", + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/HW7/Data/home.jpg b/HW7/Data/home.jpg new file mode 100644 index 0000000..9849ae3 Binary files /dev/null and b/HW7/Data/home.jpg differ diff --git a/HW7/Data/svet1.jpg b/HW7/Data/svet1.jpg new file mode 100644 index 0000000..53460aa Binary files /dev/null and b/HW7/Data/svet1.jpg differ diff --git a/HW7/Data/svet2.jpg b/HW7/Data/svet2.jpg new file mode 100644 index 0000000..8e9d518 Binary files /dev/null and b/HW7/Data/svet2.jpg differ diff --git a/HW7/Data/usual_day.jpg b/HW7/Data/usual_day.jpg new file mode 100644 index 0000000..2049cec Binary files /dev/null and b/HW7/Data/usual_day.jpg differ diff --git a/HW7/requirments.txt b/HW7/requirments.txt new file mode 100644 index 0000000..84d2af8 --- /dev/null +++ b/HW7/requirments.txt @@ -0,0 +1,5 @@ +matplotlib==3.8.2 +matplotlib-inline==0.1.6 +torch==2.2.2 +torchvision==0.17.2 +