diff --git a/translation.ipynb b/translation.ipynb new file mode 100644 index 0000000..6504c6e --- /dev/null +++ b/translation.ipynb @@ -0,0 +1,1059 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### T5 трансформер для перевода с немецкого на английский" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "В этом задании вам предстоит обучить encoder-decoder трансформер переводить короткие предложения с английского на немецкий.\n", + "\n", + "Токенизацию и блоки трансформера писать не придётся — будем пользоваться библиотекой `transformers`.\n", + "\n", + "Наши задачи:\n", + "- Познакомиться с наиболее популярным методом токенизации текстов — byte pair encoding\n", + "- Научиться пользоваться токенизатором из `transformers`\n", + "- Подготовить данные\n", + "- Обучить готовую архитектуру T5-small под нашу задачу\n", + "- Исследовать разные стратегии декодирования с помощью обученной модели" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "from pathlib import Path\n", + "from typing import cast\n", + "\n", + "import lightning as L\n", + "import torch\n", + "import torch.nn.functional as F\n", + "from torch import Tensor, nn\n", + "from torch.utils.data import DataLoader, Dataset\n", + "from transformers import T5ForConditionalGeneration, T5Tokenizer\n", + "\n", + "from datasets import load_dataset, load_from_disk" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(42)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Готовим данные" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "train_dataset = load_dataset(\"bentrevett/multi30k\", split=\"train\")\n", + "test_dataset = load_dataset(\"bentrevett/multi30k\", split=\"test\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'en': 'Two young, White males are outside near many bushes.',\n", + " 'de': 'Zwei junge weiße Männer sind im Freien in der Nähe vieler Büsche.'}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_dataset[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0cAAAESCAYAAAA2Zq7uAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjUsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvWftoOwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAO+FJREFUeJzt3X9cVvX9//Env1ERCBUumOAw81f5EwuuT+b8QaJRq8l2m43SkvSWgz4JpcVyltaiXGZZmGs5sS1nuk/a0lIJBVeCKckyLVNjw6YXNB1cYQkI5/tHX868Es0LLrz48bjfbucW13m/zuH1Pnjj3Yv3+5zjYRiGIQAAAADo5DzdnQAAAAAAtAUURwAAAAAgiiMAAAAAkERxBAAAAACSKI4AAAAAQBLFEQAAAABIojgCAAAAAEmSt7sTaC0NDQ06fvy4unfvLg8PD3enAwCdhmEY+uqrrxQRESFPT/4Gdy7GJgBwj0sdmzpscXT8+HFFRka6Ow0A6LSOHTum3r17uzuNNoWxCQDc6/vGpg5bHHXv3l3StxcgMDDQzdkAQOdht9sVGRlp/h7GfzE2AYB7XOrY5FRx9MMf/lD//Oc/z9v/y1/+UtnZ2Tpz5oweeOABrV27VjU1NUpISNDy5csVFhZmxpaVlWn27NnasWOHAgICNH36dGVlZcnb+7+p5OfnKyMjQwcOHFBkZKTmz5+vu+66y5lUzeUKgYGBDEAA4AYsGzsfYxMAuNf3jU1OLQbfs2ePTpw4YW65ubmSpJ/97GeSpPT0dL311ltav369CgoKdPz4cU2ZMsU8vr6+XomJiaqtrdWuXbu0evVq5eTkaMGCBWZMaWmpEhMTNW7cOJWUlGjOnDm65557tHXrVmdSBQAAAACneBiGYTT34Dlz5mjTpk06fPiw7Ha7evXqpTVr1uinP/2pJOnTTz/VoEGDVFhYqLi4OL3zzju6+eabdfz4cXM2acWKFXrooYf05ZdfytfXVw899JA2b96sjz/+2Pw+U6dOVWVlpbZs2XLJudntdgUFBamqqoq/zgHAZcTv3wvj2gCAe1zq799mP0aotrZWf/rTnzRjxgx5eHiouLhYdXV1io+PN2MGDhyoqKgoFRYWSpIKCws1ZMgQh2V2CQkJstvtOnDggBlz7jkaYxrPcSE1NTWy2+0OGwAAAABcqmY/kGHjxo2qrKw07wWy2Wzy9fVVcHCwQ1xYWJhsNpsZc25h1Nje2HaxGLvdrm+++UZdunRpMp+srCwtXLiwud0BAKfU19errq7O3Wm4hY+Pj7y8vNydBgDgHA0NDaqtrXV3Gm7jqrGp2cXRypUrNXnyZEVERLQ4CVfIzMxURkaG+bnxiRQA4EqGYchms6mystLdqbhVcHCwLBYLD10AgDagtrZWpaWlamhocHcqbuWKsalZxdE///lPvfvuu3rjjTfMfRaLRbW1taqsrHSYPSovL5fFYjFjPvjgA4dzlZeXm22N/23cd25MYGDgBWeNJMnPz09+fn7N6Q4AXLLGwig0NFRdu3btdMWBYRj6+uuvVVFRIUkKDw93c0YA0LkZhqETJ07Iy8tLkZGRnfLl264cm5pVHK1atUqhoaFKTEw098XExMjHx0d5eXlKSkqSJB06dEhlZWWyWq2SJKvVqt/85jeqqKhQaGioJCk3N1eBgYEaPHiwGfP22287fL/c3FzzHADgLvX19WZh1KNHD3en4zaNf6hq/F3OEjsAcJ+zZ8/q66+/VkREhLp27erudNzGVWOT06VlQ0ODVq1apenTpzu8mygoKEgpKSnKyMjQjh07VFxcrLvvvltWq1VxcXGSpIkTJ2rw4MG688479fe//11bt27V/PnzlZqaas763Hvvvfr88881b948ffrpp1q+fLnWrVun9PT0ZnUQAFyl8R6jzjz4NGq8Bp31visAaCvq6+slSb6+vm7OxP1cMTY5PXP07rvvqqysTDNmzDivbenSpfL09FRSUpLDS2AbeXl5adOmTZo9e7asVqu6deum6dOna9GiRWZMdHS0Nm/erPT0dD3//PPq3bu3XnnlFSUkJDSzi2htS3M/a9Zx6Tf2d3EmwOXR2ZbSNYVrgEu2I6v5x47LdF0eQAfH72XXXAOni6OJEyfqQq9G8vf3V3Z2trKzsy94fJ8+fc5bNvddY8eO1b59+5xNDa52iQNaXNnJJvcXRc1yZTYAAABAq+p8d2wBAAAAQBOa/ShvAMC3mru0tLlYkgoA+D6MTc3DzBEAAAAAiOIIAAAAACRRHAFAp9DQ0KCsrCxFR0erS5cuGjZsmP7yl79IkvLz8+Xh4aG8vDyNGjVKXbt21f/8z//o0KFDbs4aANCRtcWxieIIADqBrKwsvfrqq1qxYoUOHDig9PR03XHHHSooKDBjHnnkES1ZskR79+6Vt7d3k69sAADAVdri2MQDGQCgg6upqdGTTz6pd999V1arVZLUt29fvffee/rd736nWbO+fez+b37zG/3oRz+SJD388MNKTEzUmTNn5O/v77bcAQAdU1sdmyiOAKCDO3LkiL7++mvdeOONDvtra2s1YsQI8/PQoUPNr8PDwyVJFRUVioqKujyJAgA6jbY6NlEcAUAHV11dLUnavHmzfvCDHzi0+fn56ejRo5IkHx8fc3/jW8YbGhouU5YAgM6krY5NFEcA0MENHjxYfn5+KisrM5cmnKtxAAIA4HJpq2MTxREAdHDdu3fXgw8+qPT0dDU0NGj06NGqqqrS+++/r8DAQPXp08fdKQKXTUtfjNlRXnQJuFtbHZsojgCghdrD/yw9/vjj6tWrl7KysvT5558rODhYI0eO1K9+9SuWzgFAB8TY1DwURwDQCXh4eOj+++/X/fff32S7YRgOn4cPH37ePgAAXKktjk285wgAAAAARHEEAAAAAJIojgAAAABAEsURAAAAAEiiOAIAAAAASRRHAAAAACCJ4ggAAAAAJFEcAQAAAICkZhRH//rXv3THHXeoR48e6tKli4YMGaK9e/ea7YZhaMGCBQoPD1eXLl0UHx+vw4cPO5zj1KlTSk5OVmBgoIKDg5WSkqLq6mqHmI8++kg33HCD/P39FRkZqcWLFzeziwCApowdO1Zz5sxxdxoAAJjcPTZ5OxP8n//8R9dff73GjRund955R7169dLhw4d1xRVXmDGLFy/WsmXLtHr1akVHR+vXv/61EhISdPDgQfn7+0uSkpOTdeLECeXm5qqurk533323Zs2apTVr1kiS7Ha7Jk6cqPj4eK1YsUL79+/XjBkzFBwcrFmzZrmw+wDgAjuyLu/3G5d5eb8fAKD9YWxqFqeKo6efflqRkZFatWqVuS86Otr82jAMPffcc5o/f75uvfVWSdKrr76qsLAwbdy4UVOnTtUnn3yiLVu2aM+ePRo1apQk6YUXXtBNN92kZ555RhEREXrttddUW1urP/zhD/L19dXVV1+tkpISPfvssxRHAAAAAFqFU8vq/vrXv2rUqFH62c9+ptDQUI0YMUK///3vzfbS0lLZbDbFx8eb+4KCghQbG6vCwkJJUmFhoYKDg83CSJLi4+Pl6emp3bt3mzFjxoyRr6+vGZOQkKBDhw7pP//5T5O51dTUyG63O2wAgG+dPn1a06ZNU0BAgMLDw7VkyRKH9pqaGj344IP6wQ9+oG7duik2Nlb5+fnuSRYA0Cm0xbHJqeLo888/10svvaSrrrpKW7du1ezZs/W///u/Wr16tSTJZrNJksLCwhyOCwsLM9tsNptCQ0Md2r29vRUSEuIQ09Q5zv0e35WVlaWgoCBzi4yMdKZrANChzZ07VwUFBXrzzTe1bds25efn68MPPzTb09LSVFhYqLVr1+qjjz7Sz372M02aNOm8e0YBAHCVtjg2OVUcNTQ0aOTIkXryySc1YsQIzZo1SzNnztSKFStaK79LlpmZqaqqKnM7duyYu1MCgDahurpaK1eu1DPPPKMJEyZoyJAhWr16tc6ePStJKisr06pVq7R+/XrdcMMNuvLKK/Xggw9q9OjRDsuo27OsrCxde+216t69u0JDQ3Xbbbfp0KFDDjFnzpxRamqqevTooYCAACUlJam8vNwhpqysTImJieratatCQ0M1d+5c8zoCAC5dWx2bnLrnKDw8XIMHD3bYN2jQIP3f//2fJMlisUiSysvLFR4ebsaUl5dr+PDhZkxFRYXDOc6ePatTp06Zx1sslvMGpMbPjTHf5efnJz8/P2e6AwCdwtGjR1VbW6vY2FhzX0hIiAYMGCBJ2r9/v+rr69W/f3+H42pqatSjR4/LmmtrKSgoUGpqqq699lqdPXtWv/rVrzRx4kQdPHhQ3bp1kySlp6dr8+bNWr9+vYKCgpSWlqYpU6bo/ffflyTV19crMTFRFotFu3bt0okTJzRt2jT5+PjoySefdGf3AKDdaatjk1PF0fXXX3/eX9o+++wz9enTR9K3D2ewWCzKy8sziyG73a7du3dr9uzZkiSr1arKykoVFxcrJiZGkrR9+3Y1NDSYF8dqteqRRx5RXV2dfHx8JEm5ubkaMGCAw5PxAAAtV11dLS8vLxUXF8vLy8uhLSAgwE1ZudaWLVscPufk5Cg0NFTFxcUaM2aMqqqqtHLlSq1Zs0bjx4+XJK1atUqDBg1SUVGR4uLitG3bNh08eFDvvvuuwsLCNHz4cD3++ON66KGH9NhjjzncJwsAaBl3jU1OLatLT09XUVGRnnzySR05ckRr1qzRyy+/rNTUVEmSh4eH5syZoyeeeEJ//etftX//fk2bNk0RERG67bbbJH070zRp0iTNnDlTH3zwgd5//32lpaVp6tSpioiIkCT94he/kK+vr1JSUnTgwAG9/vrrev7555WRkeHa3gNAJ3DllVfKx8fHfOiN9O2rGT777DNJ0ogRI1RfX6+Kigr169fPYbvQbH17V1VVJenbv1JKUnFxserq6hweKDRw4EBFRUU5PFBoyJAhDvfEJiQkyG6368CBA01+Hx4WBABNa6tjk1MzR9dee602bNigzMxMLVq0SNHR0XruueeUnJxsxsybN0+nT5/WrFmzVFlZqdGjR2vLli3mO44k6bXXXlNaWpomTJggT09PJSUladmyZWZ7UFCQtm3bptTUVMXExKhnz55asGABj/EGgGYICAhQSkqK5s6dqx49eig0NFSPPPKIPD2//ftY//79lZycrGnTpmnJkiUaMWKEvvzyS+Xl5Wno0KFKTEx0cw9cq6GhQXPmzNH111+va665RtK3D/vx9fVVcHCwQ+x3HyjUnIcFLVy40MU9AID2r62OTU4VR5J088036+abb75gu4eHhxYtWqRFixZdMCYkJMR84euFDB06VH/729+cTQ8A0ITf/va3qq6u1i233KLu3bvrgQceMGdPpG+XkD3xxBN64IEH9K9//Us9e/ZUXFzcRX/ft1epqan6+OOP9d5777X698rMzHRY9WC323maKgD8f21xbHK6OAIAfEc7eCt4QECA/vjHP+qPf/yjuW/u3Lnm1z4+Plq4cGGHn+VIS0vTpk2btHPnTvXu3dvcb7FYVFtbq8rKSofZo/LycoeHBX3wwQcO5+NhQQDaLMamZnHqniMAANojwzCUlpamDRs2aPv27YqOjnZoj4mJkY+Pj/Ly8sx9hw4dUllZmaxWq6RvHxa0f/9+hyeu5ubmKjAw8LwnuQIA2idmjgAAHV5qaqrWrFmjN998U927dzfvEQoKClKXLl0UFBSklJQUZWRkKCQkRIGBgbrvvvtktVoVFxcnSZo4caIGDx6sO++8U4sXL5bNZtP8+fOVmprK7BAAdBAURwCADu+ll16SJI0dO9Zh/6pVq3TXXXdJkpYuXWo+JKimpkYJCQlavny5Gevl5aVNmzZp9uzZslqt6tatm6ZPn37Re2wBAO0LxRFaTVzZyxcP2PE9L/BqB2tlAbQPhmF8b4y/v7+ys7OVnZ19wZg+ffro7bffdmVqAIA2hHuOAAAAAEAURwDgtIaGBnen4HZcAwBoWy5lhryjc8XYxLI6ALhEvr6+8vT01PHjx9WrVy/5+vrKw8PD3WldVoZhqLa2Vl9++aU8PT3l6+vr7pQAoFPz8fGRh4eHvvzyS/Xq1avTjUuSa8cmiiMAuESenp6Kjo7WiRMndPz4cXen41Zdu3ZVVFSU+SZzAIB7eHl5qXfv3vriiy/0j3/8w93puJUrxiaKI7hN4ecnL9pedPazC7al39jf1ekAl8TX11dRUVE6e/as6uvr3Z2OW3h5ecnb27tT/nUSANqigIAAXXXVVaqrq3N3Km7jqrGJ4ggAnOTh4SEfHx/5+Pi4OxUAACR9Wxx4eXm5O412j/UQAAAAACCKIwAAAACQRHEEAAAAAJIojgAAAABAEg9k6Nh2ZLk7AwAAAKDdYOYIAAAAAERxBAAAAACSKI4AAAAAQBLFEQAAAABIojgCAAAAAElOFkePPfaYPDw8HLaBAwea7WfOnFFqaqp69OihgIAAJSUlqby83OEcZWVlSkxMVNeuXRUaGqq5c+fq7NmzDjH5+fkaOXKk/Pz81K9fP+Xk5DS/hwAAAABwCZyeObr66qt14sQJc3vvvffMtvT0dL311ltav369CgoKdPz4cU2ZMsVsr6+vV2Jiompra7Vr1y6tXr1aOTk5WrBggRlTWlqqxMREjRs3TiUlJZozZ47uuecebd26tYVdBQAAAIALc/o9R97e3rJYLOftr6qq0sqVK7VmzRqNHz9ekrRq1SoNGjRIRUVFiouL07Zt23Tw4EG9++67CgsL0/Dhw/X444/roYce0mOPPSZfX1+tWLFC0dHRWrJkiSRp0KBBeu+997R06VIlJCS0sLsAAAAA0DSnZ44OHz6siIgI9e3bV8nJySorK5MkFRcXq66uTvHx8WbswIEDFRUVpcLCQklSYWGhhgwZorCwMDMmISFBdrtdBw4cMGPOPUdjTOM5LqSmpkZ2u91hAwAAAIBL5dTMUWxsrHJycjRgwACdOHFCCxcu1A033KCPP/5YNptNvr6+Cg4OdjgmLCxMNptNkmSz2RwKo8b2xraLxdjtdn3zzTfq0qVLk7llZWVp4cKFznQHAADgPHFlL1+4cUePix88LtO1yQC4rJwqjiZPnmx+PXToUMXGxqpPnz5at27dBYuWyyUzM1MZGRnmZ7vdrsjISDdmBAAAAKA9adGjvIODg9W/f38dOXJEFotFtbW1qqysdIgpLy8371GyWCznPb2u8fP3xQQGBl60APPz81NgYKDDBgAAAACXqkXFUXV1tY4eParw8HDFxMTIx8dHeXl5ZvuhQ4dUVlYmq9UqSbJardq/f78qKirMmNzcXAUGBmrw4MFmzLnnaIxpPAcAAAAAtAaniqMHH3xQBQUF+sc//qFdu3bpJz/5iby8vHT77bcrKChIKSkpysjI0I4dO1RcXKy7775bVqtVcXFxkqSJEydq8ODBuvPOO/X3v/9dW7du1fz585Wamio/Pz9J0r333qvPP/9c8+bN06effqrly5dr3bp1Sk9Pd33vAQAAAOD/c+qeoy+++EK33367Tp48qV69emn06NEqKipSr169JElLly6Vp6enkpKSVFNTo4SEBC1fvtw83svLS5s2bdLs2bNltVrVrVs3TZ8+XYsWLTJjoqOjtXnzZqWnp+v5559X79699corr/AYbwAAAACtyqniaO3atRdt9/f3V3Z2trKzsy8Y06dPH7399tsXPc/YsWO1b98+Z1IDAAAAgBZp0T1HAAAAANBRUBwBAAAAgCiOAAAAAEASxREAAAAASKI4AgAAAABJFEcAAAAAIIniCAAAAAAkURwBAAAAgCSKIwAAAACQRHEEAOgEdu7cqVtuuUURERHy8PDQxo0bHdrvuusueXh4OGyTJk1yiDl16pSSk5MVGBio4OBgpaSkqLq6+jL2AgDQ2iiOAAAd3unTpzVs2DBlZ2dfMGbSpEk6ceKEuf35z392aE9OTtaBAweUm5urTZs2aefOnZo1a1Zrpw4AuIy83Z0AAACtbfLkyZo8efJFY/z8/GSxWJps++STT7Rlyxbt2bNHo0aNkiS98MILuummm/TMM88oIiLC5TkDAC4/Zo4AAJCUn5+v0NBQDRgwQLNnz9bJkyfNtsLCQgUHB5uFkSTFx8fL09NTu3fvvuA5a2pqZLfbHTYAQNtFcQQA6PQmTZqkV199VXl5eXr66adVUFCgyZMnq76+XpJks9kUGhrqcIy3t7dCQkJks9kueN6srCwFBQWZW2RkZKv2AwDQMiyrAwB0elOnTjW/HjJkiIYOHaorr7xS+fn5mjBhQrPPm5mZqYyMDPOz3W6nQAKANoyZIwAAvqNv377q2bOnjhw5IkmyWCyqqKhwiDl79qxOnTp1wfuUpG/vYwoMDHTYAABtFzNHAAB8xxdffKGTJ08qPDxckmS1WlVZWani4mLFxMRIkrZv366GhgbFxsa6M9XLbmnuZ07Fx5WddPhs7dvDlekAgEtRHAEAOrzq6mpzFkiSSktLVVJSopCQEIWEhGjhwoVKSkqSxWLR0aNHNW/ePPXr108JCQmSpEGDBmnSpEmaOXOmVqxYobq6OqWlpWnq1Kk8qQ4AOhCW1QEAOry9e/dqxIgRGjFihCQpIyNDI0aM0IIFC+Tl5aWPPvpIP/7xj9W/f3+lpKQoJiZGf/vb3+Tn52ee47XXXtPAgQM1YcIE3XTTTRo9erRefvlld3UJANAKmDnqYM5d7vDdpQzfh6UOADqqsWPHyjCMC7Zv3br1e88REhKiNWvWuDItAEAbw8wRAAAAAIjiCAAAAAAktbA4euqpp+Th4aE5c+aY+86cOaPU1FT16NFDAQEBSkpKUnl5ucNxZWVlSkxMVNeuXRUaGqq5c+fq7NmzDjH5+fkaOXKk/Pz81K9fP+Xk5LQkVQAAAAC4qGYXR3v27NHvfvc7DR061GF/enq63nrrLa1fv14FBQU6fvy4pkyZYrbX19crMTFRtbW12rVrl1avXq2cnBwtWLDAjCktLVViYqLGjRunkpISzZkzR/fcc88lrQkHAAAAgOZoVnFUXV2t5ORk/f73v9cVV1xh7q+qqtLKlSv17LPPavz48YqJidGqVau0a9cuFRUVSZK2bdumgwcP6k9/+pOGDx+uyZMn6/HHH1d2drZqa2slSStWrFB0dLSWLFmiQYMGKS0tTT/96U+1dOnSC+ZUU1Mju93usAEAAADApWpWcZSamqrExETFx8c77C8uLlZdXZ3D/oEDByoqKkqFhYWSpMLCQg0ZMkRhYWFmTEJCgux2uw4cOGDGfPfcCQkJ5jmakpWVpaCgIHOLjIxsTtcAAAAAdFJOF0dr167Vhx9+qKysrPPabDabfH19FRwc7LA/LCxMNpvNjDm3MGpsb2y7WIzdbtc333zTZF6ZmZmqqqoyt2PHjjnbNQAAAACdmFPvOTp27Jjuv/9+5ebmyt/fv7VyahY/Pz+Hl/Wh/Ysru8jLFXdcwjuZxmW6LhkAAAB0eE7NHBUXF6uiokIjR46Ut7e3vL29VVBQoGXLlsnb21thYWGqra1VZWWlw3Hl5eWyWCySJIvFct7T6xo/f19MYGCgunTp4lQHAQAAAOBSOFUcTZgwQfv371dJSYm5jRo1SsnJyebXPj4+ysvLM485dOiQysrKZLVaJUlWq1X79+9XRUWFGZObm6vAwEANHjzYjDn3HI0xjecAAAAAAFdzalld9+7ddc011zjs69atm3r06GHuT0lJUUZGhkJCQhQYGKj77rtPVqtVcXFxkqSJEydq8ODBuvPOO7V48WLZbDbNnz9fqamp5rK4e++9Vy+++KLmzZunGTNmaPv27Vq3bp02b97sij4DAAC4xdLcz1p0fPqN/V2UCYCmOFUcXYqlS5fK09NTSUlJqqmpUUJCgpYvX262e3l5adOmTZo9e7asVqu6deum6dOna9GiRWZMdHS0Nm/erPT0dD3//PPq3bu3XnnlFSUkJLg6XQAA0I60tLgAgItpcXGUn5/v8Nnf31/Z2dnKzs6+4DF9+vTR22+/fdHzjh07Vvv27WtpegAAAABwSZr1niMAAAAA6GgojgAAAABAFEcAAAAAIIniCAAAAAAkURwBAAAAgCSKIwAAAACQRHEEAAAAAJJa4SWwAAAALrEj67xdcWUnL+nQoqhZrs4GQCfAzBEAAAAAiOIIAAAAACRRHAEAAACAJIojAAAAAJBEcQQAAAAAkiiOAAAAAEASxREAAAAASKI4AgAAAABJFEcAAAAAIIniCAAAAAAkSd7uTgBtR+HnJ92dAgAAAOA2zBwBAAAAgCiOAAAAAECSk8XRSy+9pKFDhyowMFCBgYGyWq165513zPYzZ84oNTVVPXr0UEBAgJKSklReXu5wjrKyMiUmJqpr164KDQ3V3LlzdfbsWYeY/Px8jRw5Un5+furXr59ycnKa30MAANBmFH5+skUbALQmp4qj3r1766mnnlJxcbH27t2r8ePH69Zbb9WBAwckSenp6Xrrrbe0fv16FRQU6Pjx45oyZYp5fH19vRITE1VbW6tdu3Zp9erVysnJ0YIFC8yY0tJSJSYmaty4cSopKdGcOXN0zz33aOvWrS7qMgAAAACcz6ni6JZbbtFNN92kq666Sv3799dvfvMbBQQEqKioSFVVVVq5cqWeffZZjR8/XjExMVq1apV27dqloqIiSdK2bdt08OBB/elPf9Lw4cM1efJkPf7448rOzlZtba0kacWKFYqOjtaSJUs0aNAgpaWl6ac//amWLl3q+t4DADqFnTt36pZbblFERIQ8PDy0ceNGh3bDMLRgwQKFh4erS5cuio+P1+HDhx1iTp06peTkZAUGBio4OFgpKSmqrq6+jL0AALS2Zt9zVF9fr7Vr1+r06dOyWq0qLi5WXV2d4uPjzZiBAwcqKipKhYWFkqTCwkINGTJEYWFhZkxCQoLsdrs5+1RYWOhwjsaYxnNcSE1Njex2u8MGAIAknT59WsOGDVN2dnaT7YsXL9ayZcu0YsUK7d69W926dVNCQoLOnDljxiQnJ+vAgQPKzc3Vpk2btHPnTs2aNetydQEAcBk4/Sjv/fv3y2q16syZMwoICNCGDRs0ePBglZSUyNfXV8HBwQ7xYWFhstlskiSbzeZQGDW2N7ZdLMZut+ubb75Rly5dmswrKytLCxcudLY7AIBOYPLkyZo8eXKTbYZh6LnnntP8+fN16623SpJeffVVhYWFaePGjZo6dao++eQTbdmyRXv27NGoUaMkSS+88IJuuukmPfPMM4qIiLhsfQEAtB6nZ44GDBigkpIS7d69W7Nnz9b06dN18ODB1sjNKZmZmaqqqjK3Y8eOuTslAEA7UFpaKpvN5rBqISgoSLGxsQ4rH4KDg83CSJLi4+Pl6emp3bt3X/DcrGoAgPbF6ZkjX19f9evXT5IUExOjPXv26Pnnn9fPf/5z1dbWqrKy0mH2qLy8XBaLRZJksVj0wQcfOJyv8Wl258Z89wl35eXlCgwMvOCskST5+fnJz8/P2e4AADq5xpULTa1aOHdVQ2hoqEO7t7e3QkJCzJimsKoBANqXFr/nqKGhQTU1NYqJiZGPj4/y8vLMtkOHDqmsrExWq1WSZLVatX//flVUVJgxubm5CgwM1ODBg82Yc8/RGNN4DgAA2gtWNQBA++LUzFFmZqYmT56sqKgoffXVV1qzZo3y8/O1detWBQUFKSUlRRkZGQoJCVFgYKDuu+8+Wa1WxcXFSZImTpyowYMH684779TixYtls9k0f/58paammrM+9957r1588UXNmzdPM2bM0Pbt27Vu3Tpt3rzZ9b1Hp7M097NmHZd+Y38XZwKgrWhcuVBeXq7w8HBzf3l5uYYPH27GnPuHPUk6e/asTp06ZR7fFFY1AED74tTMUUVFhaZNm6YBAwZowoQJ2rNnj7Zu3aobb7xRkrR06VLdfPPNSkpK0pgxY2SxWPTGG2+Yx3t5eWnTpk3y8vKS1WrVHXfcoWnTpmnRokVmTHR0tDZv3qzc3FwNGzZMS5Ys0SuvvKKEhAQXdRkAgP+Kjo6WxWJxWLVgt9u1e/duh5UPlZWVKi4uNmO2b9+uhoYGxcbGXvacAQCtw6mZo5UrV1603d/fX9nZ2Rd8VKok9enTR2+//fZFzzN27Fjt27fPmdQAALig6upqHTlyxPxcWlqqkpIShYSEKCoqSnPmzNETTzyhq666StHR0fr1r3+tiIgI3XbbbZKkQYMGadKkSZo5c6ZWrFihuro6paWlaerUqTypDgA6EKcfyAAAQHuzd+9ejRs3zvyckZEhSZo+fbpycnI0b948nT59WrNmzVJlZaVGjx6tLVu2yN/f3zzmtddeU1pamiZMmCBPT08lJSVp2bJll70vAIDWQ3EEAOjwxo4dK8MwLtju4eGhRYsWOSzz/q6QkBCtWbOmNdIDALQRLX5aHQAAAAB0BMwcAQCAi9uRZX4ZV3bSjYlcuriyl92dAoB2iJkjAAAAABDFEQAAAABIYlkdAADAJSv8/OLLCovONu9l4wDaBmaOAAAAAEAURwAAAAAgieIIAAAAACRRHAEAAACAJIojAAAAAJBEcQQAAAAAkniUNzqyc97o3siZN7sXRc1yZTYAAABo45g5AgAAAABRHAEAAACAJIojAAAAAJBEcQQAAAAAkiiOAAAAAEASxREAAAAASKI4AgAAAABJTr7nKCsrS2+88YY+/fRTdenSRf/zP/+jp59+WgMGDDBjzpw5owceeEBr165VTU2NEhIStHz5coWFhZkxZWVlmj17tnbs2KGAgABNnz5dWVlZ8vb+bzr5+fnKyMjQgQMHFBkZqfnz5+uuu+5qeY/bkybe0/N9nHmPT3tW+Hnn6CcAAAAuH6dmjgoKCpSamqqioiLl5uaqrq5OEydO1OnTp82Y9PR0vfXWW1q/fr0KCgp0/PhxTZkyxWyvr69XYmKiamtrtWvXLq1evVo5OTlasGCBGVNaWqrExESNGzdOJSUlmjNnju655x5t3brVBV0GAAAAgPM5NXO0ZcsWh885OTkKDQ1VcXGxxowZo6qqKq1cuVJr1qzR+PHjJUmrVq3SoEGDVFRUpLi4OG3btk0HDx7Uu+++q7CwMA0fPlyPP/64HnroIT322GPy9fXVihUrFB0drSVLlkiSBg0apPfee09Lly5VQkKCi7oOAAAAAP/VonuOqqqqJEkhISGSpOLiYtXV1Sk+Pt6MGThwoKKiolRYWChJKiws1JAhQxyW2SUkJMhut+vAgQNmzLnnaIxpPEdTampqZLfbHTYAAAAAuFTNLo4aGho0Z84cXX/99brmmmskSTabTb6+vgoODnaIDQsLk81mM2POLYwa2xvbLhZjt9v1zTffNJlPVlaWgoKCzC0yMrK5XQMAAADQCTW7OEpNTdXHH3+stWvXujKfZsvMzFRVVZW5HTt2zN0pAQAAAGhHnLrnqFFaWpo2bdqknTt3qnfv3uZ+i8Wi2tpaVVZWOswelZeXy2KxmDEffPCBw/nKy8vNtsb/Nu47NyYwMFBdunRpMic/Pz/5+fk1pzsAAAAA4NzMkWEYSktL04YNG7R9+3ZFR0c7tMfExMjHx0d5eXnmvkOHDqmsrExWq1WSZLVatX//flVUVJgxubm5CgwM1ODBg82Yc8/RGNN4DgAAAABwNadmjlJTU7VmzRq9+eab6t69u3mPUFBQkLp06aKgoCClpKQoIyNDISEhCgwM1H333Ser1aq4uDhJ0sSJEzV48GDdeeedWrx4sWw2m+bPn6/U1FRz5ufee+/Viy++qHnz5mnGjBnavn271q1bp82bN7u4+wAAAADwLadmjl566SVVVVVp7NixCg8PN7fXX3/djFm6dKluvvlmJSUlacyYMbJYLHrjjTfMdi8vL23atEleXl6yWq264447NG3aNC1atMiMiY6O1ubNm5Wbm6thw4ZpyZIleuWVV3iMNwAAAIBW49TMkWEY3xvj7++v7OxsZWdnXzCmT58+evvtty96nrFjx2rfvn3OpAcAAAAAzdai9xwBAAAAQEdBcQQAAAAAauajvAEAAHC+uLKXm31sUdQsF2YCoDmYOQIAAAAAMXMEXJKluZ81+9j0G/u7MBMAAAC0FoojAAA6gx1Z7s4AANo8ltUBAAAAgCiOAADQY489Jg8PD4dt4MCBZvuZM2eUmpqqHj16KCAgQElJSSovL3djxgCA1kBxBACApKuvvlonTpwwt/fee89sS09P11tvvaX169eroKBAx48f15QpU9yYLQCgNXDPEQAAkry9vWWxWM7bX1VVpZUrV2rNmjUaP368JGnVqlUaNGiQioqKFBcXd8Fz1tTUqKamxvxst9tdnzgAwGWYOQIAQNLhw4cVERGhvn37Kjk5WWVlZZKk4uJi1dXVKT4+3owdOHCgoqKiVFhYeNFzZmVlKSgoyNwiIyNbtQ8AgJahOAIAdHqxsbHKycnRli1b9NJLL6m0tFQ33HCDvvrqK9lsNvn6+io4ONjhmLCwMNlstoueNzMzU1VVVeZ27NixVuwFAKClWFYHAOj0Jk+ebH49dOhQxcbGqk+fPlq3bp26dOnS7PP6+fnJz8/PFSkCAC4DZo4AAPiO4OBg9e/fX0eOHJHFYlFtba0qKysdYsrLy5u8RwkA0H5RHAEA8B3V1dU6evSowsPDFRMTIx8fH+Xl5Znthw4dUllZmaxWqxuzBAC4GsvqAACd3oMPPqhbbrlFffr00fHjx/Xoo4/Ky8tLt99+u4KCgpSSkqKMjAyFhIQoMDBQ9913n6xW60WfVAcAaH8ojoALiCt7uUXHF0XNclEmAFrbF198odtvv10nT55Ur169NHr0aBUVFalXr16SpKVLl8rT01NJSUmqqalRQkKCli9f7uasAQCuRnEEAOj01q5de9F2f39/ZWdnKzs7+zJlBDRtae5nLTo+/cb+LsoE6Ji45wgAAAAAxMwRAABAm9Dc5dws4wZch5kjAAAAAFAziqOdO3fqlltuUUREhDw8PLRx40aHdsMwtGDBAoWHh6tLly6Kj4/X4cOHHWJOnTql5ORkBQYGKjg4WCkpKaqurnaI+eijj3TDDTfI399fkZGRWrx4sfO9AwAAAIBL5PSyutOnT2vYsGGaMWOGpkyZcl774sWLtWzZMq1evVrR0dH69a9/rYSEBB08eFD+/v6SpOTkZJ04cUK5ubmqq6vT3XffrVmzZmnNmjWSJLvdrokTJyo+Pl4rVqzQ/v37NWPGDAUHB2vWLKaOAQAA3IEHQqCjc7o4mjx5siZPntxkm2EYeu655zR//nzdeuutkqRXX31VYWFh2rhxo6ZOnapPPvlEW7Zs0Z49ezRq1ChJ0gsvvKCbbrpJzzzzjCIiIvTaa6+ptrZWf/jDH+Tr66urr75aJSUlevbZZymOAAAAALQKlz6QobS0VDabTfHx8ea+oKAgxcbGqrCwUFOnTlVhYaGCg4PNwkiS4uPj5enpqd27d+snP/mJCgsLNWbMGPn6+poxCQkJevrpp/Wf//xHV1xxxXnfu6amRjU1NeZnu93uyq4BAAB0PDuynAqPKztpfs2DINARufSBDDabTZIUFhbmsD8sLMxss9lsCg0NdWj39vZWSEiIQ0xT5zj3e3xXVlaWgoKCzC0yMrLlHQIAAADQaXSYR3lnZmYqIyPD/Gy32ymQAABAh+fUI8B39Gi9RIAOwKUzRxaLRZJUXl7usL+8vNxss1gsqqiocGg/e/asTp065RDT1DnO/R7f5efnp8DAQIcNAAAAAC6VS2eOoqOjZbFYlJeXp+HDh0v6dgZn9+7dmj17tiTJarWqsrJSxcXFiomJkSRt375dDQ0Nio2NNWMeeeQR1dXVycfHR5KUm5urAQMGNHm/UUe0NPczh3W9AAAAAFqX0zNH1dXVKikpUUlJiaRvH8JQUlKisrIyeXh4aM6cOXriiSf017/+Vfv379e0adMUERGh2267TZI0aNAgTZo0STNnztQHH3yg999/X2lpaZo6daoiIiIkSb/4xS/k6+urlJQUHThwQK+//rqef/55h2VzAAAAAOBKTs8c7d27V+PGjTM/NxYs06dPV05OjubNm6fTp09r1qxZqqys1OjRo7VlyxbzHUeS9NprryktLU0TJkyQp6enkpKStGzZMrM9KChI27ZtU2pqqmJiYtSzZ08tWLCAx3gDAAAAaDVOF0djx46VYRgXbPfw8NCiRYu0aNGiC8aEhISYL3y9kKFDh+pvf/ubs+kBAAAAQLO49IEMAAAAANBeURwBAAAAgCiOAAAAAEBSB3oJLAAAncKOLHdnAAAdFjNHAAAAACCKIwAAAACQxLI6oM1amvtZs45Lv7G/izMB0JEUfn7S3SnAjdr7z7+5Y2Mjxkh8H2aOAAAAAEDMHAGtrqV/5QIAAMDlwcwRAAAAAIiZI6DNiit7uXkH7ujx7X/HZbouGQAAgE6AmSMAAAAAEDNHQKtp9swPAAAA3IKZIwAAAAAQxREAAAAASGJZHdDhNL7gr+is848Q5+V4AIC2rmXL1p9xWR7omJg5AgAAAAAxcwQAAIDLbUdWsw6LKzvp4kQARxRHQAfVrGUHje9IknhPEgCg42lmUSbJJePi0lznl7yfi+XvrY/iCAAAAJdFY3HADBDaKoqj1tSCv07wSwMA0JTGh64AAFyP4giA6dz/6XLmaXdM8wNA58PLztERteniKDs7W7/97W9ls9k0bNgwvfDCC7ruuuvcnRbQKTg16J17r1Ij7llCB8XYBLRfLZp5/fxBWfs2Md45wdmVQUVRs1r0/eC8Nlscvf7668rIyNCKFSsUGxur5557TgkJCTp06JBCQ0PdnR6A79OSm14liiu0SYxNANypvT/QoT3k32aLo2effVYzZ87U3XffLUlasWKFNm/erD/84Q96+OGHz4uvqalRTU2N+bmqqkqSZLfbL0/CTTl9pvmHflPz/UFAG/HugeOuP+mB+y7afN0PQy5+/JgHXJgMnNH4e9cwDDdn4nptYWxifADcp1XGu4s4c7rapee75N89O5c0/5tcZPxtaX9a8rvzkscmow2qqakxvLy8jA0bNjjsnzZtmvHjH/+4yWMeffRRQxIbGxsbWxvZjh07dhlGjMuHsYmNjY2t/W/fNza1yZmjf//736qvr1dYWJjD/rCwMH366adNHpOZmamMjAzzc0NDg06dOqUePXrIw8PDIdZutysyMlLHjh1TYGCg6zvQCXFNXYvr6VpcT9e72DU1DENfffWVIiIi3JRd62jtscnVOsO/e/rYcXSGftJH97rUsalNFkfN4efnJz8/P4d9wcHBFz0mMDCwzf3g2juuqWtxPV2L6+l6F7qmQUFBbsim7WnO2ORqneHfPX3sODpDP+mj+1zK2OR5GfJwWs+ePeXl5aXy8nKH/eXl5bJYLG7KCgDQmTE2AUDH1yaLI19fX8XExCgvL8/c19DQoLy8PFmtVjdmBgDorBibAKDja7PL6jIyMjR9+nSNGjVK1113nZ577jmdPn3afEJQS/j5+enRRx89b6kDmo9r6lpcT9fierpeZ72mrTk2uVpn+BnRx46jM/STPrYPHobRdp+1+uKLL5ov2hs+fLiWLVum2NhYd6cFAOjEGJsAoONq08URAAAAAFwubfKeIwAAAAC43CiOAAAAAEAURwAAAAAgieIIAAAAACR10uIoOztbP/zhD+Xv76/Y2Fh98MEH7k6pXdi5c6duueUWRUREyMPDQxs3bnRoNwxDCxYsUHh4uLp06aL4+HgdPnzYPcm2A1lZWbr22mvVvXt3hYaG6rbbbtOhQ4ccYs6cOaPU1FT16NFDAQEBSkpKOu8FlPivl156SUOHDjXfzG21WvXOO++Y7VzP5nvqqafk4eGhOXPmmPu4nm3TY489Jg8PD4dt4MCB7k6rRTrD+PN9fbzrrrvO+7lOmjTJPck2U2cY9y6lj2PHjj3vZ3nvvfe6KWPndfSxttMVR6+//royMjL06KOP6sMPP9SwYcOUkJCgiooKd6fW5p0+fVrDhg1TdnZ2k+2LFy/WsmXLtGLFCu3evVvdunVTQkKCzpw5c5kzbR8KCgqUmpqqoqIi5ebmqq6uThMnTtTp06fNmPT0dL311ltav369CgoKdPz4cU2ZMsWNWbdtvXv31lNPPaXi4mLt3btX48eP16233qoDBw5I4no21549e/S73/1OQ4cOddjP9Wy7rr76ap04ccLc3nvvPXen1CKdYfz5vj5K0qRJkxx+rn/+858vY4Yt1xnGvUvpoyTNnDnT4We5ePFiN2XsvA4/1hqdzHXXXWekpqaan+vr642IiAgjKyvLjVm1P5KMDRs2mJ8bGhoMi8Vi/Pa3vzX3VVZWGn5+fsaf//xnN2TY/lRUVBiSjIKCAsMwvr1+Pj4+xvr1682YTz75xJBkFBYWuivNdueKK64wXnnlFa5nM3311VfGVVddZeTm5ho/+tGPjPvvv98wDP59tmWPPvqoMWzYMHen0Wo6w/jz3T4ahmFMnz7duPXWW92ST2vpDOPed/toGIbD79KOoiONtZ1q5qi2tlbFxcWKj48393l6eio+Pl6FhYVuzKz9Ky0tlc1mc7i2QUFBio2N5dpeoqqqKklSSEiIJKm4uFh1dXUO13TgwIGKioriml6C+vp6rV27VqdPn5bVauV6NlNqaqoSExMdrpvEv8+27vDhw4qIiFDfvn2VnJyssrIyd6fUajrT+JOfn6/Q0FANGDBAs2fP1smTJ92dUot0hnHvu31s9Nprr6lnz5665pprlJmZqa+//tod6bVYRxxrvd2dwOX073//W/X19QoLC3PYHxYWpk8//dRNWXUMNptNkpq8to1tuLCGhgbNmTNH119/va655hpJ315TX19fBQcHO8RyTS9u//79slqtOnPmjAICArRhwwYNHjxYJSUlXE8nrV27Vh9++KH27NlzXhv/Ptuu2NhY5eTkaMCAATpx4oQWLlyoG264QR9//LG6d+/u7vRcrrOMP5MmTdKUKVMUHR2to0eP6le/+pUmT56swsJCeXl5uTs9p3WGca+pPkrSL37xC/Xp00cRERH66KOP9NBDD+nQoUN644033JitczryWNupiiOgrUpNTdXHH3/c7u8LaAsGDBigkpISVVVV6S9/+YumT5+ugoICd6fV7hw7dkz333+/cnNz5e/v7+504ITJkyebXw8dOlSxsbHq06eP1q1bp5SUFDdmhpaYOnWq+fWQIUM0dOhQXXnllcrPz9eECRPcmFnzdIZx70J9nDVrlvn1kCFDFB4ergkTJujo0aO68sorL3eazdKRx9pOtayuZ8+e8vLyOu+JGeXl5bJYLG7KqmNovH5cW+elpaVp06ZN2rFjh3r37m3ut1gsqq2tVWVlpUM81/TifH191a9fP8XExCgrK0vDhg3T888/z/V0UnFxsSoqKjRy5Eh5e3vL29tbBQUFWrZsmby9vRUWFsb1bCeCg4PVv39/HTlyxN2ptIrOOv707dtXPXv2bJc/184w7l2oj02JjY2VpHb1s+zIY22nKo58fX0VExOjvLw8c19DQ4Py8vJktVrdmFn7Fx0dLYvF4nBt7Xa7du/ezbW9AMMwlJaWpg0bNmj79u2Kjo52aI+JiZGPj4/DNT106JDKysq4pk5oaGhQTU0N19NJEyZM0P79+1VSUmJuo0aNUnJysvk117N9qK6u1tGjRxUeHu7uVFpFZx1/vvjiC508ebJd/Vw7w7j3fX1sSklJiSS1q5/ld3WosdbND4S47NauXWv4+fkZOTk5xsGDB41Zs2YZwcHBhs1mc3dqbd5XX31l7Nu3z9i3b58hyXj22WeNffv2Gf/85z8NwzCMp556yggODjbefPNN46OPPjJuvfVWIzo62vjmm2/cnHnbNHv2bCMoKMjIz883Tpw4YW5ff/21GXPvvfcaUVFRxvbt2429e/caVqvVsFqtbsy6bXv44YeNgoICo7S01Pjoo4+Mhx9+2PDw8DC2bdtmGAbXs6W++4Qlrmfb9MADDxj5+flGaWmp8f777xvx8fFGz549jYqKCnen1mydYfy5WB+/+uor48EHHzQKCwuN0tJS49133zVGjhxpXHXVVcaZM2fcnfol6wzj3vf18ciRI8aiRYuMvXv3GqWlpcabb75p9O3b1xgzZoybM790HX2s7XTFkWEYxgsvvGBERUUZvr6+xnXXXWcUFRW5O6V2YceOHYak87bp06cbhvHt41R//etfG2FhYYafn58xYcIE49ChQ+5Nug1r6lpKMlatWmXGfPPNN8Yvf/lL44orrjC6du1q/OQnPzFOnDjhvqTbuBkzZhh9+vQxfH19jV69ehkTJkwwf1kbBtezpb5bHHE926af//znRnh4uOHr62v84Ac/MH7+858bR44ccXdaLdIZxp+L9fHrr782Jk6caPTq1cvw8fEx+vTpY8ycObPd/WG3M4x739fHsrIyY8yYMUZISIjh5+dn9OvXz5g7d65RVVXl3sSd0NHHWg/DMIzWnZsCAAAAgLavU91zBAAAAAAXQnEEAAAAAKI4AgAAAABJFEcAAAAAIIniCAAAAAAkURwBAAAAgCSKIwAAAACQRHEEAAAAAJIojgAAAABAEsURAAAAAEiiOAIAAAAASdL/A1YdNL5st3NAAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "def length_histogram(dataset, ax, bins=20) -> None:\n", + " en_lengths = []\n", + " de_lengths = []\n", + " for sample in dataset:\n", + " en_lengths.append(len(sample[\"en\"].split(\" \")))\n", + " de_lengths.append(len(sample[\"de\"].split(\" \")))\n", + "\n", + " ax.hist(en_lengths, alpha=0.5, bins=bins, label=\"en\")\n", + " ax.hist(de_lengths, alpha=0.5, bins=bins, label=\"de\")\n", + " ax.legend()\n", + "\n", + "\n", + "fig, axes = plt.subplots(1, 2, figsize=(10, 3))\n", + "length_histogram(train_dataset, axes[0])\n", + "length_histogram(test_dataset, axes[1])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Оставим только сравнительно короткие предложения, чтобы можно было чему-то научиться за короткое время" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5144 174\n" + ] + } + ], + "source": [ + "maxlen = 8\n", + "\n", + "\n", + "def filter_dataset(dataset, maxlen: int) -> list[dict[str, str]]:\n", + " return [\n", + " dataset[i]\n", + " for i in range(len(dataset))\n", + " if len(dataset[i][\"en\"].split(\" \")) <= maxlen\n", + " ]\n", + "\n", + "\n", + "train_filtered = filter_dataset(train_dataset, maxlen)\n", + "test_filtered = filter_dataset(test_dataset, maxlen)\n", + "\n", + "print(len(train_filtered), len(test_filtered))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Токенизация: byte-pair encoding" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Построение:\n", + "\n", + "Начинаем со словаря, состоящего из отдельных символов (начальные токены).\n", + "На каждом шаге:\n", + "1. Оцениваем частоту всех пар токенов внутри слов, находим самую частую\n", + "2. Добавляем её в список токенов и в таблицу слияний\n", + "3. Останавливаемся, когда достигаем максимального размера словаря\n", + "\n", + "\n", + "Применение:\n", + "\n", + "1. Разбиваем текст на символы\n", + "2. Находим первое возможное слияние в таблице и применяем его\n", + "3. Останавливаемся, когда дальнейшие слияния невозможны" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Реализаций много, мы будем использовать токенизатор из библиотеки `transformers`, где помимо самого подготовленного токенизатора (`sentencepiece.SentencePieceProcessor`) много полезных методов для кодирования и декодирования.\n", + "\n", + "Добавим при создании новый токен, который будет указывать на начало перевода" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "323ca854de9d4b50a4261a7860f01bd4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "tokenizer_config.json: 0%| | 0.00/2.32k [00:00\", legacy=False\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Размер словаря: 32101\n" + ] + } + ], + "source": [ + "print(\"Размер словаря: \", len(tokenizer))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Посмотрим на примеры токенов, в них встречаются и целые слова из разных языков, и числительные, и знаки препинания:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('▁contractor', 8131),\n", + " ('▁outside', 1067),\n", + " ('▁China', 1473),\n", + " ('▁joyful', 26128),\n", + " ('welt', 9304),\n", + " (\"'\", 31),\n", + " ('▁Adam', 7124),\n", + " ('▁gently', 10772),\n", + " ('▁Solid', 12974),\n", + " ('▁Reiki', 26420)]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import random\n", + "\n", + "random.choices(list(tokenizer.get_vocab().items()), k=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Токенизатор в работе: кодирование и декодирование предложений на немецком и английском:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[11280, 16815, 7838, 15, 16282, 436, 256, 8671, 35, 16, 74, 13271, 2221, 49, 21162, 3992, 5, 1]\n", + "Zwei junge weiße Männer sind im Freien in der Nähe vieler Büsche.\n", + "[2759, 1021, 6, 1945, 5069, 7, 33, 1067, 1084, 186, 3, 30271, 5, 1]\n", + "Two young, White males are outside near many bushes.\n" + ] + } + ], + "source": [ + "encoded_german = tokenizer.encode(train_dataset[0][\"de\"])\n", + "encoded_english = tokenizer.encode(train_dataset[0][\"en\"])\n", + "print(encoded_german)\n", + "print(tokenizer.decode(encoded_german))\n", + "print(encoded_english)\n", + "print(tokenizer.decode(encoded_english))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Об упаковке в батчи можно больше не беспокоиться — токенизатор умеет обрабатывать сразу пачку примеров" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([4, 19])\n", + "KeysView({'input_ids': tensor([[ 2759, 1021, 6, 1945, 5069, 7, 33, 1067, 1084, 186,\n", + " 3, 30271, 5, 1, 0, 0, 0, 0, 0],\n", + " [ 3, 8656, 1076, 16, 614, 3, 547, 7, 33, 2699,\n", + " 3, 9, 6079, 3197, 15, 63, 358, 5, 1],\n", + " [ 71, 385, 3202, 11908, 139, 3, 9, 5726, 577, 1840,\n", + " 5, 1, 0, 0, 0, 0, 0, 0, 0],\n", + " [ 71, 388, 16, 3, 9, 1692, 8677, 19, 4125, 30,\n", + " 3, 9, 15812, 2327, 3, 9, 2034, 5, 1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n", + " [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n", + " [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n", + " [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])})\n" + ] + } + ], + "source": [ + "batch = [train_dataset[i][\"en\"] for i in range(4)]\n", + "\n", + "encoded_batch = tokenizer.batch_encode_plus(\n", + " batch, padding=\"longest\", return_tensors=\"pt\"\n", + ")\n", + "print(encoded_batch[\"input_ids\"].shape)\n", + "print(encoded_batch.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Возвращается два значения: `input_ids` — это наши токены, а `attention_mask` — это тензор, равный по размеру батчу токенов, где на месте `pad_token` стоят нули, в остальных позициях — единицы. Это нам понадобится потом." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "А ещё можно кодировать сразу входные и выходные данные:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KeysView({'input_ids': tensor([[ 2759, 1021, 6, 1945, 5069, 7, 33, 1067, 1084, 186,\n", + " 3, 30271, 5, 32100, 1, 0, 0, 0, 0, 0],\n", + " [ 3, 8656, 1076, 16, 614, 3, 547, 7, 33, 2699,\n", + " 3, 9, 6079, 3197, 15, 63, 358, 5, 32100, 1],\n", + " [ 71, 385, 3202, 11908, 139, 3, 9, 5726, 577, 1840,\n", + " 5, 32100, 1, 0, 0, 0, 0, 0, 0, 0],\n", + " [ 71, 388, 16, 3, 9, 1692, 8677, 19, 4125, 30,\n", + " 3, 9, 15812, 2327, 3, 9, 2034, 5, 32100, 1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n", + " [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n", + " [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n", + " [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]), 'labels': tensor([[11280, 16815, 7838, 15, 16282, 436, 256, 8671, 35, 16,\n", + " 74, 13271, 2221, 49, 21162, 3992, 5, 1, 0, 0,\n", + " 0, 0],\n", + " [11093, 49, 15, 16282, 181, 10956, 17757, 35, 1953, 23,\n", + " 35, 35, 236, 389, 13835, 7, 5672, 3734, 5, 1,\n", + " 0, 0],\n", + " [ 890, 5683, 7, 21644, 3, 157, 16024, 17, 16, 236,\n", + " 3012, 2989, 403, 8589, 5, 1, 0, 0, 0, 0,\n", + " 0, 0],\n", + " [ 890, 6362, 16, 665, 3, 16193, 35, 216, 51, 26,\n", + " 4186, 219, 645, 24484, 64, 474, 172, 17, 236, 16814,\n", + " 5, 1]])})\n" + ] + } + ], + "source": [ + "inputs = [train_dataset[i][\"en\"] + tokenizer.bos_token for i in range(4)]\n", + "targets = [train_dataset[i][\"de\"] for i in range(4)]\n", + "\n", + "encoded_batch = tokenizer(\n", + " inputs, text_target=targets, padding=\"longest\", return_tensors=\"pt\"\n", + ")\n", + "print(encoded_batch.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Используем это в `collate_fn` для сборки батчей:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def collate_fn(\n", + " tokenizer: T5Tokenizer, batch: list[tuple[str, str]]\n", + ") -> tuple[Tensor, Tensor]:\n", + " prompt = tokenizer.bos_token\n", + " inputs, targets = zip(*[(pair[\"de\"], prompt + pair[\"en\"]) for pair in batch])\n", + " encoded_batch = tokenizer(\n", + " inputs, text_target=targets, padding=\"longest\", return_tensors=\"pt\"\n", + " )\n", + " return encoded_batch" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'input_ids': tensor([[11280, 16815, 7838, 15, 16282, 436, 256, 8671, 35, 16,\n", + " 74, 13271, 2221, 49, 21162, 3992, 5, 1, 0, 0,\n", + " 0, 0],\n", + " [11093, 49, 15, 16282, 181, 10956, 17757, 35, 1953, 23,\n", + " 35, 35, 236, 389, 13835, 7, 5672, 3734, 5, 1,\n", + " 0, 0],\n", + " [ 890, 5683, 7, 21644, 3, 157, 16024, 17, 16, 236,\n", + " 3012, 2989, 403, 8589, 5, 1, 0, 0, 0, 0,\n", + " 0, 0],\n", + " [ 890, 6362, 16, 665, 3, 16193, 35, 216, 51, 26,\n", + " 4186, 219, 645, 24484, 64, 474, 172, 17, 236, 16814,\n", + " 5, 1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0],\n", + " [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0],\n", + " [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],\n", + " [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]), 'labels': tensor([[32100, 382, 210, 32, 1021, 6, 1945, 5069, 7, 33,\n", + " 1067, 1084, 186, 3, 30271, 5, 1, 0, 0, 0],\n", + " [32100, 8656, 1076, 16, 614, 3, 547, 7, 33, 2699,\n", + " 3, 9, 6079, 3197, 15, 63, 358, 5, 1, 0],\n", + " [32100, 188, 385, 3202, 11908, 139, 3, 9, 5726, 577,\n", + " 1840, 5, 1, 0, 0, 0, 0, 0, 0, 0],\n", + " [32100, 188, 388, 16, 3, 9, 1692, 8677, 19, 4125,\n", + " 30, 3, 9, 15812, 2327, 3, 9, 2034, 5, 1]])}\n" + ] + } + ], + "source": [ + "batch = [train_dataset[i] for i in range(4)]\n", + "encoded_batch = collate_fn(tokenizer, batch)\n", + "print(encoded_batch)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([4, 22])\n", + "torch.Size([4, 22])\n", + "torch.Size([4, 20])\n" + ] + } + ], + "source": [ + "print(encoded_batch[\"input_ids\"].shape)\n", + "print(encoded_batch[\"attention_mask\"].shape)\n", + "print(encoded_batch[\"labels\"].shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Всё готово для получения минибатчей из датасетов:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "train_loader = DataLoader(\n", + " train_filtered,\n", + " batch_size=32,\n", + " shuffle=True,\n", + " collate_fn=lambda batch: collate_fn(tokenizer, batch),\n", + ")\n", + "test_loader = DataLoader(\n", + " test_filtered,\n", + " batch_size=32,\n", + " shuffle=False,\n", + " collate_fn=lambda batch: collate_fn(tokenizer, batch),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Задание 1 (2 балла). DataModule\n", + "\n", + "Реализуйте подкласс `lightning.LightningDataModule` для работы с обучающим и тестовым датасетами" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "from lightning.pytorch.utilities.types import EVAL_DATALOADERS, TRAIN_DATALOADERS\n", + "\n", + "\n", + "class Multi30kDataset(L.LightningDataModule):\n", + " train_dataset: Dataset\n", + " test_dataset: Dataset\n", + " tokenizer: T5Tokenizer\n", + "\n", + " def __init__(self, maxlen: int = 0, batch_size: int = 32) -> None:\n", + " self.batch_size = batch_size\n", + "\n", + " train_dataset = load_dataset(\"bentrevett/multi30k\", split=\"train\")\n", + " test_dataset = load_dataset(\"bentrevett/multi30k\", split=\"test\")\n", + "\n", + " self.train_dataset = filter_dataset(train_dataset, maxlen)\n", + " self.test_dataset = filter_dataset(test_dataset, maxlen)\n", + "\n", + " self.tokenizer = T5Tokenizer.from_pretrained(\n", + " \"t5-small\", padding_size=\"right\", bos_token=\"\", legacy=False\n", + " )\n", + " \n", + "\n", + " def train_dataloader(self) -> TRAIN_DATALOADERS:\n", + " return DataLoader(\n", + " self.train_dataset,\n", + " batch_size=self.batch_size,\n", + " shuffle=True,\n", + " collate_fn=lambda batch: collate_fn(self.tokenizer, batch),\n", + " )\n", + "\n", + " def test_dataloader(self) -> EVAL_DATALOADERS:\n", + " return DataLoader(\n", + " self.test_dataset,\n", + " batch_size=self.batch_size,\n", + " shuffle=False,\n", + " collate_fn=lambda batch: collate_fn(self.tokenizer, batch),\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "multi30k = Multi30kDataset(maxlen=8, batch_size=12)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Задание 2 (4 балла). Обучение T5" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "paper: https://arxiv.org/abs/1910.10683" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Обучите `t5-small` на наших данных, длительность обучения — на ваше усмотрение. В [документации](https://huggingface.co/docs/transformers/model_doc/t5) вы найдёте примеры использования, которые помогут вам разобраться в интерфейсе модели." + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [], + "source": [ + "t5 = T5ForConditionalGeneration.from_pretrained(\"t5-small\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Обернём в `LightningModule`" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [], + "source": [ + "import lightning as L\n", + "from lightning.pytorch.utilities.types import STEP_OUTPUT, OptimizerLRScheduler\n", + "\n", + "\n", + "class Seq2Seq(L.LightningModule):\n", + " def __init__(\n", + " self,\n", + " model: nn.Module,\n", + " tokenizer: T5Tokenizer,\n", + " lr: float = 0.01,\n", + " ) -> None:\n", + " super().__init__()\n", + " self.save_hyperparameters()\n", + " self.model = model\n", + " self.tokenizer = tokenizer\n", + " self.learning_rate = lr\n", + " \n", + "\n", + " def training_step(self, batch: dict[str, Tensor], batch_idx: int) -> STEP_OUTPUT:\n", + " outputs = self.model(\n", + " input_ids=batch['input_ids'], \n", + " attention_mask=batch['attention_mask'], \n", + " labels=batch['labels']\n", + " )\n", + " loss = outputs.loss\n", + " self.log(\"train_loss\", loss, on_epoch=True, on_step=False)\n", + " return loss\n", + "\n", + " def configure_optimizers(self) -> OptimizerLRScheduler:\n", + " return torch.optim.AdamW(self.model.parameters(), lr=self.learning_rate)" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "💡 Tip: For seamless cloud uploads and versioning, try installing [litmodels](https://pypi.org/project/litmodels/) to enable LitModelCheckpoint, which syncs automatically with the Lightning model registry.\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params | Mode\n", + "------------------------------------------------------------\n", + "0 | model | T5ForConditionalGeneration | 60.5 M | eval\n", + "------------------------------------------------------------\n", + "60.5 M Trainable params\n", + "0 Non-trainable params\n", + "60.5 M Total params\n", + "242.026 Total estimated model params size (MB)\n", + "0 Modules in train mode\n", + "277 Modules in eval mode\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d5c5349f26ae47e4942c47a23b15ee1c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Training: | | 0/? [00:00…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`Trainer.fit` stopped: `max_epochs=15` reached.\n" + ] + } + ], + "source": [ + "from pytorch_lightning.loggers import TensorBoardLogger\n", + "\n", + "trainer = L.Trainer(\n", + " accelerator=\"auto\", max_epochs=15, logger=TensorBoardLogger(name=\"tb_logs\", save_dir=\".\")\n", + ")\n", + "seq2seq = Seq2Seq(t5, multi30k.tokenizer, lr=0.001)\n", + "trainer.fit(model=seq2seq, train_dataloaders=multi30k.train_dataloader())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Задание 3 (2 балла). Генерация перевода\n", + "\n", + "Сгенерируйте перевод для одного батча из тестового датасета (используйте метод `t5.generate()`), используя разные стратегии декодирования:\n", + "- *greedy decoding*\n", + "- *multinomial sampling*\n", + "- *beam-search multinomial sampling*\n", + "\n", + "Эти стратегии (а также другие стратегии декодирования) можно задавать через `transformers.GenerationConfig`, который можно передать в метод `.generate()` нашей модели.\n", + "Выберите параметры, которые, на ваш взгляд, работают лучше всего." + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import GenerationConfig\n", + "\n", + "confs = {\n", + " \"greedy decoding\": GenerationConfig(\n", + " max_new_tokens=20,\n", + " do_sample=False\n", + " ),\n", + " \"multinomial sampling\": GenerationConfig(\n", + " max_new_tokens=20,\n", + " do_sample=True, \n", + " temperature=1.0,\n", + " top_p=0.9, \n", + " top_k=50 \n", + " ),\n", + " \"beam-search_multinomial_sampling\": GenerationConfig(\n", + " max_new_tokens=20,\n", + " num_beams=4, \n", + " do_sample=True,\n", + " temperature=0.8,\n", + " top_p=0.9,\n", + " num_return_sequences=1\n", + " )\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "**greedy decoding**" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deutsch: Leute Reparieren das Dach eines Hauses.\n", + "English: People are fixing the roof of a house.\n", + "Translate: People are cleaning a house.\n", + "\n", + "Deutsch: Ein Typ arbeitet an einem Gebäude.\n", + "English: A guy works on a building.\n", + "Translate: A guy working in a building.\n", + "\n", + "Deutsch: Drei Leute sitzen in einer Höhle.\n", + "English: Three people sit in a cave.\n", + "Translate: Three people are sitting in a cave.\n", + "\n", + "Deutsch: Leute, die vor einem Gebäude stehen.\n", + "English: People standing outside of a building.\n", + "Translate: People standing in front of a building.\n", + "\n", + "Deutsch: Ein Mann schneidet ste von Bäumen.\n", + "English: A man cutting branches of trees.\n", + "Translate: A man is cutting tree branches.\n", + "\n", + "Deutsch: Frauen, die traditionelle Kleidung tragen, spielen das Leben Einheimischer nach.\n", + "English: Women, wearing traditional clothing, are reenacting native life.\n", + "Translate: Women wearing traditional clothing playing the drums.\n", + "\n", + "Deutsch: Ein Kind planscht im Wasser.\n", + "English: A child is splashing in the water\n", + "Translate: A kid splashes in the water.\n", + "\n", + "Deutsch: Eine schöne Frau spielt auf einer Harfe.\n", + "English: A pretty woman plays a harpsichord.\n", + "Translate: A beautiful woman plays on a harp.\n", + "\n", + "Deutsch: Die junge Dame sieht auf die Pizza.\n", + "English: The young lady is looking at the pizza.\n", + "Translate: The young lady is checking on pizza.\n", + "\n", + "Deutsch: Leute sitzen in einem Zug.\n", + "English: People sit inside a train.\n", + "Translate: People are sitting on a train.\n", + "\n", + "Deutsch: Ein kleines Kind kocht mit einer anderen Person.\n", + "English: A toddler is cooking with another person.\n", + "Translate: A small child cooking with another person.\n", + "\n", + "Deutsch: Ein Mann bereitet am Herd Essen zu.\n", + "English: A man cooking food on the stove.\n", + "Translate: A man is preparing food for the stove.\n", + "\n" + ] + }, + { + "data": { + "text/markdown": [ + "**multinomial sampling**" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deutsch: Leute Reparieren das Dach eines Hauses.\n", + "English: People are fixing the roof of a house.\n", + "Translate: People are fixing a house.\n", + "\n", + "Deutsch: Ein Typ arbeitet an einem Gebäude.\n", + "English: A guy works on a building.\n", + "Translate: A guy working in a building.\n", + "\n", + "Deutsch: Drei Leute sitzen in einer Höhle.\n", + "English: Three people sit in a cave.\n", + "Translate: Three people are sitting in a cave.\n", + "\n", + "Deutsch: Leute, die vor einem Gebäude stehen.\n", + "English: People standing outside of a building.\n", + "Translate: People standing in front of a building.\n", + "\n", + "Deutsch: Ein Mann schneidet ste von Bäumen.\n", + "English: A man cutting branches of trees.\n", + "Translate: A man is cutting tree bark from trees.\n", + "\n", + "Deutsch: Frauen, die traditionelle Kleidung tragen, spielen das Leben Einheimischer nach.\n", + "English: Women, wearing traditional clothing, are reenacting native life.\n", + "Translate: Women wearing traditional clothing play the game.\n", + "\n", + "Deutsch: Ein Kind planscht im Wasser.\n", + "English: A child is splashing in the water\n", + "Translate: A kid splashes in the water.\n", + "\n", + "Deutsch: Eine schöne Frau spielt auf einer Harfe.\n", + "English: A pretty woman plays a harpsichord.\n", + "Translate: A lovely woman plays on a harp.\n", + "\n", + "Deutsch: Die junge Dame sieht auf die Pizza.\n", + "English: The young lady is looking at the pizza.\n", + "Translate: The young lady is checking out pizza.\n", + "\n", + "Deutsch: Leute sitzen in einem Zug.\n", + "English: People sit inside a train.\n", + "Translate: People are sitting on a train.\n", + "\n", + "Deutsch: Ein kleines Kind kocht mit einer anderen Person.\n", + "English: A toddler is cooking with another person.\n", + "Translate: A small child cooking with another person.\n", + "\n", + "Deutsch: Ein Mann bereitet am Herd Essen zu.\n", + "English: A man cooking food on the stove.\n", + "Translate: A man is preparing food for her.\n", + "\n" + ] + }, + { + "data": { + "text/markdown": [ + "**beam-search_multinomial_sampling**" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deutsch: Leute Reparieren das Dach eines Hauses.\n", + "English: People are fixing the roof of a house.\n", + "Translate: People are cleaning a house.\n", + "\n", + "Deutsch: Ein Typ arbeitet an einem Gebäude.\n", + "English: A guy works on a building.\n", + "Translate: A guy working in a building.\n", + "\n", + "Deutsch: Drei Leute sitzen in einer Höhle.\n", + "English: Three people sit in a cave.\n", + "Translate: Three people are sitting in a cave.\n", + "\n", + "Deutsch: Leute, die vor einem Gebäude stehen.\n", + "English: People standing outside of a building.\n", + "Translate: People standing in front of a building.\n", + "\n", + "Deutsch: Ein Mann schneidet ste von Bäumen.\n", + "English: A man cutting branches of trees.\n", + "Translate: A man is cutting tree branches.\n", + "\n", + "Deutsch: Frauen, die traditionelle Kleidung tragen, spielen das Leben Einheimischer nach.\n", + "English: Women, wearing traditional clothing, are reenacting native life.\n", + "Translate: Women wearing traditional clothing playing the drums.\n", + "\n", + "Deutsch: Ein Kind planscht im Wasser.\n", + "English: A child is splashing in the water\n", + "Translate: A kid splashes in the water.\n", + "\n", + "Deutsch: Eine schöne Frau spielt auf einer Harfe.\n", + "English: A pretty woman plays a harpsichord.\n", + "Translate: A beautiful woman plays on a harp.\n", + "\n", + "Deutsch: Die junge Dame sieht auf die Pizza.\n", + "English: The young lady is looking at the pizza.\n", + "Translate: The young lady is checking on pizza.\n", + "\n", + "Deutsch: Leute sitzen in einem Zug.\n", + "English: People sit inside a train.\n", + "Translate: People are sitting on a train.\n", + "\n", + "Deutsch: Ein kleines Kind kocht mit einer anderen Person.\n", + "English: A toddler is cooking with another person.\n", + "Translate: A small child cooking with another person.\n", + "\n", + "Deutsch: Ein Mann bereitet am Herd Essen zu.\n", + "English: A man cooking food on the stove.\n", + "Translate: A man is preparing food on the stove.\n", + "\n" + ] + } + ], + "source": [ + "from IPython.display import Markdown, display\n", + "\n", + "batch = next(iter(multi30k.test_dataloader()))\n", + "\n", + "for strategy, gen_conf in confs.items():\n", + " preds = t5.generate(batch[\"input_ids\"], generation_config=gen_conf)\n", + " display(Markdown(f\"**{strategy}**\"))\n", + " Deutsch = multi30k.tokenizer.batch_decode(batch[\"input_ids\"], skip_special_tokens=True)\n", + " English = multi30k.tokenizer.batch_decode(batch[\"labels\"], skip_special_tokens=True)\n", + " Translate = multi30k.tokenizer.batch_decode(preds, skip_special_tokens=True)\n", + " for i in range(len(Deutsch)):\n", + " print(f\"Deutsch: {Deutsch[i]}\\nEnglish: {English[i]}\\nTranslate: {Translate[i]}\\n\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}