knowledge_distillation_for_squeezenet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import sys\n",
    "sys.path.append('utils/')\n",
    "\n",
    "import keras\n",
    "from keras import optimizers\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
    "\n",
    "# use non standard flow_from_directory\n",
    "from image_preprocessing_ver2 import ImageDataGenerator\n",
    "# it outputs y_batch that contains onehot targets and logits\n",
    "# logits came from xception\n",
    "\n",
    "from keras.models import Model\n",
    "from keras.layers import Lambda, concatenate, Activation\n",
    "from keras.losses import categorical_crossentropy as logloss\n",
    "from keras.metrics import categorical_accuracy, top_k_categorical_accuracy\n",
    "from keras import backend as K\n",
    "\n",
    "from squeezenet import SqueezeNet, preprocess_input\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_dir = '/home/ubuntu/data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_logits = np.load(data_dir + 'train_logits.npy')[()]\n",
    "val_logits = np.load(data_dir + 'val_logits.npy')[()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 25600 images belonging to 256 classes.\n",
      "Found 5120 images belonging to 256 classes.\n"
     ]
    }
   ],
   "source": [
    "data_generator = ImageDataGenerator(\n",
    "    data_format='channels_last',\n",
    "    preprocessing_function=preprocess_input\n",
    ")\n",
    "\n",
    "# note: i'm also passing dicts of logits\n",
    "train_generator = data_generator.flow_from_directory(\n",
    "    data_dir + 'train', train_logits,\n",
    "    target_size=(299, 299),\n",
    "    batch_size=64\n",
    ")\n",
    "\n",
    "val_generator = data_generator.flow_from_directory(\n",
    "    data_dir + 'val', val_logits,\n",
    "    target_size=(299, 299),\n",
    "    batch_size=64\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Show effect of the temperature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def softmax(x):\n",
    "    return np.exp(x)/np.exp(x).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get a random batch\n",
    "pictures, labels_and_logits = train_generator.next()\n",
    "onehot_target, logits = labels_and_logits[:, :256], labels_and_logits[:, 256:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAFACAYAAAASxGABAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcHFd57//P09ss2jWWjKyRrB1JtrCwFBnLNsFcOazB\ngA0xyQ1wwXH4Bf8SILmBLD9jyH0lJJgtQBKMIBCS4GsMTkQwEIJZjGRsy1hmZElGI1lCkhVL1jrS\nbN1dz++Pqu6pGfXM9Mx0T8/yffNquurUqaqnu9rTj845fcrcHRERERGpnUStAxARERGZ7JSQiYiI\niNSYEjIRERGRGlNCJiIiIlJjSshEREREakwJmYiIiEiNKSETERERqTElZCIiIiI1poRMREREpMZS\ntQ5gqC666CJftGhRrcMQERERGdTjjz/+vLvPGazeuEvIFi1axPbt22sdhoiIiMigzOxgOfXUZSki\nIiJSY0rIRERERGpMCZmIiIhIjY27MWSlZLNZDh8+TGdnZ61DGTX19fU0NzeTTqdrHYqIiIiM0IRI\nyA4fPsy0adNYtGgRZlbrcKrO3Tlx4gSHDx9m8eLFtQ5HRERERmhCdFl2dnbS1NQ0KZIxADOjqalp\nUrUIioiITGQTIiEDJk0yVjDZXq+IiMhENmESMhEREZHxSgmZiIiISI0pIRMRERGpgg98/edl11VC\nViGbN29m7dq1rF27lkQiUVx+73vfO+RjveMd72Du3LlcfvnlVYhUREREqq0zm+cbTxwpu74Ssgq5\n9dZb2bFjB9/61rdYsGABO3bsYMeOHXziE58Y8rHe/va3853vfKcKUYqIiMho2H7gFN25oOz6Ssgq\nbOfOnaxZs2ZEx3jpS1/K7NmzKxSRiIiIjLat+54fUv0JMTFs3KIPfKsqxz3wkdeUVa+lpaXfrsbr\nrruOtra2C8rvuusuNm3aNKL4REREZOzY1jrJE7Ja27lzJzfccEPJbQ899NAoRyMiIiKj7UxHlpYj\nZ0glyp8zdMIlZOW2ZFVLS0tLvwP51UImIiIy8f10/wkCh3WXzmRfmftUNSEzs1cCnwKSwGZ3/0if\n7Z8Aro9WG4G57j6zmjFVUxAE7N27l1WrVpXcrhYyERGRia/QXblx6UXcV+Y+VRvUb2ZJ4LPAq4DV\nwFvMbHW8jru/193Xuvta4NPAN6oVz2hobW2lubmZTCYzouO85S1v4eqrr+bpp5+mubmZL3zhCxWK\nUERERKpt674TAFyz7KKy96lmC9kGoNXd9wOY2T3AjcCufuq/BfhgFeOpuhUrVrBrV38vr3xf/epX\nKxCNiIiIjLbnznbSeuwcDekkaxeU3+lXzWkv5gOHYuuHo7ILmNmlwGLgwX6232Zm281s+/Hjxyse\nqIiIiEglbIumu9iweDaZVPlp1liZh+wW4D53z5fa6O53u/t6d18/Z86cUQ5NREREpDxbWwvdlU1D\n2q+aCdkRYEFsvTkqK+UWQP10IiIiMm65e68B/UNRzYTsMWC5mS02swxh0rWlbyUzWwnMAh6uYiwi\nIiIiVXXgRDvPnulkVmOa1fOmD2nfqiVk7p4Dbge+C+wG7nX3p8zsw2b2uljVW4B73N2rFYuIiIhI\ntW2NWseuXtpEYgiTwkKV5yFz9weAB/qU3dFn/c5qxiAiIiIyGgoD+ofaXQljZ1C/iIiIyLgVBM7D\nw5h/rEAJmYiIiMgI7Tp6llPtWebPbGBRU+OQ91dCViGbN29m7dq1rF27lkQiUVzu776W/Tl06BDX\nX389q1ev5rLLLuNTn/pUlSIWERGRSunprmzCbGjjx2AC3ly8Vm699VZuvfVWjhw5wsaNG9mxY8ew\njpNKpfjYxz7GlVdeSVtbG+vWreOGG25g9erVg+8sIiIiNdEz/9jQuytBLWQVt3PnTtasWTPs/efN\nm8eVV14JwLRp01i1ahVHjvQ3fZuIiIjUWncu4NFnTgJhC9lwTLwWsjtnVOm4Z8qq1tLSwuWXX15y\n23XXXUdbW9sF5XfddRebNm26oPzAgQM88cQTXHXVVUOLVUREREbNjkOn6cjmWT53KnOn1w/rGBMv\nIauxnTt3csMNN5Tc9tBDD5V9nHPnznHTTTfxyU9+kunThza5nIiIiIyen0Tzjw23uxImYkJWZktW\ntbS0tPQ7kL/cFrJsNstNN93Eb/3Wb/HGN76xarGKiIjIyPXcLml43ZUwEROyGgqCgL1797Jq1aqS\n28tpIXN33vnOd7Jq1Sre9773VTpEERERqaDzXTl2HDpNwuCqJcNPyDSov4JaW1tpbm4mk8kM+xhb\nt27lK1/5Cg8++GBx6owHHnhg8B1FRERk1D36zElygbOmeSYzGtLDPo5ayCpoxYoV7Nq1a0THuPba\na9FtPUVERMaHwv0rrxlBdyWohUxERERk2LaO4HZJcUrIRERERIbhxLkudh89SyaVYN2ls0Z0LCVk\nIiIiIsPw8P6wdWz9pbOoTydHdCwlZCIiIiLDMNLbJcUpIRMREREZhvgNxUdKCZmIiIjIEB0+1c7B\nE+1Mq0uxZv7Ib9uohExERERkiLZF3ZVXLWkilRx5OqWETERERGSItu4r3L9y5N2VoISsYjZv3lyc\nWT+RSBSX+7uvZX86OzvZsGEDV1xxBZdddhkf/OAHqxSxiIiIDIe7s61C848VaKb+Crn11lu59dZb\nOXLkCBs3bmTHjh3DOk5dXR0PPvggU6dOJZvNcu211/KqV72Kl7zkJRWOWERERIZj77FzHG/rYs60\nOpbPnVqRY6qFrMJ27tzJmjVrhr2/mTF1anhxs9ks2WwWM6tUeCIiIjJChdslbVzaVLHv6AnXQrbm\ny8NPhgbS8raW8uq1tHD55ZeX3HbdddfR1tZ2Qfldd93Fpk2biuv5fJ5169bR2trKu9/9bq666qrh\nBS0iIiIVV5x/bGlluithAiZktbZz505uuOGGktseeuihso6RTCbZsWMHp0+f5g1veAM7d+7sN8kT\nERGR0ZPLBzwSzdB/zXIlZP0qtyWraudvael3IH+5LWQFM2fO5Prrr+c73/mOEjIREZExoOXIGdq6\ncixqamT+zIaKHbeqCZmZvRL4FJAENrv7R0rUeTNwJ+DAk+7+m9WMqZqCIGDv3r2sWrWq5PZyWsiO\nHz9OOp1m5syZdHR08L3vfY/3v//9lQ5VREREhqHw68qNFfp1ZUHVEjIzSwKfBW4ADgOPmdkWd98V\nq7Mc+BPgGnc/ZWZzqxXPaGhtbaW5uZlMJjPsYxw9epS3ve1t5PN5giDgzW9+M6997WsrGKWIiIgM\nV2FAfyXHj0F1W8g2AK3uvh/AzO4BbgR2xer8DvBZdz8F4O7HqhhP1a1YsYJdu3YNXnEAL3rRi3ji\niScqFJGIiIhUSmc2z/aDpwC4ugL3r4yr5rQX84FDsfXDUVncCmCFmW01s59GXZwXMLPbzGy7mW0/\nfvx4lcIVERER6d/jB0/RnQtYPW86s6cMvzeslFrPQ5YClgMvA94CfN7MZvat5O53u/t6d18/Z86c\nUQ5RREREJNZdWaHbJcVVMyE7AiyIrTdHZXGHgS3unnX3Z4BfECZoIiIiImNKcULYCg/oh+omZI8B\ny81ssZllgFuALX3q/Bth6xhmdhFhF+b+KsYkIiIiMmRnOrK0HDlDKmFsWDS74sevWkLm7jngduC7\nwG7gXnd/ysw+bGavi6p9FzhhZruAHwD/291PVCsmERERkeH46f4TBA4vXjiTKXWV/01kVechc/cH\ngAf6lN0RW3bgfdFDREREZEzaVrx/ZeW7K6H2g/pFRERExryt0YSw11Rh/BgoIRMREREZ0HNnO2k9\ndo6GdJK1Cy6YDKIilJBVyObNm1m7di1r164lkUgUl/u7r2V/nn766eK+a9euZfr06Xzyk5+sUtQi\nIiIymG37wu7KDYtnk0lVJ3WacDcXr5Vbb72VW2+9lSNHjrBx40Z27NgxrOO88IUvLO6bz+eZP38+\nb3jDGyoZqoiIiAzB1tZCd2Xl5x8rUAtZhe3cuZM1a9ZU5Fjf//73Wbp0KZdeemlFjiciIiJD4+5V\nH9APE7CFbPfKVVU57qo9u8uq19LSwuWXX15y23XXXUdbW9sF5XfddRebNm26oPyee+7hLW95y9AC\nFRERkYo5cKKdZ890Mqsxzep506t2ngmXkNXazp07ueGGG0pue+ihh8o+Tnd3N1u2bOGv/uqvKhWa\niIiIDFFhdv6rlzaRSFjVzjPhErJyW7KqpaWlpd+B/ENpIfv2t7/NlVdeycUXX1yVOEVERGRwhQH9\n1eyuhAmYkNVSEATs3buXVatKd5sOpYXsq1/9qrorRUREaigInIerPP9YgQb1V1BrayvNzc1kMpkR\nHef8+fN873vf441vfGOFIhMREZGh2nX0LKfas1wyo55FTY1VPZdayCpoxYoV7Nq1a8THmTJlCidO\n6JaeIiIitVTsrlx2EWbVGz8GaiETERERKWk05h8rUEImIiIi0kd3LuDRZ04CcE2VB/TDBErI3L3W\nIYyqyfZ6RURERtOOQ6fpyOZZPncqc6fXV/18EyIhq6+v58SJE5MmSXF3Tpw4QX199T8gIiIik1Fh\n/rFq/7qyYEIM6m9ububw4cMcP3681qGMmvr6epqbm2sdhoiIyITUM/9Y9cePwQRJyNLpNIsXL651\nGCIiIjIBnO/K8cQvT5MwuGrJ6CRkE6LLUkRERKRSHj1wklzgrGmeyYyG9KicUwmZiIiISMy2wvix\nUequBCVkIiIiIr30zD82OgP6QQmZiIiISNGJc13sOnqWTCrBuktnjdp5lZCJiIiIRB7eH7aOrb90\nFvXp5KidVwmZiIiISKQW3ZWghExERESkaLTnHytQQiYiIiICHD7VzsET7UyrS7Fm/oxRPXdVEzIz\ne6WZPW1mrWb2gRLb325mx81sR/S4tZrxiIiIiPRnW9RdedWSJlLJ0W2zqtpM/WaWBD4L3AAcBh4z\nsy3uvqtP1f/r7rdXKw4RERGRcmzdV7h/5eh2V0J1W8g2AK3uvt/du4F7gBureD4RERGRYXF3tu2r\nzYB+qG5CNh84FFs/HJX1dZOZ/dzM7jOzBaUOZGa3mdl2M9s+mW4gLiIiIqNj77FzHG/rYs60OpbP\nnTrq56/1oP5vAovc/UXA94Avl6rk7ne7+3p3Xz9nzpxRDVBEREQmvq2tPb+uNLNRP381E7IjQLzF\nqzkqK3L3E+7eFa1uBtZVMR4RERGRkorzjy0d/e5KqG5C9hiw3MwWm1kGuAXYEq9gZvNiq68Ddlcx\nHhEREZEL5PIBj0Qz9G+swYB+qOKvLN09Z2a3A98FksAX3f0pM/swsN3dtwC/b2avA3LASeDt1YpH\nREREpJSWI2do68pxaVMjzbMaaxJD1RIyAHd/AHigT9kdseU/Af6kmjGIiIiIDKTw68qNNequhNoP\n6hcRERGpqcKA/lrMP1aghExEREQmrc5snu0HTwFqIRMRERGpiccPnqI7F7B63nRmT8nULA4lZCIi\nIjJpjYXuSlBCJiIiIpPY1sKA/hrcLilOCZmIiIhMSmc6srQcPk0qYWxYNLumsSghExERkUnpkf0n\nCBxevHAmU+qqOhPYoJSQiYiIyKQ0FuYfK1BCJiIiIpNSz4B+JWQiIiIio+7Y2U72HjtHQzrJ2gUz\nax2OEjIRERGZfLbuC1vHNiyeTSZV+3So9hGIiIiIjLKtreH4sVrPP1aghExEREQmFXdnWzR+bCwM\n6AclZCIiIjLJHDjRzrNnOpnVmGb1vOm1DgdQQiYiIiKTTOHXlVcvbSKRsBpHE1JCJiIiIpPKtn1j\nq7sSykzIzCxZ7UBEREREqi0InIf3FQb0j7OEDNhrZh81s9VVjUZERESkinYdPcup9iyXzKhnUVNj\nrcMpKjchuwL4BbDZzH5qZreZ2dgYBSciIiJSpmJ35bKLMBsb48egzITM3dvc/fPuvhF4P/BB4KiZ\nfdnMllU1QhEREZEKGWvzjxWUPYbMzF5nZvcDnwQ+BiwBvgk8UMX4RERERCqiOxfw6DMngbE1oB8g\nVWa9vcAPgI+6+7ZY+X1m9tLKhyUiIiJSWTsOnaYjm2fZ3KlcPL2+1uH0Um5C9lZ3/0m8wMyucfet\n7v77VYhLREREpKIK849ds3RsdVdC+YP6/7ZE2acrGYiIiIhINRUG9I+l6S4KBmwhM7OrgY3AHDN7\nX2zTdEBzk4mIiMi4cL4rxxO/PE3C4Kol46+FLANMJUzcpsUeZ4GbBzu4mb3SzJ42s1Yz+8AA9W4y\nMzez9eWHLiIiIlKeRw+cJBc4a5pnMqMhXetwLjBgC5m7/wj4kZl9yd0PDuXA0ez+nwVuAA4Dj5nZ\nFnff1afeNOAPgEeGFLmIiIhImbaN4fFjMHiX5Sfd/T3AZ8zM+25399cNsPsGoNXd90fHuge4EdjV\np95fAH8N/O+hBC4iIiJSrp75x8be+DEY/FeWX4me7xrGsecDh2Lrh4Gr4hXM7Epggbt/y8z6TcjM\n7DbgNoCFCxcOIxQRERGZrE6e72bX0bNkUgnWXTqr1uGUNFiX5ePR848qfWIzSwAfB94+WF13vxu4\nG2D9+vUXtNSJiIiI9KdwM/H1l86iPj02f5M4WJdlC9BvAuTuLxpg9yPAgth6c1RWMA24HPhhdC+p\nFwBbzOx17r59kLhFREREyrJ1DE93UTBYl+VrR3Dsx4DlZraYMBG7BfjNwkZ3PwMU3xkz+yHwR0rG\nREREpJIKA/o3jtEB/TB4l+WQflnZZ9+cmd0OfJdwzrIvuvtTZvZhYLu7bxnusUVERETKceR0BwdO\ntDOtLsWa+TNqHU6/Buuy/Im7X2tmbYRdlxZ/dvfpA+3v7g/Q5+bj7n5HP3VfNoS4RURERAZVuF3S\nVUuaSCXLvUHR6Busheza6Hna6IQjIiIiUjnF+1cuG7vdlVD+zcULU1RcS9hC9hN3f6JqUYmIiIiM\nkLuzbd/Ynn+soKy2OzO7A/gy0EQ4EP9LZvbn1QxMREREZCT2HjvH8bYu5kyrY/ncqbUOZ0DltpD9\nFnCFu3cCmNlHgB3A/6lWYCIiIiIjsTX268poiq0xq9zRbc8C9bH1OnrPKSYiIiIyphRvl7R0bHdX\nwuC/svw04ZixM8BTZva9aP0G4NHqhyciIiIydLl8wCP7w4Rs4xgf0A+Dd1kWJml9HLg/Vv7DqkQj\nIiIiUgEtR87Q1pXj0qZGmmc11jqcQQ027cWXRysQERERkUop/Lpy4zjoroQyB/Wb2XLgr4DVxMaS\nufuSKsUlIiIiMmzjZf6xgnIH9f8j8PdADrge+Cfgn6sVlIiIiMhwdWbzbD94CoCrl0yshKzB3b8P\nmLsfdPc7gddULywRERGR4Xn84Cm6cwGr5k2naWpdrcMpS7nzkHWZWQLYG90w/AgwtmdYExERkUmp\n2F25dHy0jkH5LWR/ADQCvw+sA34beFu1ghIREREZrq2F2yUtHx8D+qHMFjJ3fwwgaiX7fXdvq2pU\nIiIiIsNwpiNLy+HTpBLGhkWzax1O2cq9l+V6M2sBfg60mNmTZrauuqGJiIiIDM0j+08QOLx44Uym\n1JU7Mqv2yo30i8DvuftDAGZ2LeEvL19UrcBEREREhmq8zT9WUO4YsnwhGQNw958QToEhIiIiMmb0\nzD82vhKywe5leWW0+CMz+xzwVcJ7Wf4Gun2SiIiIjCHHznay99g5GtJJ1i6YWetwhmSwLsuP9Vn/\nYGzZKxyLiIiIyLAVuis3LJ5NJlVuJ+DYMNi9LK8frUBERERERmK83S4prtxfWc4ws4+b2fbo8TEz\nm1Ht4ERERETK4e7jdkA/lD+o/4tAG/Dm6HGW8FeWIiIiIjV38EQ7R053MKsxzep502sdzpCVO+3F\nUne/Kbb+ITPbUY2ARERERIZq676wu/LqpU0kElbjaIau3BayjmjuMQDM7BqgozohiYiIiAzNttbx\n210J5beQvQv4p9i4sVPoXpYiIiIyBgSBs23f+Jx/rGDQFrLo/pUvdPcrCGfmf5G7v9jdf17Gvq80\ns6fNrNXMPlBi+7vMrMXMdpjZT8xs9bBehYiIiExau46e5VR7lktm1LOoqbHW4QzLoAmZuwfAH0fL\nZ939bDkHNrMk8FngVcBq4C0lEq5/dfc17r4W+Bvg40MJXkRERKTQOrZx2UWYjb/xY1D+GLL/MrM/\nMrMFZja78Bhknw1Aq7vvd/du4B7gxniFPsndFDTZrIiIiAzR1mj82Hicf6yg3DFkv0GYLP1en/Il\nA+wzHzgUWz8MXNW3kpm9G3gfkAFeXupAZnYbcBvAwoULywxZREREJrruXMCjz5wExu+Afii/hWw1\nYffjk8AO4NPAZZUIwN0/6+5LgfcDf95Pnbvdfb27r58zZ04lTisiIiITwI5Dp+nI5lk2dyoXT6+v\ndTjDVm5C9mVgFfC3hMnY6qhsIEeABbH15qisP/cAry8zHhEREZGe2yUtHb/dlVB+l+Xl7h4fkP8D\nM9s1yD6PAcvNbDFhInYL8JvxCma23N33RquvAfYiIiIiUqb4gP7xrNyE7Gdm9hJ3/ymAmV0FbB9o\nB3fPmdntwHeBJPBFd3/KzD4MbHf3LcDtZrYJyKK5zURERGQIznfleOKXp0kYvGTJ5GghWwdsM7Nf\nRusLgafNrAVwd39RqZ3c/QHggT5ld8SW/2DoIYuIiIjAowdOkgucK5pnMKMhXetwRqTchOyVVY1C\nREREZIi2tU6M7kooMyFz94PVDkRERERkKArzj107ARKycn9lKSIiIjJmnDzfza6jZ8mkEqy7dFat\nwxkxJWQiIiIy7jy8L2wdW3/pLOrTyRpHM3JKyERERGTc2RpNd3HNBOiuBCVkIiIiMg4VB/SP8wlh\nC5SQiYiIyLhy5HQHB060M60uxZr5M2odTkUoIRMREZFxpXC7pKuWNJFKToxUZmK8ChEREZk0Ct2V\n1yybGN2VoIRMRERExhF3Z2v0C8uJMqAflJCJiIjIONJ67BzH27qYM62O5XOn1jqcilFCJiIiIuPG\n1tivK82sxtFUjhIyERERGTeK3ZVLJ053JSghExERkXEilw/46f4wIds4gQb0gxIyERERGSdajpyh\nrTPHpU2NNM9qrHU4FaWETERERMaFbVF35cYJ1l0JSshERERknNg6AecfK1BCJiIiImNeZzbP9oOn\nALh6iRIyERERkVH3+MFTdOcCVs2bTtPUulqHU3FKyERERGTMK3ZXLp14rWOghExERETGgYl4u6Q4\nJWQiIiIypp3pyNJy+DSphLFh8exah1MVSshERERkTHtk/wkCh7ULZjKlLlXrcKpCCZmIiIiMadsm\neHclKCETERGRMa5n/jElZMNiZq80s6fNrNXMPlBi+/vMbJeZ/dzMvm9ml1YzHhERERlfjp3tZO+x\nczSkk6xdMLPW4VRN1RIyM0sCnwVeBawG3mJmq/tUewJY7+4vAu4D/qZa8YiIiMj4U+iu3LB4NpnU\nxO3Yq+Yr2wC0uvt+d+8G7gFujFdw9x+4e3u0+lOguYrxiIiIyDgzkW+XFFfNhGw+cCi2fjgq6887\ngW+X2mBmt5nZdjPbfvz48QqGKCIiImOVu0/oG4rHjYm2PzP7n8B64KOltrv73e6+3t3Xz5kzZ3SD\nExERkZo4eKKdI6c7mNWYZvW86bUOp6qqOZnHEWBBbL05KuvFzDYBfwb8qrt3VTEeERERGUe27gu7\nK69e2kQiYTWOprqq2UL2GLDczBabWQa4BdgSr2BmLwY+B7zO3Y9VMRYREREZZ7a1jvPuyn0/KLtq\n1RIyd88BtwPfBXYD97r7U2b2YTN7XVTto8BU4GtmtsPMtvRzOBEREZlEgsDZtm+czj+W64b//HP4\nyuvL3qWq9x9w9weAB/qU3RFb3lTN84uIiMj4tPu/z3KqPcslM+pZ1NRY63DK93wrfP0dcPRJsGTZ\nu42JQf0iIiIiccXuymUXYTYOxo+5w8++Ap+7LkzGZi6Ed3yn7N0n5h06RUREZFzbum8czT/WcRr+\n4z3w1P3h+uU3w2s/DvUzyj6EEjIREREZU7pzAY8+cxIYBwP6Dz4M3/gdOHMIMlPh1XfBFbfAEFv1\nlJCJiIjImPLk4dO0d+dZNncqF0+vr3U4peVz8OOPwo//BjyAS66EmzZD09JhHU4JmYiIiIwpxdsl\nLR2j3ZWnDoatYoceAQyufS9c/2eQTA/7kErIREREZEwpJGQbx+J0Fzu/Dt98L3SdgWnz4A2fgyW/\nOuLDKiETERGRMeN8V44nfnmahMFLloyhFrKuNvj2+2HHv4TrL3wN3PgZaJxdkcMrIRMREZEx49ED\nJ8kFzhXNM5jRMPwuwIo68jh8/VY4uR9S9fCKv4T17xjywP2BKCETERGRMWPbWOquDALY9rfw4F9A\nkIOLL4ebvgBzV1b8VErIREREZMzYGk0Ie02tp7s4exTu/1145kfh+lXvgk0fgnR1fvWphExERETG\nhJPnu9l19CyZVIL1i2bVLpA9D8C/vxs6TkLjRfD6v4MVr6jqKZWQiYiIyJjw8L6wdWzdwlnUp8u/\nD2TFZDvCm4I/tjlcX/pyeP0/wLSLq35qJWQiIiIyJhRul3Tt8hp0Vz73FNz3Tji+GxJp2HQnvOT3\nIDE6t/1WQiYiIiJjQnFA/2hOCOsOj34+bBnLd0HTcrj5CzDvitGLASVkIiIiMgYcOd3BgRPtTKtL\nsWZ++TflHpHzz4djxX7xnXD9yrfCKz8CmSmjc/4YJWQiIiJSc4XZ+a9a0kQqOQrdhPsehPvfBeee\ng/oZ8Ot/C5e9vvrn7YcSMhEREam5QnflNcuq3F2Z64YHPwzbPh2uL9wIb7wbZi6o7nkHoYRMRERE\nasrd2Rr9wvKaak4I+3wrfP0dcPRJsCS87ANw3R9Coga/6OxDCZmIiIjUVOuxcxxv62LOtDqWz51a\n+RO4wxP/DN/+Y8i2w8yF8MbNsPCqyp9rmJSQiYiISE1tjf260ip4f0gAOk7Df7wHnro/XL/8Znjt\nx8NxY2OIEjIRERGpqWJ3ZaVvl3TwYfjG78CZQ5CZCq++C664paI3Ba8UJWQiIiJSM7l8wE/3hwnZ\nxkoN6M8UpaQlAAAgAElEQVTn4McfhR//DXgAl1wJN22GpqWVOX4VKCETERGRmtn57FnaOnNc2tRI\n86zGkR/w1MGwVezQI4DBte+F6/8MkumRH7uKlJCJiIhIzfSMH6tAd2XLffAf74WuszBtHrzhc7Dk\nV0d+3FGghExERERqZtu+Csw/1tUG334/7PiXcP2Fr4EbPwONsysQ4eio6lS4ZvZKM3vazFrN7AMl\ntr/UzH5mZjkzu7masYiIiMjY0pnNs/3AKQCuXjLMhOzI4/C5l4bJWKoeXvNxuOVfxlUyBlVsITOz\nJPBZ4AbgMPCYmW1x912xar8E3g78UbXiEBERkbHpZwdP0ZULWDVvOk1T64a2cxDAtr+FB/8CghzM\nvQxu/iLMXVmdYKusml2WG4BWd98PYGb3ADcCxYTM3Q9E24IqxiEiIiJjTGc2z7/tOALANUuH2Dp2\n9ijc/7vwzI/C9aveBZs+BOn6Ckc5eqqZkM0HDsXWDwPDmhLXzG4DbgNYuHDhyCMTERGRUefu/Pzw\nGe7dfogtTz5LW2cOgJevnFv+QfY8AP/+bug4CY0Xwev/Dla8okoRj55xMajf3e8G7gZYv3691zgc\nERERGYLnz3Xxb08c4d7th/jFc+eK5VcsmMnbN17KxnLuX5ntgP/8c3hsc7i+9OXw+r+HaS+oUtSj\nq5oJ2REgfuv05qhMREREJrhcPuCHTx/na48f4vu7j5ELwvaUpikZ3vDi+bxp/QJe+IJp5R3suafg\nvnfC8d2QSMOmO+ElvweJqv42cVRVMyF7DFhuZosJE7FbgN+s4vlERESkxlqPneNrjx/iGz87wvG2\nLgCSCWPTqrncvG4BL185l0yqzETKHR79fNgylu+CpmVw0xfgkrVVfAW1UbWEzN1zZnY78F0gCXzR\n3Z8ysw8D2919i5n9CnA/MAv4dTP7kLtfVq2YREREpPLaOrN86+dH+drjh3n84Kli+ZI5U3jz+gW8\n8cXzmTt9iAPuzz8fjhX7xXfC9Rf/NrzqryEzpYKRjx1VHUPm7g8AD/QpuyO2/BhhV6aIiIiMI+7O\no8+c5N7th3mg5Sgd2TwAUzJJfv2KS3jT+mauXDgLG86NvPc9CPe/C849B/Uz4Nc/BZe9ocKvYGwZ\nF4P6RUREZGw4eqaDb/zsCF/bfogDJ9qL5RsWz+bN6xfw6jUvoDEzzPQi1w0Pfhi2fTpcX7gR3ng3\nzFww8H4TgBIyERERGVBXLs9/7TrGvdsP8dDe40Tj83nB9HpuWjefm9ctYPFFI+xKfL4Vvv4OOPok\nWBJe9gG47g8hkRz5CxgHlJCJiIhISU89e4avbT/Mv+04wun2LADppPGq1S/gTeubuW75HJKJYXRJ\nxrnDE/8M3/5jyLbDzIXwxs2wcFhTl45bSshERESk6HR7N/++41nu3X6Ip549WyxfNW86b17fzI1r\n5zN7SmbkJ8rn4OR++OFfwlP3h2WX3wyv/Xg4bmySUUImIiIyyeUD5yetz/O17Yf4z6eeozsf3tFw\nRkOa16+9hDetX8Dl84eZJOWzYeJ1fA8cfxqO7Q6fT+yFfHdYJzMVXn0XXHELDOdHABOAEjIREZFJ\n6uCJ89z3+GHue/wwR890AmE+9NIVc3jz+mY2rbqY+nSZY7hy3XByX5h4HdvTk4CdaIUgW3qfGQvC\nOcU2fQiallboVdVWR66DX5z6BXtO7GH3yd1l76eETEREZBLp6M7z7Z1HuXf7IX66/2SxfOHsRt60\nrpmb1jVzycyG/g+Q6wqTrEJL1/Eo+TqxDzxfep+ZC2HOyj6PFVBX5kz9Y9SZrjPsPrm7mHztObmH\nA2cPEHgw5GMpIRMREZng3J0nDp3ma9sP8c0nj3KuK7ypd306wavXzOPN6xewYdFsEvEB+tnOsFux\n2NoVPU4+00/iZTBrEcxZBXNeGCVdLwwf43wyV3fnufbn2HNyT68E7Oj5oxfUTVqS5bOWs2r2KlbO\nXslbeWtZ51BCJiIiMkEdb+vi/icOc+/2w7Qe67mp94sXzuTN6xfw2hfNY1oiC8//Alr+q6eb8fhu\nOHUASrX0WAJmL40lXCth7kpoWg6ZxtF7cVUSeMAvz/6SPSf3sOvkLvac2MOek3s41XXqgrr1yXpW\nzF5RTL5WzV7FslnLqEvWFesoIRMREZmEsvmAH+w5xr3bD/ODp4+RjyYNa57ivPOF3bz6BWe4uOtR\naN0DD++BUwcBv/BAlgyTrGLSFbV8NS2D9ABdmuNINp+l9XRrT8vXyT08ffJp2nPtF9SdnpleTLxW\nNoXJ16Lpi0hWaJ40JWQiIiITwN7n2vja44f5zuN7mdX+DMsTR/hA8ghXzTjO8sQRGs4fhl2Ej7hE\nKmrxemFP0jVnZZh4pepKnWpcas+28/Spp9l9Yncx+Wo93UouyF1Qd27jXFbPXs3KppXFlq95U+YN\n7zZQZVJCJiIiMh51nuH8kV20PPEoz+17ghnn9vPWxBH+1J6HeB7VET0n0mGSNXdl7+7G2UshVYF5\nxcaQk50new2033NyDwfPHsT7tAQaxqLpi8JWryjxWtm0ktn1s0c9ZiVkIiIiY1WQh84z8PzecFzX\n8afxY3vo/u9d1LX/N1OAlxTqRj1nQSKDXbQMm7sqlnitgtmLIZmu0QupDnfn6PmjPYlXlIQ91/7c\nBXVTiRTLZi4rJl+rm1azYtYKpqTHxg8OlJCJiIgMRz4b3uon2xF7jpa72/vflu2A7PkSZSWOke+6\n4LRG2ADW6Wn2+SWcmrKEWZeuYdllv0LdvNUkZi2C5MT7es8HeQ6ePdhroP2eU3s403XmgroNqYbe\nrV6zV7Js5jLSYzghnXhXTEREJjf3cK6skolQqUSpRELUfX6AZCpKqEqMPaqGXKqRQ3YJOzpfwC+C\nZvb6fM5MXcLGdVdy0/pFXNY0/n/Z2FdXvovWU63Flq/dJ3ez99ReOnIdF9SdVTer10D7lbNXsnDa\nwooNth8tSshERCYr93Bag3w2TC6CXNhFFsTW87nYtmy0vbAtvp7t2b/X8fo8qnK8LOQ6eydOw5iY\nc8gsAekp4S8O0w3hXFvpBjzdQD7ZQDZRR7fV00kdnWRo9zrOe4ZzQZq2fIYzuRRncilOZVOc6k5y\nojvN811JTnQl6SBDB3V0kSZsE4NMKsErLn8Bb13XzDXLLhr5Tb3HiHPd54rjvHafDAfcP3P6GXJ+\nYcI7b8q8YqvXqqYw+bq48eKqDrYfLUrIRERqwT1MNHIdUWtO9JzrCCfkzMUelVgPsqUTnokqmYkS\npcboOZY4Fcsae5dlGotluUQ97WQ4l0/TVkyg0pzOpjiZTXGyO8npTjjbmeNsZ5YzHVnOns+G6x1Z\nckGJaSTKZAbT69PMbUgxvT7N7CkZblh9Ma+74hJmNo6twffd+W7auts4lz0XPrqjR7Te1t3Wa724\n3H2Otmwb57PnOZ89f8FxE5ZgyYwlvQbar5y1kpn1M2vwKkeHEjIRmXzco0c+TFA8CG9yXEiI+iZI\n1VovNffTaLNEOO1BIhX+Ci+RDJeTseWS21LRejpWJxnb1udR7ePFE6xUA4ElaesKk6MzHVnOdmY5\n25HlbEcsgerIcqatJ4nqqZejI1uYiT4AuqJH+RrSSWY0pJkeJVXhcvRcn2J6tN6zLarXmGZqJtV7\nxvwqCDygPdveK0lq6w4TpLZsW6/EqVSdwnK2v3tUDkE6ke41s/3K2StZMWsFjemJ1xU7ECVkIhLK\ndUFXG3SdDZ87o+dCWbY9Sl7yEARhEhNPaIrl0Xqhrgc95b3qFpaDQcr7HG/Ac8fL+5y77/HGgkQa\nUvWQrg+fC49e63VRklEHqYYB1vvbPzpGqq50YpNIVPxl5QMnmw/ozgdkcwHZfLjelQvI5nse3TmP\nLUf18062u6csm/foOeg5Zj4gG+3blc/SneviXOeZXolWW1cOH0G+m0rYoAnUjGj9gnr1aTKpyr+v\nBdkgW2yJKrQyFVupSiRSfeuc7w4Tqr5TQAxHKpFiWnoaU9JTmJaZxtTM1HA5HS5PTU/teY6Wp2Wm\n9Vqfkp5Cwqr3fo0XSshExrtsZ+9EaqDlvklWfDnfXetXMvosGbayWNTSkq6PJTkDJUlDWS+RRKXr\nIVlXsV/CuTud2YD27hzt3Xk6s3nau/O0n+9Z7sp1k8130p33KEkqJDi9E554olQoLyZBee+TPPUk\nRvE6+RF011VOwLS6JNMaEkxrSDKt3phWn2BqQ4IpmQRT6xM01hlT6oyGjNFQZzRmjPoMNGQSpBJO\n3vPhI8iT8y7yQbieC3LF8g7P05bPcagtIH+297ac50ru02s9VtbfPtkgS3u2vZhQdZX45eVwNKQa\nSidOJRKpaZlY0hXbVpesmxDjt8YCJWQiteAeju0ZMGmKJ0zx9T7llUqkEimomw5103qe6wvr08Lu\noGLykgy7uiwZtrBYtF7YXlxO9KlbWC5Vt7Bc4niJZDiw5oJzD3Y86yfmaFsFuTuBBwQekPNc+Bz7\nEm7PZmnvynLu3Enau7po785xPttNezZLR3eWjmzh0U1nNk9HrpuuXJbObI6ufI6uXI6uXJauXI7u\nfO9HNp8Dc4wALHrQ82zm4ToO5uFzcZloOcCIr8fq9V1PhY9i/Wib4WSiuglzEhZdNvPoQc8zYRnx\n5Whfix0zjJ+oNaf3wwmK5U5A4HmCKMkptP60RY+ijuhx4UwJ40bSkiWTowtaotLTmJIp3Vo1JT2F\nVEIpwFiiqyGVV/jlVrGLKdZNVBy7E3059Lsc1S2We4nyYID9KeO4fZcp47h9l2P7B3noPlcikeon\nyarA2Asg7PaKJ06xpCqfmUp3ppHuTCPZzBSymQa60/V0p+rIpuroTmXIJtN0J9JkDbqDLN1BN9l8\nlmyQpTvfTXfQTXe+m1yQw3Hcvfdz9H4HnsPJFrcBBEGABz11w4+HExAU6xTqBx70qlP4XxBdm2JJ\n3/PH6vZb1jfW6PzxuvkgIBuErRLZfE/rRC6ItWBEj/CLP0wAwqQgwMmXvj6VYkA6esQkKc4HOiYV\n0qch7RB/HoGUpUgmkiQtSTKR7LWeSqSK5UkrXafcffpuK7lPfL/+zjPAtimZKcVkqyHVoFapCUgJ\n2VAFQc8vlOI/xc5no7LYz7nzsV8y5bO968V/rn3BT7r7G3sTGzPT79iaEnVLJUe9jtN3v1LH8RJl\nA4wZmiDiXybeZ534NoMAC5MajJwZ3cUHZKPlrBndGNn6FN3JBrrTDWTT9WRTYZLUncrQnUqTTabJ\nJlJ0J5LRw8hagizQbdDtAVk8fPYc3fnuWAJ1mmz+ONnzWfLnqpwkyAXck+EHwhNAAo+ejQRGEiNB\nggRmSZKWIGFJEr0SgiSpRLicTqRIJZKkkikyiRTpZJJ0MkUm9qhLpahLpkkle5KEhCV6JQgJS5C0\nJIaRsARmVlwurIcx9S4HetUpbutT1yx6xorLQzneBc9lHK9XHSx8vVHyEj+HyEh4EOC5HGSzeC6H\nx5+zhfUsDLStTOMvITv/PPz0H4pJj+eyBEE3QT5LPsgS5LPRcjdBkAvXg3CbB/moTo7Aw215zxME\nWYIgTxD9i7i4HOTwwr+Ig55/FectbKjPA4EZQZ/lAMjHlgMz8oRf3vnoizuwqF5suVBe6FwgKnMg\nsMJ6WIYVOyB66kZx9ewblhWWB6xrhWTDCBJR3VTvGIodCJbASeBkeo5rJepG74GbFY8dlscTm3DH\neGJTqFtwQUJkF5bHt5ezrZhU2QDbRo1T7EcJgIoO5TKSliZBigQpIIV5CjyFB0kCTxIESYJ8klyQ\nIMgnobDdU+DJaDkZLhfeMAyK18jArdf1LJTRt6y/8lL7Q5TUlHNcomOUc65S8UMykaAulaYulaIh\nlaEulaI+naYhnaIhlaYhk6YhnaYxnaEhk2JKpo7GdJopmTRT6jI0ptNMrcvQkEnSkE7SmAkfhfVU\nUoOWZWJw96hxIuhZzudLlns++gf9AOWeD/BsFnJlJj3ZPnUG3BarM4xt5EfvH7XjLiHb3fEcV+75\nTDEh8uH+C8gY5NWnBqsgk4hFX96Ff6EX1sPcOFxOWIJ0Ik0mmSGVSJFJZMgk0tQlMqQtTSaRIkWa\nhKdIkoIggXmKhCfxfArPGx4kCHJJPEiQyxr5fIJcLlzO5iDbbXRnoTt6DvIJEp4ET2CBYZ6EIIl5\nAgui1gOPxugAiaj10jxMj80hEXW/FtLshnSCxnSS+pTRmEpQHz3qUkbSjAThMK+ERe0+BgkrPBeW\nvdc2i7YlLXy3etUlGvZFYd+ecxjesx6dI76/Fc5LbL2wXKiXMMw9eo7Wo+MWj2896VmhKzXsjo5a\nfQtTHuQcckBHvE7ffQoHCp+y7mR7bYvX6bt/7/Ley/2dkwvrDrpP+ecbdJ9y47vgOP2dZ6j1y4ur\n7PolrtPovZb+67sHEHjx18QeeJQE9ZS7B5DvkygNWH7hMXqXl068Jp10GkulsFLP6RSk0gNsS8HT\ne8o6jRU/CFVgZq8EPkU4xGGzu3+kz/Y64J+AdcAJ4Dfc/cBAx1wyo8H/4prF4f6F43jPv3cLX5UX\nLHv4F9c8/HIwt2KdnvLoyyv60qBPGcT2KZw32k7sfIVtFPalxPZiPetZj9UlWo9vC//Pev1BKdYv\n9ce5EEd8e4m65oUjl/qjYGFBn+MUj1tyn1J1S3zO+h6jlP42DfVz209961VusRPGl8s9d/g+mQfF\nJEhEZEJKhD/AMbOe5ei5/3LDLLacSEbbLUp6SiU24TLFsnTJpKe/bZZKxRKqTPic6X0OCtvSFyZd\npFIj7vo2s8fdff1g9arWBGRmSeCzwA3AYeAxM9vi7rti1d4JnHL3ZWZ2C/DXwG8MdNzGLrhy/0Bf\ndPoSlLGl2I1rYTuVW+GnZ+CWKPwMLXpE42SiJqXCHzVLGInoj5wlo/ExCQv/0BXqF/4ARse2wrH7\n1imW9a0TW4+fv1eM0Yuywj8krNd6r+ULnuNV+vzTpG/dEvv1u88A5+r9h7S/uOyCKsXX1/ecA72u\nC/YZ2jlH9PrKOucgr22A9/6CfQa7fv3GV9n65cc1stdRcp9BzjXU+v3GVnguJDZWSGbCZUsWfpkc\nJTYjLrdiAjVggiUVV80+uQ1Aq7vvBzCze4AbgXhCdiNwZ7R8H/AZMzMfoNnu2alN3PGSd1D4FPeM\nJ7Jez1xQTvGD3dNY3Hu95Pb+9in+h9pne+yPWk/dPufpZ5+oPSn67gufsUSxKyZcLTwnetUJvz97\n9i18uSZix7KonkX/wSX6OU/CgESCRPSehP+NJnq2WdithBW6pBLRd3bP8Qvl4Zd6FBuGJa3nNfa8\n5X3KCn+YoDCot1C32Hpp4TWwPseK1ye+HxbGEjtnz7P1c5zw2vQ6P7F9CutW6IYzGuvTNNalmFKf\nZmp9msa6NFPrUkypS9KYSU2Ye8+JiEhlVTMhmw8ciq0fBq7qr46758zsDNAEPB+vZGa3AbcBzJq/\nmCt/47UkLBrPYuGYkIQZyejLPxklDGE5UXmp+rExK33Wi8dKWKxO7FiJ+NiZ2Hrh2NZn/+jYPbHR\nb8wiIiIyuYyLUevufjdwN8D69ev9T161qsYRiYiIiFRONTuCjwALYuvNUVnJOmaWAmYQDu4XERER\nmTSqmZA9Biw3s8VmlgFuAbb0qbMFeFu0fDPw4EDjx0REREQmoqp1WUZjwm4Hvks47cUX3f0pM/sw\nsN3dtwBfAL5iZq3AScKkTURERGRSqeoYMnd/AHigT9kdseVO4E3VjEFERERkrNNkIiIiIiI1poRM\nREREpMaUkImIiIjUmBIyERERkRpTQiYiIiJSY0rIRERERGpMCZmIiIhIjdl4mxjfzI4DB2sdRxVd\nRJ+bq8u4oWs3vun6jW+6fuPXRL92l7r7nMEqjbuEbKIzs+3uvr7WccjQ6dqNb7p+45uu3/ilaxdS\nl6WIiIhIjSkhExEREakxJWRjz921DkCGTddufNP1G990/cYvXTs0hkxERESk5tRCJiIiIlJjSshE\nREREakwJ2SgyswVm9gMz22VmT5nZH/TZ/odm5mZ2UazsZWa2I6r/o9GPWgqGev3MbIaZfdPMnozq\n/6/aRC79XTszu9PMjkT/je0ws1fH9vkTM2s1s6fN7BW1i16Gev3M7AYze9zMWqLnl9f2FUxuw/nv\nL9q+0MzOmdkf1Sby0aUxZKPIzOYB89z9Z2Y2DXgceL277zKzBcBmYCWwzt2fN7OZwDbgle7+SzOb\n6+7HavcKJrdhXL8/BWa4+/vNbA7wNPACd++u2YuYpPq7dsCbgXPuflef+quBrwIbgEuA/wJWuHt+\ndCMXGNb1ezHwnLs/a2aXA9919/mjHrgAQ79+sf3uAxx4pL86E4layEaRux91959Fy23AbqDwR+IT\nwB8TfvgKfhP4hrv/MtpHyVgNDeP6OTDNzAyYCpwEcqMXsRQMcu1KuRG4x9273P0ZoJUwOZMaGOr1\nc/cn3P3ZaPUpoMHM6qofqZQyjP/+MLPXA88QXr9JQQlZjZjZIuDFwCNmdiNwxN2f7FNtBTDLzH4Y\nNbu/dZTDlH6Uef0+A6wCngVagD9w92A045QLxa9dVHS7mf3czL5oZrOisvnAodhuhxnkC0RGR5nX\nL+4m4Gfu3jVKIcoAyrl+ZjYVeD/woZoEWSNKyGog+rB9HXgPYYvJnwJ3lKiaAtYBrwFeAfx/ZrZi\ntOKU0oZw/V4B7CDs8loLfMbMpo9WnHKh+LVz97PA3wNLCa/PUeBjNQxPBjHU62dmlwF/DfzuKIcq\nJQzh+t0JfMLdz9UizlpJ1TqAycbM0oQfyH9x92+Y2RpgMfBk2LNFM/AzM9tA+K/yE+5+HjhvZj8G\nrgB+UZvoZYjX738BH/FwoGarmT1DOMbs0dpEP7n1vXYA7v5cbPvngf+IVo8AC2K7N0dlUiNDvH6Y\nWTNwP/BWd983yuFKH0O8flcBN5vZ3wAzgcDMOt39M6Mc9qhSC9koisYSfQHY7e4fB3D3Fnef6+6L\n3H0RYRJ2pbv/N/DvwLVmljKzRsIP6e4ahT/pDeP6/RL4H9G+FwMvBPbXJPhJrtS1i8rnxaq9AdgZ\nLW8BbjGzOjNbDCxHiXTNDPX6RT+I+hbwAXffOpqxyoWGev3c/brY39RPAn850ZMxUAvZaLsG+G2g\nxcx2RGV/6u4PlKrs7rvN7DvAz4EA2OzuO0vVlVExpOsH/AXwJTNrAQx4v7s/PwpxyoVKXjvgLWa2\nlvAHGAeIurbc/SkzuxfYRdgt/W79wrKmhnT9gNuBZcAdZlYYTvBr+mFUzQz1+k1KmvZCREREpMbU\nZSkiIiJSY0rIRERERGpMCZmIiIhIjSkhExEREakxJWQiIiIiNaaETKRCzOxOM/ujWscxEma2yMx+\ncxj7fcnMbi5R/kMzW1+Z6IYc0yIzG9I0Mf3Fa2avM7MPRMvF62xmHzazTdHye6L5AseM6Ab31Tp2\nyWs+QP2Xmdl/9LPtgJldVLnoRMYfJWQiAoCZpYBFhDe1HxeimKvO3be4+0dKlN/h7v8Vrb4HGFMJ\nGeFcT8M2Wu/vSJhZxsym1DoOkZFSQiYyDGb21uiGuE+a2VdKbP8dM3ss2v71QsuJmb3JzHZG5T+O\nyi4zs0fNbEd0zOVR+f+MlX/OzJLR40vRMVrM7L0lzl3qHPVm9o/RPk+Y2fVR+dvNbIuZPQh8H/gI\ncF10zvdG5/to9Fp+bma/G+1nZvYZM3vazP4LmDvA2/Xb0fF2mtkGM0uY2V4zmxMdK2FmrYX12Ou4\n08y+YmYPR/V/Jyp/mZk9ZGZbCCduxczeFx1/p5m9J3aYlJn9i5ntNrP7Ytfhjug17TSzu83C+16V\nijf2Pl0wU3ihlcjMfp/wnqU/MLMfmNk7zOyTfT4PnxjgPSq0Ev1VdO7tZnalmX3XzPaZ2bti7/tH\nY9f/N6LyeWb241jc15nZR4CGqOxfSpzvnJl9wsyeMrPvx67HD83sk2a2HfgDC1saH4yu//fNbGHs\nMJuiWH9hZq+N9l8UXZ+fRY+NsfrTzexb0efmH8zsgu8gK/G5H+h9A2YBT0V1f2WQuiJjl7vroYce\nQ3gAlxHeT/SiaH129Hwn8EfRclOs/v8B/t9ouQWYHy3PjJ4/DfxWtJwBGoBVwDeBdFT+d8BbCW82\n/73YsWeWiK/UOf4Q+GK0vJLwtk71wNsJb/dUeA0vA/4jdqzbgD+PluuA7YT37nwj8D0gSZiInAZu\nLhHLD4HPR8svBXZGyx8kvMEwwK8BXy+x753Ak9H7cRFwKDrXy4DzwOKo3rroNU8BpgJPAS8mbO1z\n4Jqo3hdj12d27DxfAX59kHjfDnymxHX+UuF1E840XvhMTAX2xa7fNmDNIJ+rA8D/Ey1/gvAOHdOA\nOcBzUflNsff94ug6zouu759FdZLAtGj53ADnc3o+d3fEXt8Pgb+L1fsm8LZo+R3Av8Ve+3cI/2G/\nnPBzVE/YSlgf1VkObI99tjqBJVGM3+v73tHP576M/ybrgFuA/wSeAH4/fo310GM8PNRCJjJ0Lwe+\n5tFtkNz9ZIk6l0etBC3AbxEmcQBbCW+n9DuEX0oADwN/ambvBy519w7Ce2CuAx6z8FYj/4Pwi2w/\nsMTMPm1mrwTOljh3qXNcC/xzFO8e4CCwItr2vX5eA4TJ0lujGB4Bmgi/ZF8KfNXd8+7+LPBgf28W\n8NXovD8mbCGZSZgcvTXa/g7gH/vZ99/dvSN6r38AbIjKH3X3Z2Kv7X53P+/u54BvANdF2w55z70M\n/zmqC3C9mT0SXZ+X03N9+ot3SKI4HgRea2YrCROMljJ23RI9twCPuHubux8HuqI4rqXnfX8O+BHw\nK8BjwP8yszsJE7+2Ms4VAP83Wo6/N8TKAa4G/jVa/kqfeve6e+Duewk/myuBNPD56L39GrA6Vv9R\nd9/v4W2ovtrnWND/535A7t7l7ve4+68BNwKbgGfN7JLB9hUZK5SQiVTHl4Db3X0N8CHClgPc/V3A\nnwBNlNwAAAQTSURBVAMLgMfNrMnd/xV4HdAB/397ZxdiVRmF4ec1AgnL6EeISDPDbiIlwggqqMuC\nMlAEg1KvDOkiwyKwEKJuCiSJBGfo57aYLC/KBJOaKFQYGzFJL4KK1GmikLKUSd8u1rdnzpz2OWeU\ncE+wHjicc/b+zv7Wt/aCvXjX2vvwkaT7if++fMf24vK6xfYm278BiwgVYy3Q3z5x3Rw9bD3VZZ8I\nda+yY77tXVP0wbhJ/zbRPwIjZa1LgI+n+tsp2Nz195JmEsrLsnJ++ijnp8ec50s/oaytpnPC2c6Z\n8n6u5XP1vWM/V0ke7wV+IpLxxzqN7ULrOi/Yv8BTwAgRp3cQqm+38a3Uxv2kAdKdpZz5taSHWrbP\nkfQ0obBdQvRCjkxxHUnSOJmQJcn58ymwvEp0JF1VM+Zy4LikSwmFjDJ2ge29tl8ARoEbJN0EfGd7\nC/AhcBvRz7VM0pxqDknzFHeizbA9QCRdt7dPXDcHMFjZIWkhMBc4UmP378X2ik+AJ8o6kLRQ0UD9\nObBC0WN2HXBfF39VfU53Aydtnyzb+wll5j13/uPuhxX9b1cTJa/9NWMGgaWSLiu2PVK2AcyVdFf5\nvBL4gonk6xdJs4D2OwU72duLSb6zvZfw/UqK6laOu1vS9VM8ZjuDTPj9WiIJ2ydpHlHW7CP8WsXF\nWHXuapjBxNor39TxJVEOhIihwZZ9yxU9gAsIJesIMBs4bvsc8YfSrT1gSyTNL71jK2rmrI371gEl\ntquEbYek2ZI+IGJyJvCA7Qdtv98lrpJk2jHt76BJkumG7W8kvQR8Juks0bOyqm3Y80SJb7S8Vxfq\nVxRN+yIuPsPAs0Qj+RhwAnjZ9q+SNgK7ysVrDFhHqGhvtTRDP1djYt0c3wJbSxnpb2CV7TOa1MsO\nRN/SWUnDhMr3GtGLNaQYPAosBbYTpb7DRB/TV11cdlrSAaKUtaZl+w5COeqmHh0kSpXXAC/aPlYS\nynFsD0l6G9hXNvXbPiDpRiJBWCfpzWLrVtt/SuoDDhH+bk/yOtnbi23ATknHbFcJ6rvA4qJsUs7b\nzUCnEnEvthMlxGFCXXrG9glJjwMbSgz9wUQ5eBtwUNKQ7UfbjnWKSJA2Aj9TEtEaniRibgNx/le3\n7PuB8PsVwFrbpyW9AQwUlW4nk9W2/cDrhA/2lPWMY/twh7j/vodftgB7bF+ompkkjaOM3yRJmkDx\nvK/Ntu/psH8T0ZT+6kU17D9E8dytzbZ3l++3Amtsr2/WsrjL0vaspu1IkiTIkmWSJBcdxUNWB6hX\n+P73SLpS0lHgryoZA7B9aDokY0mSTD9SIUuSJEmSJGmYVMiSJEmSJEkaJhOyJEmSJEmShsmELEmS\nJEmSpGEyIUuSJEmSJGmYTMiSJEmSJEka5h8WpRbTVsyUWwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fee533901d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# logits for a random image\n",
    "v = logits[4]\n",
    "plt.figure(figsize=(10, 5));\n",
    "plt.plot(np.sort(softmax(v)), label='$T=1$', linewidth=2);\n",
    "plt.plot(np.sort(softmax(v/2)), label='$T=2$', linewidth=2);\n",
    "plt.plot(np.sort(softmax(v/3)), label='$T=3$', linewidth=2);\n",
    "plt.plot(np.sort(softmax(v/7)), label='$T=7$', linewidth=2);\n",
    "plt.legend();\n",
    "plt.xlabel('classes sorted by probability, most probable ->');\n",
    "plt.ylabel('probability');\n",
    "plt.xlim([245, 255]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmsAAAFACAYAAADjzzuMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl8XHW9//HXZ2Yy2bume0pbutE90EKR7QdKAQVFEAXU\nK6iVe3/qdb9XxQ0R0auouF8BcfupcF1QLEgvyCKCLAXSJm0pDV3okrZpumWf7fv748wkk2SSTNrM\nTJb38/GYx5w553u+80mmTd75nnO+x5xziIiIiMjg5Mt1ASIiIiLSM4U1ERERkUFMYU1ERERkEFNY\nExERERnEFNZEREREBjGFNREREZFBTGFNREREZBBTWBMREREZxBTWRERERAaxQK4LGChlZWVu5syZ\nuS5DREREpE8vvPDCQefchHTaDpuwNnPmTNatW5frMkRERET6ZGY7022rw6AiIiIig5jCmoiIiMgg\nprAmIiIiMogNm3PWUgmHw+zevZvW1tZcl5I1BQUFlJeXk5eXl+tSREREZAAM67C2e/duSktLmTlz\nJmaW63IyzjlHfX09u3fvZtasWbkuR0RERAbAsD4M2trayvjx40dEUAMwM8aPHz+iRhJFRESGu2Ed\n1oARE9QSRtrXKyIiMtwN+7AmIiIiMpQprImIiIgMYgprIiIiIlm0fteRfrVXWMuwu+66i4qKCioq\nKvD5fO3LH//4x/vd1/ve9z4mTpzI4sWLM1CpiIiIZFokGuOzf6zq1z4Kaxm2evVqKisreeCBB5g+\nfTqVlZVUVlbyne98p999XX/99Tz00EMZqFJERESy4WdP7WBT7bF+7aOwliXV1dUsWbLkhPo477zz\nGDdu3ABVJCIiItm061Az3374lX7vN6wnxU028zMPZKTfHV+/NK12VVVVPR6+PPfcc2loaOi2/rbb\nbuPCCy88ofpEREQk95xzfPHP1bSEo1y2dAo/7Me+Iyas5Vp1dTWrVq1Kue3JJ5/McjUiIiKSTQ9W\n7eOxLXWUFgT44psXKqylku4IWKZUVVX1eFGBRtZERESGr6MtYW76y0YAPvPGU5hYWtCv/UdMWMul\nWCzG1q1bWbBgQcrtGlkTEREZvr659mXqGtpYPmMs155+Ur/31wUGWVBTU0N5eTnBYPCE+rn22mt5\n3etex5YtWygvL+enP/3pAFUoIiIimfDCzsP8+tnXCPiMW69Ygs/X/9tCamQtC+bNm8emTZtOuJ/f\n/va3A1CNiIiIZEM4GuPGP1bhHPzr+Sczf3LpcfWjkTURERGRDLjzyW1s2d/AjPFF/Pvr5x53Pwpr\nIiIiIgPstfpmvvvIVgC++tYlFOT5j7svhTURERGRAeSc43N/qqItEuOKU6dxztyyE+pPYU1ERERk\nAN2/fi9Pbj3ImKI8Pndp6pkg+kNhTURERGSAHGkO8ZU13kWFN75xAWUl+Sfcp8KaiIiIyAD5+l9f\n5mBjiDNmjePtK8oHpE+FNREREZEB8Nz2Q9zz/C7y/N6camb9n1MtFYU1ERERkRPUFoly431VAPzf\n8+cwZ2LJgPWtsJZhd911FxUVFVRUVODz+dqXe7pPaE927drFBRdcwMKFC1m0aBHf/e53M1SxiIiI\n9NcdT2yj5kAjJ5cV88HzZw9o37qDQYatXr2a1atXs2fPHs466ywqKyuPq59AIMC3vvUtTjvtNBoa\nGli+fDmrVq1i4cKFA1yxiIiI9Me2uka+/1gNALdcsfiE5lRLRSNrWVJdXc2SJUuOe/8pU6Zw2mmn\nAVBaWsqCBQvYs2fPQJUnIiIix8E5x+f/VE0oEuOq5eWcNfvE5lRLZeSMrN00OkP9Hk2rWVVVFYsX\nL0657dxzz6WhoaHb+ttuu40LL7yw2/odO3bw0ksvsXLlyv7VKiIiIgPqjy/u4elX6xlXHORzbzrx\nOdVSGTlhLceqq6tZtWpVym1PPvlk2v00Njbytre9jdtvv51Ro0YNVHkiIiLST4eaQtzygDen2ufe\ntICxxcGMvM/ICWtpjoBlSlVVVY8XFaQ7shYOh3nb297Gu971Lq688sqM1SoiIiJ9u/XBzRxuDnPW\n7PFcedq0jL3PyAlrORSLxdi6dSsLFqQeHk1nZM05x/vf/34WLFjAJz7xiYEuUURERPrh6VcP8vsX\ndhMM+PjqAM6ploouMMiCmpoaysvLCQaPf3j0qaee4le/+hWPPvpo+/QfDz744ABWKSIiIuloDUf5\n/H3VAPz7BXOYVVac0ffL6MiamV0CfBfwA3c5577eZfsngNVABKgD3uec2xnfdh3w+XjTW5xzv8hk\nrZk0b948Nm3adEJ9nHPOOTjnBqgiEREROV4/evxVth1sYs7EEv71/wzsnGqpZGxkzcz8wA+BNwIL\ngWvNrOukYC8BK5xzS4HfA9+I7zsO+BKwEjgD+JKZjc1UrSIiIiLpqDnQyI8f9+ZUu/WKJQQDmT9I\nmcl3OAOocc5tc86FgHuAy5MbOOcec841x18+AyTueHox8LBz7pBz7jDwMHBJBmsVERER6VUs5rjx\nvirCUcc1p0/njFnjsvK+mQxr04BdSa93x9f15P3AX/uzr5ndYGbrzGxdXV3dCZYrIiIi0rPfvbCL\n57YfoqwkyGffmJk51VIZFBcYmNm7gRXAN/uzn3PuDufcCufcigkTJmSmOBERERnxDja2ceuDLwPw\nhcsWMrooL2vvncmwtgeYnvS6PL6uEzO7EPgc8BbnXFt/9hURERHJhlvWbOJoS5hz55bxlmVTs/re\nmQxrzwNzzWyWmQWBa4D7kxuY2anAT/CC2oGkTWuBi8xsbPzCgovi60RERESy6smtdfypci/5AR9f\nfWtm51RLJWNTdzjnImb2YbyQ5Qfuds5tNLObgXXOufvxDnuWAL+Lf+GvOefe4pw7ZGZfwQt8ADc7\n5w5lqlYRERGRVFrDUT7/J29OtY9eOJeTxhdlvYaMzrPmnHsQeLDLui8mLXe/S3nHtruBuzNXnYiI\niEjvvv/oVnbWNzN/UikfOPfknNQwKC4wGM7uuuuu9jsO+Hy+9uWe7hPak9bWVs444wyWLVvGokWL\n+NKXvpShikVERARgy74GfvLENszg1iuXkOfPTWzSvUEzbPXq1axevZo9e/Zw1llnUVlZeVz95Ofn\n8+ijj1JSUkI4HOacc87hjW98I2eeeeYAVywiIiKJOdUiMce7zzyJ5TNyNze/RtaypLq6miVLlhz3\n/mZGSUkJAOFwmHA4nPUTHEVEREaK3z7/Gi/sPMyE0nz+4+JTclrLiBlZW/KL4w9Kvam6riq9dlVV\nLF68OOW2c889l4aGhm7rb7vtNi68sOO0vmg0yvLly6mpqeFDH/oQK1euPL6iRUREpEcHjrXy9b96\nc6rd9OZFjC7M3pxqqYyYsJZr1dXVrFq1KuW2J598Mq0+/H4/lZWVHDlyhCuuuILq6uoeA6CIiIgc\nn5vXbKKhNcLrT5nIm5ZMznU5IyespTsClrH3r6rq8aKCdEfWEsaMGcMFF1zAQw89pLAmIiIygB7b\ncoA1G2opzPPz5bcsGhSnHI2YsJZLsViMrVu3smBB6vuIpTOyVldXR15eHmPGjKGlpYWHH36YT3/6\n0wNdqoiIyIjVHIrw+fu8OdU+sWoe08dlf061VBTWsqCmpoby8nKCweBx91FbW8t1111HNBolFovx\njne8g8suu2wAqxQRERnZvvvIVvYcaWHhlFG89+yZuS6nncJaFsybN49NmzadUB9Lly7lpZdeGqCK\nREREJNmmvce46x/bMYOvXbmEQI7mVEtl8FQiIiIikgPRmOOz91URjTmue91Mlk0fk+uSOlFYExER\nkRHt18/uZP2uI0weVcAnL5qX63K6UVgTERGREWvf0Va+8dAWAL58+SJKC3I7p1oqCmsiIiIyYt10\n/0Ya2yKsWjiJixflfk61VBTWREREZER6eNN+Htq4j+KgN6faYKWwJiIiIiNOU1uEL/3Zm1PtkxfN\nZ+qYwhxX1DOFNRERERlxvv3wK+w92sqSaaO57qyZuS6nVwprIiIiMqJU7T7Kz57aji8+p5rfl/tb\nSvVGYS3D7rrrLioqKqioqMDn87Uv93Sf0J5s2bKlfd+KigpGjRrF7bffnqGqRUREhqdINMZn79tA\nzMH7zp7F4mmjc11Sn3QHgwxbvXo1q1evZs+ePZx11llUVlYeVz/z589v3zcajTJt2jSuuOKKgSxV\nRERk2PvFP3dSvecY08YU8vFVg29OtVQ0spYl1dXVLFmyZED6+tvf/sbs2bOZMWPGgPQnIiIyEuw9\n0sK3/tebU+3myxdRnD80xqyGRpUDYPMpCzLS74KXN6fVrqqqisWLF6fcdu6559LQ0NBt/W233caF\nF17Ybf0999zDtdde279CRURERjDnHF/880aaQ1HetGQyb1gwKdclpW3EhLVcq66uZtWqVSm3Pfnk\nk2n3EwqFuP/++/na1742UKWJiIgMe2s37uORzfspzQ/wpTcP3jnVUhkxYS3dEbBMqaqq6vGigv6M\nrP31r3/ltNNOY9KkofMXgYiISC41tIb50v0bAfjPS+YzaVRBjivqnxET1nIpFouxdetWFixIfSi2\nPyNrv/3tb3UIVEREpB9uW7uF/cfaqJg+hneuHHrne+sCgyyoqamhvLycYDB4Qv00NTXx8MMPc+WV\nVw5QZSIiIsNb5a4j/PKZnfh9NiTmVEtFI2tZMG/ePDZt2nTC/RQXF1NfXz8AFYmIiAx/4WiMz/6x\nCudg9XmzWDBlVK5LOi4aWRMREZFh6WdPbWdz7THKxxbysTcMjTnVUlFYExERkWFn16FmvvPwVgBu\neetiCoP+HFd0/IZ9WHPO5bqErBppX6+IiEhXzjm+8OdqWsJR3rxsKufPn5jrkk7IsA5rBQUF1NfX\nj5gA45yjvr6egoKhdUmyiIjIQHqgqpbHt9QxqiDAFy7LzKT42TSsLzAoLy9n9+7d1NXV5bqUrCko\nKKC8vDzXZYiIiOTE0ZYwX/6Ld1HfZ964gImlQ38AY1iHtby8PGbNmpXrMkRERCRLvvHQy9Q1tLFi\nxliuOX16rssZEMP6MKiIiIiMHC/sPMSvn32NPL9x65VL8A3BOdVSUVgTERGRIS8cjXHjH6sB+Nfz\nZjNvUmmOKxo4CmsiIiIy5N3x921s2d/AzPFFfPj1c3JdzoBSWBMREZEhbWd9E9/7mzen2levWEJB\n3tCdUy0VhTUREREZspxzfP5P1bRFYlx56jTOnlOW65IGnMKaiIiIDFl/rtzLk1sPMqYoj89dOvTn\nVEtFYU1ERESGpCPNIb6yxptT7cY3LWB8SX6OK8oMhTUREREZkr724MvUN4VYOWscb18+fCeEV1gT\nERGRIefZbfXcu24XQb+Pr16xBLPhMadaKgprIiIiMqS0RaLceF8VAP/3/NnMmViS44oyS2FNRERE\nhpSfPLGNV+uaOLmsmA9eMDvX5WScwpqIiIgMGdvqGvnBYzWAN6dafmB4zamWisKaiIiIDAnOOT53\nXzWhSIy3Ly/ndbPH57qkrFBYExERkSHhDy/u4Z/b6hlXHOTGNw3POdVSyWhYM7NLzGyLmdWY2WdS\nbD/PzF40s4iZXdVlW9TMKuOP+zNZp4iIiAxuh5pCfPUBb061z1+6gLHFwRxXlD2BTHVsZn7gh8Aq\nYDfwvJnd75zblNTsNeB64FMpumhxzlVkqj4REREZOr76wGYON4c5e854rjh1Wq7LyaqMhTXgDKDG\nObcNwMzuAS4H2sOac25HfFssg3WIiIjIEPZ0zUH+8OJuggEft7x1eM+plkomD4NOA3Ylvd4dX5eu\nAjNbZ2bPmNlbUzUwsxvibdbV1dWdSK0iIiIyCLWGo3zuT9UAfOT1c5hVVpzjirJvMF9gMMM5twJ4\nJ3C7mXWbSMU5d4dzboVzbsWECROyX6GIiIhk1I8eq2H7wSbmTizhhvOG/5xqqWQyrO0Bpie9Lo+v\nS4tzbk/8eRvwOHDqQBYnIiIig1vNgQZ+/MSrANx65RKCgcE8xpQ5mfyqnwfmmtksMwsC1wBpXdVp\nZmPNLD++XAacTdK5biIiIjK8xWKOG/9YTTjquPaM6Zw+c1yuS8qZjIU151wE+DCwFtgM/I9zbqOZ\n3WxmbwEws9PNbDfwduAnZrYxvvsCYJ2ZrQceA77e5SpSERERGcZ+98IunttxiLKSfD5zyciZUy2V\nTF4NinPuQeDBLuu+mLT8PN7h0a77PQ0syWRtIiIiMjjtO9rKrQ++DMAX37yQ0UV5Oa4ot0bmwV8R\nEREZlPYeaeGaO/7J0ZYw582bwJuXTsl1STmX0ZE1ERERkXTtPtzMtXc+w65DLSyaOorvXl0x4uZU\nS0VhTURERHLutXovqO050sLS8tH86n0rR/zhzwSFNREREcmpHQebuPbOZ6g92krF9DH84n1nMLpQ\nQS1BYU1ERERy5tW6Rq694xkONLSxYsZYfvbe0yktUFBLprAmIiIiObF1fwPX3vksBxvbWDlrHHdf\nfzrF+YomXek7IiIiIln38r5jvOvOZ6lvCnHW7PHcdd0KioKKJanouyIiIiJZtXHvUd5917Mcbg5z\n7twy7nzPCgry/Lkua9BSWBMREZGsqdp9lHf/9FmOtoR5/SkT+dG7TlNQ64PCmoiIiGTFS68d5j13\nP0dDa4RVCyfxg3eeSn5AQa0vad3BwMz0nRQREZHjtm7HIf7lp15Qe+PiyfzwnacpqKUp3dtNbTWz\nb5rZwoxWIyIiIsPOs9vqec/dz9HYFuGypVP43rWnEgzojpfpSvc7tQx4BbjLzJ4xsxvMbFQG6xIR\nEZFh4Omag1z/s+dpDkV5a8VUbr+6gjy/glp/pPXdcs41OOfudM6dBXwa+BJQa2a/MLM5Ga1QRERE\nhqS/v1LHe3/+PC3hKFctL+db76ggoKDWb2mfs2ZmbzGz+4DbgW8BJwN/AR7MYH0iIiIyBD328gFW\n/3IdbZEY154xnW+8bSl+n27KfjzSvRp0K/AY8E3n3NNJ639vZucNfFkiIiIyVD28aT8f+vWLhKIx\n3n3mSdz8lsX4FNSOW7ph7T3OuX8krzCzs51zTznnPpKBukRERGQIeqi6lg//5iUiMcd7z57JFy9b\niJmC2olI98Dx91Ks+/5AFiIiIiJD21/W7+VD8aB2w3knK6gNkF5H1szsdcBZwAQz+0TSplGAJkcR\nERERAP700h4+8T+VxBx88PzZ/MfF8xXUBkhfh0GDQEm8XWnS+mPAVZkqSkRERIaO37+wm//4/Xqc\ng4+8YS4fv3CugtoA6jWsOeeeAJ4ws58753ZmqSYREREZIu557jU+e18VzsEnV83j398wN9clDTt9\nHQa93Tn3MeAHZua6bnfOvSVjlYmIiMig9qtndvKFP1UD8OlLTuH/nj87xxUNT30dBv1V/Pm2TBci\nIiIiQ8fPn9rOTX/ZBMDnL13A6nNPznFFw1dfh0FfiD8/kZ1yREREZLC768lt3PLAZgBuevNCrj97\nVo4rGt76OgxaBXQ7/JngnFs64BWJiIjIoPWjx2v4xkNbALjlrYt595kzclzR8NfXYdDLslKFiIiI\nDHrf+9tWvv3wK5jB169cwtWnn5TrkkaEvg6D6gpQERGREc45x3cefoXvPVqDGXzzqmVctbw812WN\nGH0dBv2Hc+4cM2vAOxxqyc/OuVFZqFFERERyxDnHN9Zu4cePv4rP4DtXV3B5xbRclzWi9DWydk78\nubS3diIiIjL8OOe49cHN3Pnkdvw+43vXnMqlS6fkuqwRJ90buWNmpwHn4I2s/cM591LGqhIREZGc\ncs7x5b9s4udP7yDgM37wzlO5ZLGCWi6kdSN3M/si8AtgPFAG/NzMPp/JwkRERCQ3YjHHF/5czc+f\n3kHQ7+O/371cQS2H0h1ZexewzDnXCmBmXwcqgVsyVZiIiIhkXyzmuPG+Ku55fhfBgI+f/MtyLpg/\nMddljWjphrW9QAHQGn+dD+zJSEUiIiKSE9GY4z9/v4E/vLib/ICPO9+zgvPmTch1WSNeX1eDfh/v\nHLWjwEYzezj+ehXwXObLExERkWyIRGN86nfr+VPlXgrz/Pz0uhWcNacs12UJfY+srYs/vwDcl7T+\n8YxUIyIiIlkXjsb4+L2VrNlQS1HQz8+uP52VJ4/PdVkS19fUHb/IViEiIiKSfaFIjI/89iUe2riP\nkvwAv3jf6SyfMS7XZUmStM5ZM7O5wNeAhXjnrgHgnDs5Q3WJiIhIhrVFonzo1y/xyOb9lBYE+OX7\nzuDUk8bmuizpIq2pO4CfAT8GIsAFwC+B/5epokRERCSzWsNR/u1XL/DI5v2MLszjN6vPVFAbpNIN\na4XOub8B5pzb6Zy7Cbg0c2WJiIhIprSGo3zgl+t4bEsdY4vy+M0HVrKkfHSuy5IepDt1R5uZ+YCt\nZvZhvGk7SjJXloiIiGRCcyjC6l+s4+lX6xlfHOTXH1jJKZN1q+/BLN2RtY8CRcBHgOXAvwDXZaoo\nERERGXiNbRGu/9nzPP1qPWUl+dxzw5kKakNAWiNrzrnnAeKjax9xzjVktCoREREZUA2tYa7/2fO8\nsPMwk0bl85sPnMnsCTpINhSke2/QFWZWBWwAqsxsvZktz2xpIiIiMhCOtoT5l58+xws7DzN1dAH3\n3vA6BbUhJN1z1u4GPuicexLAzM7Bu0J0aaYKExERkRN3pDnEe+5+jg27jzJtTCH33HAm08cV5bos\n6Yd0w1o0EdQAnHP/MLNIhmoSERGRAXCoKcS773qWTbXHOGlcEb/5wErKxyqoDTV93Rv0tPjiE2b2\nE+C3ePcGvRrdckpERGTQOtjYxrvvepaX9zUwq6yY33xgJVNGF+a6LDkOfZ2z9q34YxkwD/gScBOw\nAKjoq3Mzu8TMtphZjZl9JsX288zsRTOLmNlVXbZdZ2Zb4w9deSoiIpKmAw2tXHvHM7y8r4HZE4q5\n54YzFdSGsL7uDXrB8XZsZn7gh8AqYDfwvJnd75zblNTsNeB64FNd9h2HFwxX4I3kvRDf9/Dx1iMi\nIjIS7D/WyrV3PsO2uibmTizhNx84kwml+bkuS05AuleDjjazb5vZuvjjW2bW11THZwA1zrltzrkQ\ncA9weXID59wO59wGINZl34uBh51zh+IB7WHgkrS+IhERkRFq75EWrv7JP9lW18Qpk0u55wYFteEg\n3Ulx7wYagHfEH8fwrgbtzTRgV9Lr3fF16UhrXzO7IREg6+rq0uxaRERk+Nl1qJmr7/gnO+qbWTR1\nFL/9wJmML1FQGw7SvRp0tnPubUmvv2xmlZkoqD+cc3cAdwCsWLHC5bgcERGRnHitvplr73yGPUda\nWFo+ml+9byWji/JyXZYMkHRH1lric6sBYGZnAy197LMHmJ70ujy+Lh0nsq+IiMiIsf1gE1ff8U/2\nHGnh1JPG8P9WK6gNN+mOrP0b8Muk89QO0/e9QZ8H5prZLLygdQ3wzjTfby1wq5mNjb++CPhsmvuK\niIiMCDUHGnnnnc9woKGNFTPG8rP3nk5pgYLacNNnWIvfD3S+c26ZmY0CcM4d62s/51zEzD6MF7z8\nwN3OuY1mdjOwzjl3v5mdDtwHjAXebGZfds4tcs4dMrOv4AU+gJudc4eO70sUEREZfl7Z38A773yW\ng41trJw1jruvP53i/HTHYGQoMef6PtXLzNY551ZkoZ7jtmLFCrdu3bpclyEiIpJxf9u8n0/9bj2H\nm8OcPWc8d75nBUVBBbWhxMxeSDdbpfvJPmJmnwLuBZoSKzXaJSIikj3NoQi3PLCZ3zz7GgAXzJ/A\nj9+9nII8f44rk0xKN6xdjTc57Qe7rD95YMsRERGRVNbvOsLH7q1k+8Emgn4f/3HxfN5/zix8Pst1\naZJh6Ya1hXhB7Ry80PYk8N+ZKkpEREQ8kWiMHz3+Kt/921aiMcf8SaXcfk0FC6aMynVpkiXphrVf\n4E2E+73463fG170jE0WJiIgI7Kxv4uP3VvLia0cAeP85s/iPi+frsOcIk25YW+ycW5j0+jEz29Rj\naxERETluzjl+t243X/7LRppCUSaNyudbb6/gnLlluS5NciDdsPaimZ3pnHsGwMxWArr0UkREZIAd\nagpx4x+reGjjPgAuXTKFr16xmDFFwRxXJrmSblhbDjxtZq/FX58EbDGzKsA555ZmpDoREZER5IlX\n6vjU79ZT19BGSX6Amy9fxBWnTsNMFxGMZOmGtUsyWoWIiMgI1hqO8vW/vszPn94BwOkzx/Ltd1Qw\nfVxRbguTQSGtsOac25npQkREREai6j1H+di9ldQcaCTgMz5x0Tz+9bzZ+DUlh8RpumMREZEciMYc\nd/x9G99+eAvhqGP2hGJuv/pUlpSP7ntnGVEU1kRERLJs9+FmPvE/63luu3cjoPe8bgaffeMCCoOa\nkkO6U1gTERHJEuccf67cyxf+VE1DW4Sykny+edVSLjhlYq5Lk0FMYU1ERCQLjjaH+dyfqlizoRaA\nVQsn8fUrlzC+JD/Hlclgp7AmIiKSYU/XHOSTv1tP7dFWioJ+vvTmhbxjxXRNySFpUVgTERHJkLZI\nlNvWbuHOJ7cDUDF9DLdfXcHMsuIcVyZDicKaiIhIBry87xgfu6eSl/c14PcZ//76OXz4gjkE/L5c\nlyZDjMKaiIjIAIrFHHc/tZ1vrN1CKBJj5vgivnN1BaeeNDbXpckQpbAmIiIyQGqPtvCp363nqZp6\nAK49Yzqfv3Qhxfn6dSvHT/96REREBsADG2q58b4qjraEGVcc5OtXLuGiRZNzXZYMAwprIiIiJ+BY\na5ib7t/IH1/cA8D58yfwjauWMrG0IMeVyXChsCYiInKcntt+iI/fW8meIy0U5Pn43JsW8O4zZ2hK\nDhlQCmsiIiL9FIrEuP2RV/jxE6/iHCyeNorbrz6VORNLcl2aDEMKayIiIv1Qc6CRj937EtV7juEz\n+OAFs/noG+YRDGhKDskMhTUREZE0OOf41TM7ufXBzbSGY5SPLeQ7V1dw+sxxuS5NhjmFNRERkT4c\naGjlP3+/gce31AHwttPKuektCyktyMtxZTISKKyJiIj0Yu3GfXz2j1UcagoxujCPW69YwqVLp+S6\nLBlBFNZpdh7nAAAgAElEQVRERERSaGqL8JU1m7jn+V0AnDOnjNvevozJozUlh2SXwpqIiEgXL752\nmI/fW8nO+maCAR+fueQUrj9rJj6fpuSQ7FNYExERiYtEY3z/0Rp+8FgN0ZjjlMmlfPeaU5k/uTTX\npckIprAmIiICbD/YxMfvraRy1xHM4IbzTuaTF80jP+DPdWkywimsiYjIiOac497nd3Hzmk00h6JM\nGV3At96xjLNml+W6NBFAYU1EREaw+sY2Pv2HKh7ZvB+ANy+byi2XL2Z0kabkkMFDYU1EREakx14+\nwH/8fgMHG9soLQhwy1sXc3nFtFyXJdKNwpqIiIwoLaEotz64mV89sxOAlbPG8a13LKN8bFGOKxNJ\nTWFNRERGjKrdR/novS+xra6JPL/xyYvm84FzT8avKTlkEFNYExGRYS8ac/z3E6/ynYdfIRJzzJ1Y\nwu3XVLBo6uhclybSJ4U1EREZ1nYdaubj91aybudhAK4/ayafeeMpFORpSg4ZGhTWRERkWHLO8YcX\n93DT/RtpbIswsTSf296+jPPmTch1aSL9orAmIiJDTizmONIS5mBjGwcb2jjYFPKeG9uobwxxsLGN\nPUdaeHlfAwCXLJrM165cwtjiYI4rF+k/hTURERkUQpEYh5q8oFWXFLoONrRRH19/ML7uUFOIaMz1\n2Wdx0M9Nb1nEVcvLMdNFBDI0KayJiEjGNLVFqG8MUdfYedQrsZy8/mhLuF99jy7MY3xJkLKSfCaU\n5LcvlyUtz5lQogluZchTWBMRkbTFYo6jLWHqm9qoawjFg1bHiNfBLmGsJRxNu2+/zxhXnAhc3vP4\n4iBlpR0BbEI8jI0rDhIM+DL4lYoMHgprIiIjXDjacfjxYGMoftixYzlxPlh9kxfAImkcfkzID/i8\n8FWaT1kiiJUGGV+ctC4exsYU5uHTfGci3SisiYiMMAcb23hk034e2riP9buOcLi5f4cfRxUEuh1u\nTF6ekBTGioN+nSsmcoIU1kRERoBdh5pZu3Ef/7txP+t2HiJ5cMxnMK446dBjlwA2IWl5fEmQ/IDm\nJxPJpoyGNTO7BPgu4Afucs59vcv2fOCXwHKgHrjaObfDzGYCm4Et8abPOOf+LZO1iogMJ845Xtnf\nyNqN+1i7cR8b9x5r3xb0+zh7znguXjSZ/zN/AhNLC3S7JZFBLGNhzcz8wA+BVcBu4Hkzu985tymp\n2fuBw865OWZ2DfBfwNXxba865yoyVZ+IyHATizkqdx9hbbUX0HbUN7dvKw76Of+UiVyyaDLnz59A\naYGukBQZKjI5snYGUOOc2wZgZvcAlwPJYe1y4Kb48u+BH5hObhARSVs4GuOZbfXthzgPNLS1bxtX\nHGTVgklcvHgSZ80u0+2VRHKp+RDsfQlqK6F2fb92zWRYmwbsSnq9G1jZUxvnXMTMjgLj49tmmdlL\nwDHg8865JzNYq4jIkNESivLEK3Ws3biPv23ez7HWSPu2aWMKuWjRJC5eNJkVM8YS8Gt6C5Gsa6zz\nAlntS7A3Hs6O7up7vx4M1gsMaoGTnHP1ZrYc+JOZLXLOHUtuZGY3ADcAnHTSSTkoU0QkO440h/jb\n5gOs3biPv2+tozUca982d2IJFy+azCWLJ7No6ihdfSmSTQ37O0bL9lZ6y8f2dG+XVwSTl8CUCpha\nAV9+V9pvkcmwtgeYnvS6PL4uVZvdZhYARgP1zjkHtAE4514ws1eBecC65J2dc3cAdwCsWLEi/Yl/\nRESGgP3HWvnfjftYu3E//9xW3+n2ShXTx3DxoslcvGgSJ08oyWGVIiPIsVovjCVGy2oroaG2e7tg\nCUxeClOWecFsSgWUzQVf8qkIgyOsPQ/MNbNZeKHsGuCdXdrcD1wH/BO4CnjUOefMbAJwyDkXNbOT\ngbnAtgzWKiIyKGyra2Ttxv2s3biPyl1H2tf7fdZ+BedFCyczeXRBDqsUGeacg2N7k4JZPJw17u/e\nNliaFMqWecFs/OwuwezEZCysxc9B+zCwFm/qjrudcxvN7GZgnXPufuCnwK/MrAY4hBfoAM4Dbjaz\nMBAD/s05dyhTtYqI5Ipzjo17j7VPsfHK/sb2bfkBH+fNm8AliybzhgUTGVMUzGGlIsOUc3B0d/dg\n1lTXvW3+aJiytGO0bEoFjDsZfJk9N9S8I45D34oVK9y6dev6bigikmPRmGPdjkM8FL+Cc8+RlvZt\npQUBLlwwiYsXTeK8eRMoCg7WU4tFhiDn4MjOzueX1a6H5vrubQvGdD6MOWUZjJ01YMHMzF5wzq1I\np61+CoiIZEFbJMpTNQdZW72fRzbvp74p1L5tQmk+Fy30ruA88+TxukG5yEBwDg5v7xzKatdDy+Hu\nbQvHdpz4n3geMwMGycU6CmsiIhnS0Brm8S11PLRxH4+/fICmULR924zxRfELBCZz6vQxuoG5yImI\nxeLB7KXOwaz1aPe2ReO7B7PR0wdNMEtFYU1EZAAlbpK+duM+nqqpJxTtmGJj4ZRRXkBbPIn5k0o1\nxYbI8YjF4NCrHSNmeyth3wZoO9a9bfHEzif+T62AUdMGdTBLRWFNROQE7T7c3H4F57odHTdJN4PT\nZ45tH0GbPq4ot4WKDDWxKBzc2jFNRiKYhRq7ty2Z3Pn8sqkVUDplyAWzVBTWRET6yTnH1gONPFTd\n/SbpeX7j3NllXLJ4MhcumMSE0vwcVioyRERC3mHM+hovnNVv9Z73VUO4qXv70qmdD2NOWQalk7Nf\nd5YorImIpKH9JunxKzi3H+z4BVIU9HPB/IlctGgSF5wykVG6SbpId85585S1h7EaL5zVb4XDO8FF\nU+83qrx7MCuZmN3ac0xhTUQkhbZIlIONIV490MjDm/bzv5v2sf9Yx03SxxblxafYmMw5c3WTdJF2\noSaofzUpkMVHyepfhVBDDzuZd/Xl+DneTP+J50mLobgsq+UPRgprIjJihCIxDja2cbCxjbqG5OcQ\ndV3WNSTdHD1h6ugCLoqff3b6TN0kXUawWNS7MXkijLUfvqxJfV/MhIIx8TA215vlP7E87mTI0105\neqKwJiJDWigSo76pjYMNofaglRy8kgPZ0ZZw2v0GfMb4kiCTRxVwztwyLl40mSXTRusKThlZmg8l\njZLFD1/Wv+o9om2p9/HlwbhZXggrmxN/jo+WFY0fFif8Z5vCmogMOuFojENNofbgdbD9OdT++mCj\nt+5Ic/oBzO8zxhcHKSvJp6w0nwkl+ZSVBplQks+E0nzKkp7HFOZp7jMZGRIn9ydGxpIPX6aa2T+h\nZHJHCEs+fDlmBvgVLwaSvpsikhWRRABLGunqfjjSW38oaXb/vvgMxhUnQlaQCYkQ1i2ABRlbFFQA\nk5Ep5cn98XDW28n9eUXe4cr20bH44cvxc6BgVHa/hhFMYU1ETsjhphC1R1u7BK6kc8Hi6w41h0j3\nVsRmtI+AdQ1cideJdWOLgvgVwEQ8oab46FhN/0/u7zZKNhdGTdVhy0FAYU1E+m3/sVYe2FDLA1W1\nvLAzxX32UugewLoHr8TzuGIFMJEeRUJw5LWOecmO5+T+xLlk4+fo5P4hQGFNRNJS19DGQ9W1/GVD\nLc/vONQ+SpYf8DFjfFHK4JV8WHJccVBXT4qkq+UIHN7hBbJD25Oed8Kx3eBiqffz5Xnhq2xu98OX\nReM0SjZEKayJSI8ONYV4qHofazbs5Zlt9e23UQoGfJw/bwKXLp3ChQsmUZyvHyUi/RKLQUNt5zB2\neEfHcksvI9bmg9EnwbiZMG5258OXOrl/WNInKiKdHG0Os3bjPv6yYS9Pv1pPNJ7Q8vzG+XMncNnS\nKaxaOIlSzdIv0rtwKxzZ2TmEHYqHssM7ep76AiBQ6E1/MXYWjJ3ZsTxuFoyeDoFgdr4GGRQU1kSE\nY61hHt64nweqanlyax3hqBfQ/D7jvHleQLt44WRGFymgiXTSfCjF6NgOb/nYXqCXq2qKJ3hBLBHC\nkoNZySQdspR2CmsiI1RTW4RHNu9nzYZanthSRyjqnQPjMzh7znguWzqVixdNZlyx/oKXESwW9U7a\nbx8R6xLMWo/2vK/5Ycz0pDA2s/Nyfml2vgYZ8hTWREaQllCUR18+wJoNe3n05QO0RbyAZgYrZ43j\nsmVTuWTRZCaU5ue4UpEsCrf0cKhyu3fVZbSXef+CJfERsRmdD1WOnQWjy8Gv0Wg5cQprIsNcazjK\n41vqWLNhL3/bfICWcMfklytmjOXSpVN405IpTBqlS/dlmHLOm4k/1ejYoe3QuK/3/UsmdQ5hyaNk\nxWU6XCkZp7AmMgy1RaI8+cpB1mzYyyObD9DY1nFT8mXTx/DmeECbOqYwh1XKsOIcxCIQDUMsDNFI\n/DnV6+R2KV73uC3dPiLeaFgsHA9pO3qZEBZvuosxJ6U+oX/sDAgWZ+u7KJKSwprIMBGOxvhHzUEe\n2FDL2o37aGjtCGiLp43isqVTuXTJFKaPK8phlZJ14VZoOQRNB73g0lzvnRSfWG49Gg82fYSfPten\nf4/WnMgf7U11keqE/tHl4PPnuECRnimsiQxhkWiMZ7YdYs2GvTy0cV+nm5qfMrmUNy/zAtrMMo0M\nDAuxqDf/ViJopQxgXdaFGrNXn/nAH/RGqvyB+HMe+ALx567r46/73KenPoKp+0vep2CMF8wKx+pw\npWRVOBrmaOgoDaEGjoWOec9t3nNDuJeR3hQU1kSGmGjM8dz2eECr3kd90k3P504s8UbQlk5hzsSS\nHFYpfXIO2o51H+nqLYC1HKHXqSBS8QWgaHzPj4LRXrBJFYz8wTRDU/y1T3eokOEj5mI0hZu6ha1j\noWMdr7s+t3W8bo22DlgtCmsiQ0As5njhtcOsWb+XB6v3UdfQMZnmrLJiLls6hcuWTmX+ZE0FkDPh\nlqSQlRS0OoWvLsHseA4dFo6NB62y+PO4zuGruMv6/FEaUZIRKxQNdQ5XbT2ErHib5O2N4UZiPd3W\nKw1+8zMqOIpR+aMozSv1noOljAp6z5/gE2n3pbAmMkg553hp1xHWrK/lwapa9h3r+Ctt+rhCLls6\nlcuWTmHhlFGYfhkfn1jMm0U+GvJujt1pOf4It3jnfCWHrVSjX+Gm/r9/sCQpbJUlha5xKcLXeO+Q\nnm4lJCNIYnQrVdhKOcLVJYyd6OhWUaCoW8gaFRzV/igNlqYMY6OCoygMFPb6s1lhTWSIcs5Rteco\nD2yoZc2GWvYcaWnfNm1MIZcuncJlS6ewZNrooRPQYrF48GnzTkSPtHVZDvcdmKKheNvk5fh+7e1S\n9Nmp/xTv7aJ9158uf7D3w42JAJYIX4XjIE/Tpcjw45yjJdJCU7iJxnAjzeHm9uWmcBPN4eaO5Ugz\njaFGmiPNPbY/kdGtgAVShq1Oz/mpQ1hpsJSAb3DEpMFRhcgI5pxjc20Dazbs5YGqWnbWN7dvmzQq\nn0uXeOegnXbSmNwFtFgMmurg6G44usub0b19eS+EmlMEq3hIikX67j9nDAL58RPV449AYjk/vpzf\n/VBj+6hX0nKwRIcbZciKxqLtgalTmEoRrDotR7q3bwo34fp7bmUvivOKewxb7YcZe9je1+jWUKGw\nJpIjr+xvYM36vaypqmVbXcchtLKSfC5dMplLl05lxYyx+HxZ+EHT1gBH96QIY0mBrLdZ3Pviz4+H\nojxv2Z/XOSR12hbsCEld23Xa1iVY9dZnT+/tCyhgyZAVjoXbw1Faj0gTTaH4c5dtLZGWvt+wHwoD\nhRQFiijOKz7+R6CYkmDJoBndyiV9B0Sy6NW6Rtasr+WBqr28sr9jSoVxxUEuWTyZy5ZOYeWs8fgH\nMqBFI9BQ2yV8JYex3dB6pO9+isZ781GNng6jpsWXy73l/JKeA5MCkUi7UDSU8nBfr8uRpk6HChOP\ntmhb32/YD4mAVJRXREleCcV5HctFeV7w6m050b4oUKSANcD03RQZYJFojOZwlJaQ92hojfD3rXWs\n2VDL5tpj7e1GF+ZxyaLJXLZsCq87eTwB/3FMe+CcN+9WcvA6trvz64Za6Oucj0BBR/gaXQ6jkpZH\nT4dRUyGoyXRl5HHO0Rpt7f3wYKpDhUmHB5PDV2QATwvwmY/iQDHFQS9kJY9KpQpcvS0XBgrxmaZe\nGawU1mTEicYcreEozfEw1RyO0ByK0hry1nlBKxLfFm8TitLSvhxvn9xHfH1rOEYo2nMwKi0IcNFC\nL6CdPbuMYKCPH47h1u6jYF3DWLi59z4wKJ3SeSRs9PTO4axovEa/ZNBxzuFwxFwM5xwxYh3LLkaM\npGUX69Q2sdwWbUv/MGGKQ4XN4WaiA3ghSsACHeEqRcjqz6PAXzAszseSvimsyaDjnKM1HOsWirzA\nFKEl5G1rSVrvtYm0h6eWTkEq0qm/tsjxX1mUDp9BUTBAQZ6foqD3OGVyKZcuncp588rID8RvaxOL\nQcO++Lliu1KHsaa6vt8wWApjpicFsfKkMDYNSqd653bJkOacIxKL0BptpS3aRigaoi3a1v5ofx1p\n67S+17bRNsLRcL9CULftPbTtKWD1tT25v4E8Sf1EBH1BSoIlJ34OVl4xQV9QAUv6TWFtkInGHMda\nwhxuDnGkJcyR5hCHm8I0hSLEYo6Yg5hz3j2TXfLrjuWYI/66S/tYP9t37T+WTvvk7Snax/uAzv2H\nIrFOI1iZVhgPUoXxMFWYl1gOUBh/7a33URIwivKiFOcZxX5HUSBGYcBR5HMUBhwFfkehP0aBL0a+\nP0aQKBaLeFdBxkLeVZFNr0LtGnh5d3ykbJcX0vqaFNUX8A5Bju4ljBWMzvj3SzzHE5hC0RCt0db2\nbV1fd92/NdJz28ESXrLJMHzmwzDMvOXEa5/5OtaRejnPl9celNI996rrcp4vL9ffBhnhFNYyxDlH\nY1uEI81hjjR3D19HWkId65vj65vDHGsNE88yGDEKCFFAiCARDIdB/NnFj1q5eNsu25KWE9u959Rt\nvL/z+ttX8jYw89r4u7Xv3hedlsFPjABR7+GLUuj3AlFhPAgV+h0FiUDkc/HnKEFfjKDFCJq3nEeU\ngEXJa+8vgp8YfhfBTwS/i2IugkUTYSp+M+q2CLQkbkgd7bg59UDOw5VKTyftj57uBbGSSYPqBtPR\nWLTjXJxQo3c+TqSZSCxCzMWIumj7s3Ou43Wsh/Xx58Sjx/2T+nF07zdGjFgsjf3Tff+k94nGooRi\nofYAlsvAFPAFyPfnd3oE/cH0Xgc6ryvwFxD0B8nz5eE3f+rQkxSK2oMRvk4hqa/t/e0vuX0ioImM\ndApraWgNR71g1dTK0YYGGhsaaGhspKmpkZaWRlqbGmltaSLc1kS4tYVIWzMu1Eyea6OAMAUWIj8e\nugoJMdNC7SGsIHmZEAXBEIUWpoA2ggzm+amywAGR+COXzN9xT8T2+yIGku6NmGLZF+i4b2Li3ozJ\n54glRsmycNJ+zMW6Xd6fOBk6+YqzntYlbxvoy/uHouTAlBx6Ur1ONzAlv+4tcPkHUXAXkewZPmEt\nGob6VyHS6p2UHWnxbhMTbomvayEaaqG1pZG25kbaWpsJtzYTaWsmGmrBhZpxkRYs0oov0oo/2kZe\nrJU810Y+IcYQYrL14z5+A/WdDRR6s5z78+MngFuK50TjVNuSn/tqE98OffSTTn+k+Z6JZ1/ncJMy\nHPW03DU05XmjUSmX+7mv+XNyc+pUM4A3hhtTBqv2gBW/Aq395OhQU/vI10AxLOWl+gFfAL/58Zmv\n/Tnx6Pd6n799hCWxfMJ9mh+fz3s2rF/9+s3fHpiC/qCmJBCRrBs+P3X2V8P3T+u1iR8ojj/S1mUE\nPkQeYV8BEV8+MX8BsUABBAqwvCJ8wUL8+YXk5ReRV1CEP1jsBa1E4AoUQl78ESjo/JxqXaBAV+gd\np64nKycOfyUfCkuc2NxxsnOMaLQZF/G2Je/T/uhXfx3tWqIt7SEqMV9S8nPXEa2myIndYqWrwkBh\ne7hKPhcn+TlxAnVJsKT9SrXk7bq8X0QkN4ZNWAsTYGdsIq0E2x9tLo8W8r3XLkib5XmBKVCABYsI\n5Bfhzy8kWFBMfkERgcIS8osKyC8spKCwkILiQgqKCgjk5xPz5xH2+YngncMSiUWIuIj3nPSIOm9b\nOBbu9Dqx7K1vIdIWIdoa7bx/Un/RWJSI8/qJxqLtl467+AltifNmOp0/40i5LbFPT+u69tPje7j0\n23btP9W25H66XhmWKvB03db16rPkADUcTsQu8Bd0uoosOUQlh6lUV5wlry8KFOnwmYjIAHHeFXsQ\n9X7nEI3iYg5iUVw06s1/2WldzJvrMnldrH9/jA+bsLYtv4T3z1+JzxfD73P4fDGwKGYxsBiOKDHX\nRNQdSxmOIq0RXKuDw7n+SmSgpDq5ObHO5+tlm3V/JJ8I3W1b/ITpTidpJ/VnZhQECrqNaHUdtep6\nJZoOt4lIQm8BgVjM++XfxzoXi3nro1Hoss5bH4uv67t9ynWxGKTqIxrDxbq0jweYTu8ZjeJcl/ap\n1qVo33cfie9ZinW9to91+76TYjAi04bNb4MIzeyPrD/hfgK+AHm+PAIWwO/zE/AF2s/HyfPldXqd\nWA5YoGO567Yu2/0+PwHz3qO9/y77d+0jz5fXHgqA9ueOp45DpYkrp7q2TXVFVddtvfXT2/oe+0na\ntad6kvtJnCdkZt2uTksOTcntegtQuopM+qt9RNi5jh/I8eX2X5axmPdLgaTXifaJX1iJts55bV0s\n5evkfrxfCl1ed+o3aT9vrhzvl123dl37ie/nXMe25P1iiV88yaP0rvv3gy7NXPf2vfdB9/adful1\nX9dt3/6+J/QrILRvS7UuZUBICkQp1yV+uSf321sf8fYp+xj6RwuGFZ8PfD6syzN+f4p1Pszn79ze\n74MtL6f9dsMmrJ1UehI/fMMPewxPiaCVCEtdw1HAAu0BYChyqX6IpfzBlnq504+B4+yrY7GPtj3u\n74jP6tHHe8U6T6mR9APaAdHuX1DaX0NHXyn2T+Nr6PEXjYv/JRqNdvzgj0Q7hsijkY710RguGun4\nazEaH1pPrI/13lfnfTteu1gUIvG/LKNJfcSS3j/xXqn27bRPoq9Y+y8Sh+v8SzVF8Ol0SL2XYNT+\nXe+pTfy5459c6j66temtrchgZ9ZDGPD+uE3eht+HWZfw4PfFL+bqIVB02dZ7H0nbUq1L1T4RWDqt\n84HPn2KdD/P7weLte1vXpY8+23dZ1/71Jvroc90AZYV+9DFswlp+7SFmfuXX8b8aE4/EX7y0/zUa\ncY4wrlM752IdvzwSf/12adO1r05/bSf/VUvfbdp/gfSzDaBfLCLZYklXNid+OCd+WJt1em3xNt7D\nvF9Y8e3t7cw6trW3i18JnXiNdf5l0P4eSfuZYb6k/YykPn3eNix1LV338/lIfQU3HSPj1nk0vNPG\nTu177qNj377apxiVT3V1eRrv2anulGHA730/fP7OAaHLL+g+1/msf+1ThQFfl6CVct3QHUyQEzds\nwlqsoZGmJ/6e6zIGj95+EPaw3OnHQJ8/hFMvWx/b++orZQ39+BpSHXrtf1+97J9WXylq8PmwgL/j\nF4M/4P0Q9vu9v/ISP5S7vg74vb9G4/sk/jpN9GV+b8oT7xdA4nW8D1+ir/hftF36at+nU1/J+ya/\n9mGBQPtfpV1rJn6FqDcTS9L3p9P3qsvh+67bU3xfu7Xp2hZLWuy5v27/Lntoq1+GIjIYDZuwFpxx\nEuX//eOOH+6JvyYTfz0mzevl/XWZ3C7x11dSu/a/euPr6dJX/NGpTXzfPvtP3i+pfzN6btPbLzMR\nEREZtoZNWPOVllJ6/vm5LkNERERkQPky2bmZXWJmW8ysxsw+k2J7vpndG9/+rJnNTNr22fj6LWZ2\ncSbrFBERERmsMhbWzMwP/BB4I7AQuNbMFnZp9n7gsHNuDvAd4L/i+y4ErgEWAZcAP4r3JyIiIjKi\nZHJk7Qygxjm3zTkXAu4BLu/S5nLgF/Hl3wNvMO8krMuBe5xzbc657UBNvD8RERGRESWTYW0asCvp\n9e74upRtnHMR4CgwPs19MbMbzGydma2rq6sbwNJFREREBoeMnrOWac65O5xzK5xzKyZMmJDrckRE\nREQGXCbD2h5getLr8vi6lG3MLACMBurT3FdERERk2MtkWHsemGtms8wsiHfBwP1d2twPXBdfvgp4\n1Hn36LkfuCZ+tegsYC7wXAZrFRERERmUMjbPmnMuYmYfBtYCfuBu59xGM7sZWOecux/4KfArM6sB\nDuEFOuLt/gfYBESADzmXfDNIERERkZHB3DC51+SKFSvcunXrcl2GiIiISJ/M7AXn3Ip02g7pCwxE\nREREhjuFNREREZFBTGFNREREZBBTWBMREREZxBTWRERERAYxhTURERGRQUxhTURERGQQGzbzrJlZ\nHbAz13VkUBlwMNdFyHHT5zd06bMb2vT5DW3D+fOb4ZxL68bmwyasDXdmti7dyfNk8NHnN3Tpsxva\n9PkNbfr8PDoMKiIiIjKIKayJiIiIDGIKa0PHHbkuQE6IPr+hS5/d0KbPb2jT54fOWRMREREZ1DSy\nJiIiIjKIKayJiIiIDGIKa4OEmU03s8fMbJOZbTSzj3bZ/kkzc2ZWlrTufDOrjLd/IvtVC/T/szOz\n0Wb2FzNbH2//3txULtDz52dmN5nZnvj/sUoze1PSPp81sxoz22JmF+eu+pGtv5+dma0ysxfMrCr+\n/PrcfgUj2/H834tvP8nMGs3sU7mpPPt0ztogYWZTgCnOuRfNrBR4AXirc26TmU0H7gJOAZY75w6a\n2RjgaeAS59xrZjbROXcgd1/ByHUcn92NwGjn3KfNbAKwBZjsnAvl7IsYwXr6/IB3AI3Oudu6tF8I\n/BY4A5gKPALMc85Fs1u5HMdndyqw3zm318wWA2udc9OyXrgA/f/8kvb7PeCAZ3tqM9xoZG2QcM7V\nOudejC83AJuBxA+R7wD/ifePM+GdwB+dc6/F91FQy5Hj+OwcUGpmBpQAh4BI9iqWZH18fqlcDtzj\nnHakq+IAAAtGSURBVGtzzm0HavCCm2RZfz8759xLzrm98ZcbgUIzy898pZLKcfzfw8zeCmzH+/xG\nDIW1QcjMZgKnAs+a2eXAHufc+i7N5gFjzezx+HD+e7JcpqSQ5mf3A2ABsBeoAj7qnItls05JLfnz\ni6/6sJltMLO7zWxsfN00YFfSbrvp4xeMZF6an12ytwEvOufaslSi9CKdz8/MSoBPA1/OSZE5pLA2\nyMT/Mf4B+BjeaMuNwBdTNA0Ay4FLgYuBL5jZvGzVKd3147O7GKjEO4RWAfzAzEZlq05JLfnzc84d\nA34MzMb7jGqBb+WwPOlFfz87M1sE/Bfwr1kuVVLox+d3E/Ad51xjLurMpUCuC5AOZpaH9w/21865\nP5rZEmAWsN47YkY58KKZnYH313y9c64JaDKzvwPLgFdyU/3I1s/P7r3A1513wmiNmW3HO6ftudxU\nL10/PwDn3P6k7XcCa+Iv9wDTk3Yvj6+THOjnZ4eZlQP3Ae9xzr2a5XKli35+fiuBq8zsG8AYIGZm\nrc65H2S57KzTyNogET9/6afAZufctwGcc1XOuYnOuZnOuZl4Ae0059w+4M/AOWYWMLMivH/Em3NU\n/oh2HJ/da8Ab4vtOAuYD23JSvKT8/OLrpyQ1uwKoji/fD1xjZvlmNguYi4J2TvT3s4tfmPUA8Bnn\n3FPZrFW66+/n55w7N+ln6u3ArSMhqIFG1gaTs4F/AarMrDK+7kbn3IOpGjvnNpvZQ8AGIAbc5Zyr\nTtVWMq5fnx3wFeDnZlYFGPBp59zBLNQpqaX8/IBrzawC74KQHcQPmTnnNprZ/wCb8A53f0hXguZM\nvz474MPAHOCLZpY4ReEiXaCVM/39/EYsTd0hIiIiMojpMKiIiIjIIKawJiIiIjKIKayJiIiIDGIK\nayIiIiKDmMKaiIiIyCCmsCaSYWZ2k5l9Ktd1nAgzm2lm7zyO/X5uZlelWP+4ma0YmOr6XdNMM+vX\nNDc91WtmbzGzz8SX2z9n+//tnXuwV1UVxz9fEL0mKPFqyJGXQZRaJkljAQPEOI2aWsHQSCEwQ+GY\nBpo5GdHtzQzN4IPBAgZoUCkJ0asVgjyEkuERCFxMNEljQonGMiBgCFZ/7HW4hx/n97sXusKPmfWZ\nOfPbZ5919l577f2bs2btfc6Wvi9piKfH+7cQqwZJ972LZRf2eQX5gZKeKXPtdUkdmk+7IDg7CWct\nCIKKSDoH6AactLN2pnCd33XMrM7MJhfkTzKz5/x0PFBVzhrpW1anzOmy7/+DpHMlXXCm9QiC5iCc\ntSBoRiSN9M2HN0uaV3B9rKT1fn1hFnGRNExSveev8rzLJK2T9KKX2dPzv5TL/7mkln7M9TK2SppQ\nUHdRHTWS5vg9myQN8vxRkuokLQeWAZOB/l7nBK9virdli6Sv+n2SNE3SdknPAZ0qmOvLXl69pL6S\nWkh6VVJHL6uFpD9n57l21EqaJ2mNy4/1/IGSVkuqI32wFkl3efn1ksbnijlH0qOS/iTp17l+mORt\nqpc0Q0p7hRXpm7PTCV9Qz6JLku4k7QG7QtIKSWMk3V8yHqZWsFEWXfqJ171B0lWSnpX0mqRxObtP\nyfX/cM/vLGlVTu/+kiYD53veowX17ZM0VdI2Scty/bFS0v2SNgBfV4pQLvf+XyapS66YIa7rK5Ju\n8Pu7ef9s9OOTOfkLJf3Gx83PJJ3wbFLBuK9kN+C9wDaXvboR2SCobswsjjjiaIYDuIy0N2sHP2/n\nv7XANzzdPif/Q+AOT28FLvZ0W/99CBjh6XOB84EPAU8DrTx/OjAS6AMszZXdtkC/ojruBmZ7ujdp\nK6waYBRpi6ysDQOBZ3JlfQWY6OnzgA2kvVA/DywFWpKclH8BQwt0WQnM9PQAoN7T3yVt5gxwLbCw\n4N5aYLPbowOw0+saCOwHurtcH2/zBUBrYBvwMVKU0IBPudzsXP+0y9UzD/hsI/qOAqYV9PPcrN2k\nL7BnY6I18Fqu/14ArmhkXL0O3ObpqaRdS9oAHYHdnv+FnN3f5/3Y2fv32y7TEmjj6X0V6jMaxt2k\nXPtWAtNzck8Dt3p6DPBkru2LScGAnqRxVEOKLta4TE9gQ25sHQR6uI5LS21HmXHfhP/kecAXgSXA\nJuDOfB/HEcfZckRkLQiaj8HAAvOto8zs7QKZyz26sBUYQXLwAP5A2oJqLOmBBbAGuE/SvUBXMztA\n2lO0D7BeaXuWT5MecjuAHpIekvQZ4N8FdRfV0Q94xPV9GXgD6OXXlpZpAyRHaqTrsBZoT3oADwDm\nm9kRM9sFLC9nLGC+17uKFFlpS3KcRvr1McCcMvc+ZWYH3NYrgL6ev87M/pJr2yIz229m+4AngP5+\nbac17A35iMsCDJK01vtnMA39U07fk8L1WA7cIKk3yfnY2oRb6/x3K7DWzPaa2R7gkOvRjwa77wae\nB64G1gOjJdWSnMK9TajrKPArT+dtQy4f4BrgMU/PK5F73MyOmtmrpLHZG2gFzHTbLgA+nJNfZ2Y7\nLG3bNb+kLCg/7itiZofM7Jdmdi1wEzAE2CXp/Y3dGwTVRDhrQXB6mQt8zcyuAL5HijhgZuOAicAl\nwB8ltTezx4AbgQPAbyUNJu0l+gszu9KPD5pZrZn9E/goKfoxDphVWnFRHY3our/CNZGigpke3c1s\nSRNtcEylE1W0ncBub2tf4HdNvbcJOle8X1INKWIz1PtnJt4/jdR5sswiReRGU94ZLeWQ/x7NpbPz\nsuvH3LEcAPyN5KiPLCdbgXw7T9m+wARgN2mcfpwULa4kn6dw3B8nIH3Cp0hflHRjLr+TpLtJkbmW\npLWXu5vYjiCoCsJZC4LmYzkwLHOCJLUrkGkDvCmpFSmyhsteamZrzWwSsAe4RFIPYIeZPQg8BXyE\ntH5sqKROWR2Suiq9MdfCzBaSHLKrSisuqgNYnekhqRfQBdheoPde1z3jWeA2bweSeikt5l4FDFda\n09YZGFTBXtm6qn7AO2b2jufPIkV0Flj5DdJvUlpv1540jba+QGY1cLOk97hun/M8gC6SrvH0LcDv\naXDM/iGpNVD6RmM5fRvjONuZ2VqS7W/Bo3Ve7jJJFzexzFJW02D3jiQHbZ2krqSp0pkku2bj4nDW\ndwW0oKHtmW2KeIE0xQhpDK3OXRumtObwUlIEbDtwEfCmmR0lbd6dX3PWV1J3X6s2vKDOwnGfF/Cx\nnTlzdZIukvQkaUzWANeZ2fVm9kSFcRUEVUnVv9ETBGcLZrZN0o+A5yUdIa2RGVUi9h3StOEe/80e\n4lOUXiAQ6cG0GbiXtKj9MPAW8GMze1vSRGCJP9gOA7eTom9zcguzv1WgYlEdLwMP+9TUf4FRZnZI\nx62rB9I6qSOSNpOigw+Q1n5tVBLeA9wMLCJNH75EWje1poLJDkraRJoeG5PLryNFnCpFnbaQpj87\nAD8ws13ubB7DzDZKmgus86xZZrZJUjeS83C7pNmu68Nm9h9JM4F6kr1LHcBy+jbGDGCxpF1mljmv\njwNXekQU77cPAOWmnRtjEWlacjMpKvVNM3tL0q3APT6G9tEwxTwD2CJpo5mNKClrP8l5mgj8HXdS\nC7iDNObuIfX/6Ny1v5LsfiEwzswOSpoOLPTo3mKOj9KtB6aRbLDC23MMM3upzLh/oxG7PAisMLNT\njYIGQVWgGMNBEFQTSt8zm2pm/ctcryUtkP/paVWsGVH6rthUM1vm55cDY8zsrjOrWXob1Mxan2k9\ngiBoIKZBgyCoGpQ+MLuQ4sjgWY+ktpJeAQ5kjhqAmdVXg6MWBEF1EpG1IAiCIAiCKiYia0EQBEEQ\nBFVMOGtBEARBEARVTDhrQRAEQRAEVUw4a0EQBEEQBFVMOGtBEARBEARVzP8A+ZsVOy3MJnYAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fee5388fe10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# logits for a random image\n",
    "v = logits[20]\n",
    "plt.figure(figsize=(10, 5));\n",
    "plt.plot(np.sort(softmax(v)), label='$T=1$', linewidth=2);\n",
    "plt.plot(np.sort(softmax(v/2)), label='$T=2$', linewidth=2);\n",
    "plt.plot(np.sort(softmax(v/3)), label='$T=3$', linewidth=2);\n",
    "plt.plot(np.sort(softmax(v/7)), label='$T=7$', linewidth=2);\n",
    "plt.legend();\n",
    "plt.xlabel('classes sorted by probability, most probable ->');\n",
    "plt.ylabel('probability');\n",
    "plt.xlim([245, 255]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temperature = 5.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SqueezeNet(weight_decay=1e-4, image_size=299)\n",
    "\n",
    "# remove softmax\n",
    "model.layers.pop()\n",
    "\n",
    "# usual probabilities\n",
    "logits = model.layers[-1].output\n",
    "probabilities = Activation('softmax')(logits)\n",
    "\n",
    "# softed probabilities\n",
    "logits_T = Lambda(lambda x: x/temperature)(logits)\n",
    "probabilities_T = Activation('softmax')(logits_T)\n",
    "\n",
    "output = concatenate([probabilities, probabilities_T])\n",
    "model = Model(model.input, output)\n",
    "# now model outputs 512 dimensional vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create custom loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def knowledge_distillation_loss(y_true, y_pred, lambda_const):    \n",
    "    \n",
    "    # split in \n",
    "    #    onehot hard true targets\n",
    "    #    logits from xception\n",
    "    y_true, logits = y_true[:, :256], y_true[:, 256:]\n",
    "    \n",
    "    # convert logits to soft targets\n",
    "    y_soft = K.softmax(logits/temperature)\n",
    "    \n",
    "    # split in \n",
    "    #    usual output probabilities\n",
    "    #    probabilities made softer with temperature\n",
    "    y_pred, y_pred_soft = y_pred[:, :256], y_pred[:, 256:]    \n",
    "    \n",
    "    return lambda_const*logloss(y_true, y_pred) + logloss(y_soft, y_pred_soft)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# For testing use usual output probabilities (without temperature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accuracy(y_true, y_pred):\n",
    "    y_true = y_true[:, :256]\n",
    "    y_pred = y_pred[:, :256]\n",
    "    return categorical_accuracy(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def top_5_accuracy(y_true, y_pred):\n",
    "    y_true = y_true[:, :256]\n",
    "    y_pred = y_pred[:, :256]\n",
    "    return top_k_categorical_accuracy(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def categorical_crossentropy(y_true, y_pred):\n",
    "    y_true = y_true[:, :256]\n",
    "    y_pred = y_pred[:, :256]\n",
    "    return logloss(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# logloss with only soft probabilities and targets\n",
    "def soft_logloss(y_true, y_pred):     \n",
    "    logits = y_true[:, 256:]\n",
    "    y_soft = K.softmax(logits/temperature)\n",
    "    y_pred_soft = y_pred[:, 256:]    \n",
    "    return logloss(y_soft, y_pred_soft)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lambda_const = 0.2\n",
    "\n",
    "model.compile(\n",
    "    optimizer=optimizers.SGD(lr=1e-2, momentum=0.9, nesterov=True), \n",
    "    loss=lambda y_true, y_pred: knowledge_distillation_loss(y_true, y_pred, lambda_const), \n",
    "    metrics=[accuracy, top_5_accuracy, categorical_crossentropy, soft_logloss]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "400/400 [==============================] - 36s - loss: 6.1129 - accuracy: 0.3999 - top_5_accuracy: 0.6179 - categorical_crossentropy: 3.0321 - soft_logloss: 5.5041 - val_loss: 5.8628 - val_accuracy: 0.5820 - val_top_5_accuracy: 0.8186 - val_categorical_crossentropy: 1.8152 - val_soft_logloss: 5.4963\n",
      "Epoch 2/30\n",
      "400/400 [==============================] - 35s - loss: 5.8049 - accuracy: 0.6626 - top_5_accuracy: 0.8586 - categorical_crossentropy: 1.5629 - soft_logloss: 5.4881 - val_loss: 5.8121 - val_accuracy: 0.6291 - val_top_5_accuracy: 0.8523 - val_categorical_crossentropy: 1.5437 - val_soft_logloss: 5.4984\n",
      "Epoch 3/30\n",
      "400/400 [==============================] - 35s - loss: 5.7414 - accuracy: 0.7319 - top_5_accuracy: 0.9034 - categorical_crossentropy: 1.2440 - soft_logloss: 5.4870 - val_loss: 5.8001 - val_accuracy: 0.6484 - val_top_5_accuracy: 0.8562 - val_categorical_crossentropy: 1.4661 - val_soft_logloss: 5.5007\n",
      "Epoch 4/30\n",
      "400/400 [==============================] - 35s - loss: 5.7077 - accuracy: 0.7751 - top_5_accuracy: 0.9242 - categorical_crossentropy: 1.0728 - soft_logloss: 5.4864 - val_loss: 5.7889 - val_accuracy: 0.6586 - val_top_5_accuracy: 0.8652 - val_categorical_crossentropy: 1.4034 - val_soft_logloss: 5.5009\n",
      "Epoch 5/30\n",
      "400/400 [==============================] - 36s - loss: 5.6860 - accuracy: 0.8019 - top_5_accuracy: 0.9362 - categorical_crossentropy: 0.9609 - soft_logloss: 5.4860 - val_loss: 5.7869 - val_accuracy: 0.6654 - val_top_5_accuracy: 0.8643 - val_categorical_crossentropy: 1.3793 - val_soft_logloss: 5.5027\n",
      "Epoch 6/30\n",
      "400/400 [==============================] - 35s - loss: 5.6694 - accuracy: 0.8250 - top_5_accuracy: 0.9465 - categorical_crossentropy: 0.8742 - soft_logloss: 5.4858 - val_loss: 5.7876 - val_accuracy: 0.6666 - val_top_5_accuracy: 0.8648 - val_categorical_crossentropy: 1.3902 - val_soft_logloss: 5.5004\n",
      "Epoch 7/30\n",
      "400/400 [==============================] - 36s - loss: 5.6573 - accuracy: 0.8438 - top_5_accuracy: 0.9534 - categorical_crossentropy: 0.8111 - soft_logloss: 5.4855 - val_loss: 5.7820 - val_accuracy: 0.6805 - val_top_5_accuracy: 0.8646 - val_categorical_crossentropy: 1.3401 - val_soft_logloss: 5.5040\n",
      "Epoch 8/30\n",
      "400/400 [==============================] - 35s - loss: 5.6475 - accuracy: 0.8541 - top_5_accuracy: 0.9609 - categorical_crossentropy: 0.7586 - soft_logloss: 5.4853 - val_loss: 5.7853 - val_accuracy: 0.6695 - val_top_5_accuracy: 0.8656 - val_categorical_crossentropy: 1.3582 - val_soft_logloss: 5.5028\n",
      "Epoch 9/30\n",
      "400/400 [==============================] - 36s - loss: 5.6398 - accuracy: 0.8653 - top_5_accuracy: 0.9645 - categorical_crossentropy: 0.7170 - soft_logloss: 5.4852 - val_loss: 5.7838 - val_accuracy: 0.6777 - val_top_5_accuracy: 0.8689 - val_categorical_crossentropy: 1.3341 - val_soft_logloss: 5.5054\n",
      "Epoch 10/30\n",
      "400/400 [==============================] - 35s - loss: 5.6333 - accuracy: 0.8761 - top_5_accuracy: 0.9675 - categorical_crossentropy: 0.6805 - soft_logloss: 5.4852 - val_loss: 5.7866 - val_accuracy: 0.6773 - val_top_5_accuracy: 0.8674 - val_categorical_crossentropy: 1.3570 - val_soft_logloss: 5.5029\n",
      "Epoch 11/30\n",
      "400/400 [==============================] - 35s - loss: 5.6178 - accuracy: 0.9008 - top_5_accuracy: 0.9756 - categorical_crossentropy: 0.6073 - soft_logloss: 5.4839 - val_loss: 5.7768 - val_accuracy: 0.6875 - val_top_5_accuracy: 0.8754 - val_categorical_crossentropy: 1.3110 - val_soft_logloss: 5.5022\n",
      "Epoch 12/30\n",
      "400/400 [==============================] - 36s - loss: 5.6153 - accuracy: 0.9062 - top_5_accuracy: 0.9758 - categorical_crossentropy: 0.5971 - soft_logloss: 5.4835 - val_loss: 5.7697 - val_accuracy: 0.6922 - val_top_5_accuracy: 0.8762 - val_categorical_crossentropy: 1.2696 - val_soft_logloss: 5.5033\n",
      "Epoch 13/30\n",
      "400/400 [==============================] - 35s - loss: 5.6151 - accuracy: 0.9054 - top_5_accuracy: 0.9755 - categorical_crossentropy: 0.5958 - soft_logloss: 5.4834 - val_loss: 5.7802 - val_accuracy: 0.6838 - val_top_5_accuracy: 0.8695 - val_categorical_crossentropy: 1.3274 - val_soft_logloss: 5.5022\n",
      "Epoch 14/30\n",
      "400/400 [==============================] - 36s - loss: 5.6137 - accuracy: 0.9082 - top_5_accuracy: 0.9767 - categorical_crossentropy: 0.5895 - soft_logloss: 5.4833 - val_loss: 5.7724 - val_accuracy: 0.6900 - val_top_5_accuracy: 0.8754 - val_categorical_crossentropy: 1.2935 - val_soft_logloss: 5.5012\n",
      "Epoch 15/30\n",
      "400/400 [==============================] - 36s - loss: 5.6126 - accuracy: 0.9096 - top_5_accuracy: 0.9762 - categorical_crossentropy: 0.5829 - soft_logloss: 5.4834 - val_loss: 5.7723 - val_accuracy: 0.6926 - val_top_5_accuracy: 0.8773 - val_categorical_crossentropy: 1.2860 - val_soft_logloss: 5.5025\n",
      "Epoch 16/30\n",
      "400/400 [==============================] - 36s - loss: 5.6126 - accuracy: 0.9095 - top_5_accuracy: 0.9770 - categorical_crossentropy: 0.5840 - soft_logloss: 5.4832 - val_loss: 5.7769 - val_accuracy: 0.6871 - val_top_5_accuracy: 0.8754 - val_categorical_crossentropy: 1.3071 - val_soft_logloss: 5.5029\n",
      "Epoch 17/30\n",
      "400/400 [==============================] - 36s - loss: 5.6119 - accuracy: 0.9113 - top_5_accuracy: 0.9770 - categorical_crossentropy: 0.5808 - soft_logloss: 5.4832 - val_loss: 5.7728 - val_accuracy: 0.6900 - val_top_5_accuracy: 0.8742 - val_categorical_crossentropy: 1.2930 - val_soft_logloss: 5.5016\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fee41efe278>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(\n",
    "    train_generator, \n",
    "    steps_per_epoch=400, epochs=30, verbose=1,\n",
    "    callbacks=[\n",
    "        EarlyStopping(monitor='val_accuracy', patience=4, min_delta=0.01),\n",
    "        ReduceLROnPlateau(monitor='val_accuracy', factor=0.1, patience=2, epsilon=0.007)\n",
    "    ],\n",
    "    validation_data=val_generator, validation_steps=80, workers=4\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss/epoch plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8XHW9//HXJ/sySZsmmaZ72qbQDehGWUq1inCxQuEi\nUK6ooF54qCjgjsvPq9778HKvXhUERRQEFFGkqIggm2Upe1tK6QZdbGjaZm2brdnz/f1xTibTNGnT\nksmZZN7Px+M8ZuacMzOfpM2853u+3/M95pxDREQEICnoAkREJH4oFEREJEKhICIiEQoFERGJUCiI\niEiEQkFERCIUCiIiEqFQEBGRCIWCiIhEpARdwLEqKChwxcXFQZchIjKkrFmzpto5V3i0/YZcKBQX\nF7N69eqgyxARGVLMrLQ/++nwkYiIRCgUREQkQqEgIiIRQ65PQUTkeLS1tVFWVkZzc3PQpcRURkYG\n48ePJzU19bier1AQkYRQVlZGTk4OxcXFmFnQ5cSEc46amhrKysqYPHnycb2GDh+JSEJobm4mPz9/\n2AYCgJmRn5//rlpDCgURSRjDORC6vNufMWahYGYZZvaqmb1hZhvN7Lu97JNuZn8ws21m9oqZFceq\nni3ldfz3Y5upb26L1VuIiAx5sWwptADvd86dAswBzjOz03vs8ylgv3OuBPgx8D+xKmbXviZ+8ewO\ntlY2xOotRET6dODAAX72s58d8/OWLl3KgQMHYlBR72IWCs7T9Qmc6i+ux24XAvf49x8EzrYYte9K\nwiEAtikURCQAfYVCe3v7EZ/36KOPMnLkyFiVdZiY9imYWbKZrQMqgSedc6/02GUcsAvAOdcO1AL5\nsahlQl4maclJbFcoiEgAbrzxRrZv386cOXM49dRTWbx4McuWLWPmzJkAXHTRRcyfP59Zs2Zxxx13\nRJ5XXFxMdXU1O3fuZMaMGVx99dXMmjWLc889l6ampgGvM6ZDUp1zHcAcMxsJ/MnMZjvnNhzr65jZ\nNcA1ABMnTjyuWlKSk5hckK2Wgojw3b9uZNOeugF9zZljc/mPC2b1uf2mm25iw4YNrFu3jmeeeYYP\nfehDbNiwITJ09K677mLUqFE0NTVx6qmn8uEPf5j8/EO/I2/dupX777+fX/7yl1x22WWsWLGCj370\nowP6cwzK6CPn3AFgJXBej027gQkAZpYCjABqenn+Hc65Bc65BYWFR53kr08lo0Nsq1IoiEjwFi5c\neMi5BLfccgunnHIKp59+Ort27WLr1q2HPWfy5MnMmTMHgPnz57Nz584BrytmLQUzKwTanHMHzCwT\nOIfDO5IfBq4EXgIuAf7hnOvZ7zBgSgpDPPbmXprbOshITY7V24hInDvSN/rBkp2dHbn/zDPP8NRT\nT/HSSy+RlZXFkiVLej3XID09PXI/OTk5JoePYtlSGAOsNLP1wGt4fQqPmNn3zGyZv8+dQL6ZbQO+\nCNwYw3ooCYfodLCjqjGWbyMicpicnBzq6+t73VZbW0teXh5ZWVls2bKFl19+eZCr6xazloJzbj0w\nt5f134663wxcGqsaeoqMQKpqYObY3MF6WxER8vPzWbRoEbNnzyYzM5PRo0dHtp133nncfvvtzJgx\ngxNPPJHTT+85en/wJNTcR5MLskkyDUsVkWD87ne/63V9eno6jz32WK/buvoNCgoK2LChe5zOl7/8\n5QGvDxJsmouM1GQmjMrSsFQRkT4kVCiA19msloKISO8SLxTCIf5Z3Uh7R2fQpYiIxJ2EC4Wp4RCt\nHZ3s2j/wQ7lERIa6hAuFaZoDSUSkTwkXClMVCiIifUq4UMjNSGV0brpCQUTiWigUCuR9Ey4UwOts\n3lbZ+5mFIiKJLKFOXutSUhhixdrdOOcS4vJ8IhK8G2+8kQkTJnDttdcC8J3vfIeUlBRWrlzJ/v37\naWtr47/+67+48MILA60zMUMhHKKhpZ3yumbGjMgMuhwRGWyP3Qjlbw7saxadBB+8qc/Ny5cv54Yb\nboiEwgMPPMDjjz/OddddR25uLtXV1Zx++uksW7Ys0C+rCRkK0Z3NCgURGQxz586lsrKSPXv2UFVV\nRV5eHkVFRXzhC1/gueeeIykpid27d1NRUUFRUVFgdSZkKERfmnPxtOO/PoOIDFFH+EYfS5deeikP\nPvgg5eXlLF++nPvuu4+qqirWrFlDamoqxcXFvU6ZPZgSMhQKQ+nkZqRoBJKIDKrly5dz9dVXU11d\nzbPPPssDDzxAOBwmNTWVlStXUlpaGnSJiRkKZuaPQFIoiMjgmTVrFvX19YwbN44xY8ZwxRVXcMEF\nF3DSSSexYMECpk+fHnSJiRkKANPCOTy9pSLoMkQkwbz5ZncHd0FBAS+99FKv+zU0BPOlNSHPUwCv\nX6G6oZX9ja1BlyIiEjcSOhTAuwqbiIh4FArqVxBJGM65oEuIuXf7MyZsKIwbmUlGapJCQSRBZGRk\nUFNTM6yDwTlHTU0NGRkZx/0aCdvRnJRkTCnQCCSRRDF+/HjKysqoqqoKupSYysjIYPz48cf9/IQN\nBfAOIa0p3R90GSIyCFJTU5k8eXLQZcS9hD18BF4o7D7QxMHW9qBLERGJCwkfCgA7qhoDrkREJD4o\nFNAIJBGRLgkdCsX52SQnmUJBRMSX0KGQlpLEpPwstuoqbCIiQIKHAnhXYVNLQUTEo1AIhyitOUhb\nR2fQpYiIBE6hEA7R3ukordEIJBERhYJGIImIRMQsFMxsgpmtNLNNZrbRzK7vZZ8lZlZrZuv85dux\nqqcvUwsVCiIiXWI5zUU78CXn3FozywHWmNmTzrlNPfZ73jl3fgzrOKLs9BTGjshQKIiIEMOWgnNu\nr3NurX+/HtgMjIvV+70bU8MhXVdBRIRB6lMws2JgLvBKL5vPMLM3zOwxM5s1GPX0VBIOsb2ykc7O\n4TulrohIf8Q8FMwsBKwAbnDO1fXYvBaY5Jw7Bfgp8Oc+XuMaM1ttZqtjMe3ttHAOTW0d7KltGvDX\nFhEZSmIaCmaWihcI9znnHuq53TlX55xr8O8/CqSaWUEv+93hnFvgnFtQWFg44HV2jUDaqn4FEUlw\nsRx9ZMCdwGbn3I/62KfI3w8zW+jXUxOrmvrSFQrbFQoikuBiOfpoEfAx4E0zW+ev+wYwEcA5dztw\nCfAZM2sHmoDLXQDXyhuVncao7DSNQBKRhBezUHDOrQLsKPvcCtwaqxqOheZAEhHRGc0RXcNSh/NF\nvUVEjkah4CsJhzhwsI2axtagSxERCYxCwac5kEREFAoRCgUREYVCxNgRGWSlJSsURCShKRR8ZkZJ\nWCOQRCSxKRSiaFiqiCQ6hUKUqeEQ5XXN1De3BV2KiEggFApRItNdVOnSnCKSmBQKUTQCSUQSnUIh\nyqRRWaQmm0JBRBKWQiFKSnISxfnZCgURSVgKhR5KwiG269KcIpKgFAo9lIRDlNY00tLeEXQpIiKD\nTqHQQ0k4RKeDndUHgy5FRGTQKRR66L40Z33AlYiIDD6FQg9TC0OYaViqiCQmhUIPGanJjM/LVCiI\nSEJSKPRCcyCJSKJSKPSiJBxiR3UjHZ26NKeIJBaFQi9KwiFa2zsp268RSCKSWBQKvdAcSCKSqBQK\nvSgpzAEUCiKSeBQKvRiRlUpBKF2hICIJR6HQh2nhEFsVCiKSYBQKfSgJh9he2YBzGoEkIolDodCH\nknCI+pZ2Kutbgi5FRGTQKBT6oBFIIpKIFAp9UCiISCJSKPQhnJNOTnqKQkFEEopCoQ9mxtSw5kAS\nkcSiUDiCknCIbbo0p4gkkJiFgplNMLOVZrbJzDaa2fW97GNmdouZbTOz9WY2L1b1HI+ScIiq+hZq\nD7YFXYqIyKCIZUuhHfiSc24mcDpwrZnN7LHPB4Fp/nIN8PMY1nPMSgr9zuYqXYVNRBJDzELBObfX\nObfWv18PbAbG9djtQuBe53kZGGlmY2JV07GaNlojkEQksQxKn4KZFQNzgVd6bBoH7Ip6XMbhwYGZ\nXWNmq81sdVVVVazKPMz4vCzSUpIUCiKSMI45FMwsycxyj2H/ELACuME5V3es7wfgnLvDObfAObeg\nsLDweF7iuCQnGVMKshUKIpIw+hUKZvY7M8s1s2xgA7DJzL7Sj+el4gXCfc65h3rZZTcwIerxeH9d\n3NAIJBFJJP1tKcz0v+VfBDwGTAY+dqQnmJkBdwKbnXM/6mO3h4GP+6OQTgdqnXN7+1nToCgJhyjb\n30RzW0fQpYiIxFxKP/dL9b/1XwTc6pxrM7OjTR+6CC843jSzdf66bwATAZxztwOPAkuBbcBB4BPH\nWH/MlYRDOAfbqxqYNXZE0OWIiMRUf0PhF8BO4A3gOTObBByxf8A5twqwo+zjgGv7WUMgoudAUiiI\nyHDXr1Bwzt0C3BK1qtTM3hebkuLL5IJskgy2q7NZRBJAfzuar/c7ms3M7jSztcD7Y1xbXEhPSWbi\nqCx1NotIQuhvR/Mn/Y7mc4E8vL6Cm2JWVZwpCYfYWqFQEJHhr7+h0NU3sBT4jXNuI0fpLxhOSsI5\n7KxppL2jM+hSRERiqr+hsMbMnsALhcfNLAdImE/IknCItg5H6b6DQZciIhJT/R199ClgDrDDOXfQ\nzPKJw+GjsRI9AmmqP0meiMhw1K+WgnOuE+9s42+Z2Q+BM51z62NaWRyZWpgNaGI8ERn++jv66Cbg\nemCTv1xnZt+PZWHxJCcjlaLcDA1LFZFhr799CkuBc5xzdznn7gLOA86PXVkx0NoIL/0MOo+vK0Rz\nIIlIIjiWWVJHRt0feqf2bvwTPP51+Md/HtfTS8Ihtlc24J2ELSIyPPW3o/m/gdfNbCXeUNT3ADfG\nrKpYmHMFlL0Gq34EeZNg/lXH9PSp4RCNrR3srW1m7MjM2NQoIhKw/k5zcb+ZPQOc6q/6mnOuPGZV\nxYIZLP0/qC2DR74IueNh2gf6/fSuS3NurWxQKIjIsHXEw0dmNq9rAcbgXRmtDBjrrxtaklPg0rsh\nPBP+eCWUv9nvp+rSnCKSCI7WUvi/I2xzDMX5j9Jz4IoH4Jdnw32Xwb8/BSMOuwLoYfKz0xiZlapQ\nEJFh7Yih4JwbnjOh5o6FK/4Id50Hv7sMPvEYZBz5CqNmRklhSMNSRWRY6+95Chf3spxtZuFYFxgz\nRbPhsnugcjP88SroaDvqUzQsVUSGu/4OSf0U8CvgCn/5JfA14AUzO+JlOeNaydlwwU9g+9Pwty/C\nUYabloRD7GtsZV9j6yAVKCIyuPobCinADOfch51zHwZm4vUpnIYXDkPXvI/D4i/D2nu94apHMDWs\nzmYRGd76GwoTnHMVUY8r/XX7gKMfd4l37/8WnHQpPP09ePPBPnfrGpaqUBCR4aq/J689Y2aPAH/0\nH1/ir8sGDsSkssFkBhfeBnV74M+fgZwxULzosN3GjcwkMzVZoSAiw1Z/WwrXAr/Gmz57DnAPcK1z\nrnHYjFBKSYflv4WRk+D3H4HqrYftkpRkTCnMZmtlfQAFiojEXn+nznbAKuAfwNPAc244TgKUNcob\nqpqUAvddAg1Vh+3SNQeSiMhw1N8hqZcBr+IdNroMeMXMLollYYEZNRk+8gDUV8D9l0ProVdbmxYO\nsae2mcaW9oAKFBGJnf4ePvomcKpz7krn3MeBhcD/i11ZARs/Hz78S9i9Bh66Gjo7Ipu6rsK2Xecr\niMgw1N9QSHLOVUY9rjmG5w5NMy6Af/k+bHkEnvx2ZHWJhqWKyDDW39FHfzezx4H7/cfLgUdjU1Ic\nOeOzcKAUXrrV64A+7Rom5WeTkmQKBREZlvo7dfZXzOzDQNc4zTucc3+KXVlx5F++Dwd2wd+/BiPG\nkzp9KZPysxQKIjIs9belgHNuBbAihrXEp6Rkr3/h7vNhxafgqr9REg6xVaEgIsPQ0a6nUG9mdb0s\n9WZWN1hFBi4tGz7yB8gugN8tZ15uHaU1B2ltP77rPYuIxKsjhoJzLsc5l9vLkuOcO/Jc08NNKAxX\nPAgdLfzb1i+R3dnAq//cF3RVIiIDaniPIBpohSfC8vvIObiLe7J+wu0PPMz+hpagqxIRGTAxCwUz\nu8vMKs1sQx/bl5hZrZmt85dv97Zf3Jm8GLvoZ8xxW/ht2xdovnk+7un/hL3rjzr1tohIvOt3R/Nx\nuBu4Fbj3CPs875w7P4Y1xMbJl2GT38sLf7sHNv6Z0c//CHv+h5A3GWZe6C1j53oT7YmIDCExayk4\n554Dhu9B95zRnLn8K/y65BbOaPs5ZWfd5E2R8eJP4Zfvg5tPhie+BWWr1YIQkSEj6D6FM8zsDTN7\nzMxmBVzLMTMzfnjpySSHCrni9enUX/oAfGWbNw134XR4+Xb41dnw49nw96/DO69Ap0YsiUj8slhO\ndmpmxcAjzrnZvWzLBTqdcw1mthS42Tk3rY/XuQa4BmDixInzS0tLY1bz8Vi9cx/L73iZpSeN4ZbL\n52Bdh42aDsBbj8Gmv3iX/Oxo9a7VMGOZd4hp4uneeRAiIjFmZmuccwuOul9QodDLvjuBBc656iPt\nt2DBArd69eoBqW8g3bZyGz94/C1uuvgkLl848fAdmuvg7cdh059h21PQ3gyh0d4cSzOWwcQzICVt\n8AsXkYTQ31CIZUfzEZlZEVDhnHNmthDvUFZNUPW8W59571Re2l7Dd/66kbkT8zixKOfQHTJy4eRL\nvaWlAbY+4bUg1v0OXvsVpGbBpEUwZYm3hGdCUtBH90Qk0cSspWBm9wNLgAKgAvgPIBXAOXe7mX0O\n+AzQDjQBX3TOvXi0143XlgJAZX0zS29eRV5WKg9/7iwy0/pxaKi1EXY8071Uv+2tzy6Eye/tDomR\nE2JUtYgkgrg4fBQL8RwKAKu2VvOxu17hsvkT+J9LTj72F6jdDf98tjskGiq89fkl3QFRfBZk5g1U\nySKSABQKAfrB41u4beV2br58DhfOGXf8L+QcVG2B7Su9gNi5CtoawZK88yCmLPGWCad515gWEemD\nQiFA7R2dXH7Hy2zeW8cj1y1mckH2AL1wq3c1uK5WRNlr4DogJRMmneG3IhZDXrHXktDJcyLiUygE\nbM+BJpbe8jzj8zJZ8ZkzSU+JwdDT5joofbE7JKo2d29LyYCcIm8IbK+3Y73b9NDA1xWtox1aG6Cj\nzZtlVkElEgiFQhx4clMFV9+7mqvOLOY7ywbh3Ly6vbDrFajbA/V7ob7cv93rbWtrPPw5aTl+SPQI\njtwxkFXgnVvR2ugvDd5t28FDHx9yv8e2jqgJA7PyYdx8f1kA4+ZB1qjY/15EJP6HpCaCc2aO5pOL\nJnPXC//kjKn5/Musoti+Ye4YmHVR39tb6qOCIuq2bo93u+sV77ajHzO/poW860ykZnXfzxgJueP8\nx1neuq5tlgTlG7zDX1ufBPwvI3mTYfyC7qAoOglSMwbk1yEix06hEGNf++CJvLZzH199cD2zx41g\n3MjM4IpJz/GWgl5PHPc4B037vXBorPIOQ6Vl+x/y/gd8Sua7O4eiuQ72rvMComw17HwB3vyjty0p\nFYpmH9qiyC95d+/X2QkttXBwn7/UdC8tdZA5qkdLqQhSB/nfqa3JG2lWXwEN5V5tIyfBmDmQnT+4\ntUhC0+GjQVBa08iHblnFiUU5/P6a00lN1klph6nb44VEZHkdWuu9bekjYNzc7qAoOtk7rNX1Ad/U\n44O+54d/036vQ/5YZIzsDoncsb33y4RGQ3Jq36/hHDQf6P6gb6j0wrahovu2Kwhaavt+ndzxMOaU\nQ5ecIvXPyDFRn0KcefiNPVx3/+t8dslUvnre9KDLiX+dHVC91Q+I1d5txUbobO/7OUkpXr9FVr73\n7T9rlP94VI/1XetGeX0qzQd6OazW4xBbfXkvwWJe53nOGG/JLvReK/KNv6L3Q3EpmZAzGkJF/q2/\n5BR560Jhb/TYvh2w943upWYbkcNu2WEYc/KhQTFy0tAKis4O77DiUKp5CFMoxKGvP7Se+1/dxb2f\nXMh7TigMupyhp63Ju5hRxQbvMFZmjw/99JzYfcB0dnitjuiwqOsRHo3VkDHC+1Dvakl03UbfP946\nWxq8nz06KCo3d4dVxoiokJjj3Y6aGj/TpbTUezMFl67yDhnued07LDlqKoyaAvlTvfv5/mMNQhhQ\nCoU41NTawYW3rWJfYyuPXr+YcI46VOVdamuGyo1eWHYFRcXG7hZKWghGz4bRs2D0TAj7txkjYl9b\n0wF45yXvpMvSF73aXIfXohs71zvpsq0J9m2Hmh1Qu4tISwi8Q3hdQREdGqMmx29gtLd4hwkbK6Gh\nCg5We/1kGbmQnuvdZozw7qfnDOosyQqFOLW1op4Lbl3FvIl5/OZTp5GcpKazDLCONqh6y/sQLl/f\nHRQtdd375I73wmH0rO6gyJ/27mbqbayB0he6l/INgIPkNG/AQPEib9LHCQu9ll5P7S2wfyfUbPeC\nYt8O//4OqC3jkMDIzDu0VTFqihciqRne4IiUdP+25+P0Y2+ltTb6H/RV3R/4jdWHfvh33R6pb6g3\naTleSESHRnru4esyRnr3C0q8n/U4KBTi2AOv7eKrK9bzpXNO4PNnH2EkkMhAcc77YK3c5AVE5Sao\n2ORNwNjZ5u2TlAIFJ3gz9Ea3KkZM6P2DtL6i+1BQ6QvelCzg9ZlMWOgFQPEiLxDe7TDjtmYvMPZt\n7w6KrhZGXdmxvVbPkEjJ7PE43Rsh1/VB39v5PeB9UIfCXv9OqNDrU4rcD/vbCrxDj821Xig313qv\n3VLn3UavP2xdXfe/TZdFN8A53z2uX6FCIY4557jhD+v46xt7+P01Z7Bwcpw2hWX4a2/1OrB7hkXt\nO937pOdCeIYXFvklUP2WdzioZpu3PS3kXTBq0pkw6Szv0NBgXhukrQn2l3onTLY3ewHS3rW0RN2P\nftziPS/6cfR+6bk9PvDD3od+9P1Y/4zOebVEB0V2gTeNzXFQKMS5hpZ2zr/leVraO3n0usXkZesC\nOxJHmuu8TuzKjV5IdIVG8wHv0MbEM/3DQWdC0SmQrFOe4p1CYQjYsLuWi3/2IounFfDLjy8gSf0L\nEs+c80ZgZebpMrJDUH9DIU7GqiWm2eNG8I2l03l6SyUfv+tVKuuagy5JpG/mn5ehQBjWFAoBu/LM\nYm66+CRWl+7jvJufZ+WWyqBLEpEEplAImJlx+cKJPPL5sxidm8En7n6N7/51Iy3txzgtg4jIAFAo\nxImScA5/+uyZXHVmMb9+YSf/etuLbKtsCLosEUkwCoU4kpGazHeWzeLOKxewt7aJC366igde28VQ\nGwwgIkOXQiEOnT1jNH+/4T3MnTiSr65Yz+fvf53aprajP1FE5F1SKMSp0bkZ/OZTp/HV807ksQ3l\nLL35edaU7gu6LBEZ5hQKcSw5yfjskhIe/PQZJCXBZb94mVv/sZWOTh1OEpHYUCgMAXMn5vG36xbz\noZPG8MMn3uaKX73M3tqmoMsSkWFIoTBE5GakcvPlc/jhpaewvqyWD978PE9sLA+6LBEZZhQKQ4iZ\nccn88Tzy+bMYn5fJNb9Zw7f/soHmNp3TICIDQ6EwBE0pDLHiM2dy9eLJ3PtSKRfd9gJbK+qDLktE\nhgGFwhCVnpLMNz80k7s/cSrVDS2c/9NV3PdKqc5pEJF3RaEwxC05Mcyj1y9m4eRRfPNPG/j0b9ew\na9/BoMsSkSFKoTAMhHMyuOcTC/nG0ums3FLFkh8+wxf+sI63dUhJRI6RrqcwzJTXNvOr53dw3yvv\n0NTWwTkzR/PZJVOZOzEv6NJEJECBX2THzO4CzgcqnXOze9luwM3AUuAgcJVzbu3RXleh0D/7G1u5\n+8Wd3P3iTmqb2jhzaj6fXVLCopJ87FgvXC4iQ148XGTnbuC8I2z/IDDNX64Bfh7DWhJOXnYaXzjn\nBF648f18c+kMtlU28NE7X+Gi217g7xvK6dRZ0SLSi5iFgnPuOeBIk/VcCNzrPC8DI81sTKzqSVSh\n9BSufs8Unv/a+/j+v57EgaY2Pv3bNZz7k+dYsaaMto7OoEsUkTgSZEfzOGBX1OMyf53EQHpKMh85\nbSJPf/G93Hz5HFKSjC/98Q2W/OAZ7n1pp06AExFgiIw+MrNrzGy1ma2uqqoKupwhLSU5iQvnjOOx\n6xdz55ULKBqRwbf/spGz/ucf3LZyG3XNmqJbJJEFGQq7gQlRj8f76w7jnLvDObfAObegsLBwUIob\n7syMs2eM5sFPn8EfrjmdmWNH8IPH32LRf/+D//37FqobWoIuUUQCkBLgez8MfM7Mfg+cBtQ65/YG\nWE9CMjNOm5LPaVPy2bC7lp89s42fP7udO1f9k+WnTuAjp01kelFu0GWKyCCJ5ZDU+4ElQAFQAfwH\nkArgnLvdH5J6K94IpYPAJ5xzRx1rqiGpsbe9qoHbn9nOn9ftpq3DMXNMLhfPG8eFc8ZRmJMedHki\nchwCP08hVhQKg6emoYVH1u/lobVlvFFWS3KS8d4TCrl43jg+MGM0GanJQZcoIv2kUJABta2ynofW\n7uZPr+9mb20zORkpnH/yGC6eN54Fk/J0QpxInFMoSEx0dDpe2VHDirW7eWzDXg62djBxVBb/Oncc\nF88bx6T87KBLFJFeKBQk5g62tvP4xnIeWrubVduqcQ4WTMrj4nnj+dDJYxiRmRp0iSLiUyjIoNpb\n28SfX9/DirVlbKtsIC0liXNmjObieeN4zwmFpCYPiVNiRIYthYIEwjnHht11rFhbxsNv7GFfYyv5\n2WlccMpYzp4R5tTiUeqgFgmAQkEC19bRybNvVfHQ62U8tamS1o5O0lKSOLU4j7NKCjmrpIBZY3NJ\nSlIntUisKRQkrjS2tPPqP/exals1q7ZW85Z/AaC8rFTOnFrAWdMKOKukgAmjsgKuVGR46m8oBHlG\nsySQ7PQU3jc9zPumhwGorG/mxW01PL+1mlXbqvjbm97J7JPys1hUUsDikgLOmJrPyKy0IMsWSThq\nKUjgnHNsr2pg1dZqVm2r5uUd+2hoaccMTh43gkUlXkti/qQ80lPUHyFyPHT4SIasto5O3th1gOe3\nVvPCtmpe33WAjk5HRmoSCyfnc1ZJPguKRzFrbK5CQqSfFAoybNQ3t/HKDr8/Yls12yobAEhLTmL2\nuFzmTcwunVbrAAAL5UlEQVRj3qQ85k3Mo2hERsDVisQnhYIMW5V1zax9Zz9r3znA2tL9rN9dS2u7\ndwW5sSMymOsHxLyJI5k1dgRpKTpHQkQdzTJshXMzOG/2GM6b7V29tbW9k01761hbup+17+zn9XcO\n8Lf1Xsd1WkoSJ40bwbyJI5k3MY+5ak2IHJFaCjIsVdQ1R0Ji7TsHePMIrYnpRblkpqlvQoY3HT4S\nidLS3sGmPXXeIad39vN66X721DYDYAaT87M5sSiHE4tymF6Uw4lFuUwclUWyTqyTYUKHj0SipKck\nM9c/fPQpJgNQXtvMul372by3ni3ldWzeW8ffN5bT9T0pMzWZE0aH/LDIZbofGPkhXWhIhi+1FESi\nHGxtZ2tFA2+V17O5vI63yut5q7yemsbWyD4FoXS/NZHjB0Uu00aHNKeTxDW1FESOQ1ZaCqdMGMkp\nE0Yesr6qvoW3yr0WxRY/KH77ciktfj9FkkFxfjZTwyGmFoaYUpjN1MIQUwuzdVa2DCkKBZF+KMxJ\npzAnnbOmFUTWdXQ6Smsa2VJez5byet4ur2d7VQPPvFVJW0d3Czw/O+3QoAhnM6UgxPi8TFI0pbjE\nGR0+Ehlg7R2d7NrfxI6qBrZXNbCjqjFyG30YKi05iUn5WYcExdSwFx65GbpAkQwsHT4SCUhKchKT\nC7KZXJDN2TNGH7Jtf2MrO6ob2B4VFG9X1vPU5graO7u/oIVz0jlzaj7nziriPScUEkrXn6oMDv1P\nExlEedlpzM8exfxJow5Z39bRyTv7DrK9soEd1Y1s2VvHs29X8ed1e0hLTuLMknzOnVnEB2aECefq\n5DuJHR0+EolT7R2drCndzxObKnhyUwXv7DsIwNyJIzln5mjOnTmaqYUhzHQuhRydTl4TGUacc7xd\n0cATG8t5cnMF68tqAZhSkM05M0dzzszRzJ2Yp5PtpE8KBZFhbG9tE09tquCJTRW8vKOGtg5HQSiN\ns6d7AXHWtAKdNyGHUCiIJIi65jaeeauKJzdV8MyWSupb2slMTeY9JxRwzswizp4eJi9b50okOo0+\nEkkQuRmpLDtlLMtOGUtreycv76jhSb8f4vGNFZhBdloKmWnJZKUlk5nq3WZFret6nJWW7K1LPXR7\nZtT25CTDgCQzzLxbgKQe683AMJIMzPxbDEsisl9qchKpyaZ+kTiiloLIMOWc483dtTz3dhX7Gtto\namvnYGsHB1s7aGrt4GCr97ipzVt3sKWdg20dBPGRkJacRFqKv/j3U5ONtJRk0lKSSI/aHlnvr0uP\nrEsiLTmZ9NSkQ14vPSWp19fv3pbcvc3fbgbOgcNFfh/Rj53/+3X+eqK2dUavxzvbPTnJDl3Mux3M\nMFRLQSTBmRknjx/JyeNHHn1nn3OOlvZOPzza/fDwg8QPlY5OF/mA7Oz0PiA7/U/Krg/ETufv0/W4\ns2u9v87ft62jk9YOR2t7p7d0dETut3V4tbR2dNLa7tVzoKmTtnbnr+v0trd3RB53Dq3vuIcGhlnv\n4ZHcve3fFk7k3xdPiWlNCgURiTAzMlKTyUhNZtQQ7Ido7+iMBEQkNKIeH3FbVLg45x/+8r/Jdx0K\n824Pfext7zo85t3v2g8znHO0dzg6naO909ERtXSt6+zssc05Ojq828g25ygYhBl6FQoiMmykJCeR\nkpyE5iA8fpqNS0REImIaCmZ2npm9ZWbbzOzGXrZfZWZVZrbOX/49lvWIiMiRxezwkZklA7cB5wBl\nwGtm9rBzblOPXf/gnPtcrOoQEZH+i2VLYSGwzTm3wznXCvweuDCG7yciIu9SLENhHLAr6nGZv66n\nD5vZejN70Mwm9PZCZnaNma02s9VVVVWxqFVERAi+o/mvQLFz7mTgSeCe3nZyzt3hnFvgnFtQWFg4\nqAWKiCSSWIbCbiD6m/94f12Ec67GOdfiP/wVMD+G9YiIyFHEMhReA6aZ2WQzSwMuBx6O3sHMxkQ9\nXAZsjmE9IiJyFDEbfeScazezzwGPA8nAXc65jWb2PWC1c+5h4DozWwa0A/uAq472umvWrKk2s9Lj\nLKsAqD7O58ZSvNYF8Vub6jo2quvYDMe6JvVnpyE3Id67YWar+zMh1GCL17ogfmtTXcdGdR2bRK4r\n6I5mERGJIwoFERGJSLRQuCPoAvoQr3VB/Namuo6N6jo2CVtXQvUpiIjIkSVaS0FERI4gYULhaDO2\nBsHMJpjZSjPbZGYbzez6oGuKZmbJZva6mT0SdC1dzGykPyXKFjPbbGZnBF0TgJl9wf833GBm95tZ\nRkB13GVmlWa2IWrdKDN70sy2+rd5cVLXD/x/x/Vm9icz6/8l4mJcW9S2L5mZM7OCeKnLzD7v/942\nmtn/DvT7JkQoRM3Y+kFgJvBvZjYz2KoA7/yMLznnZgKnA9fGSV1drif+Tii8Gfi7c246cApxUJ+Z\njQOuAxY452bjnZdzeUDl3A2c12PdjcDTzrlpwNP+48F2N4fX9SQw25/m5m3g64NdlO9uDq8Nfy62\nc4F3Brsg3930qMvM3oc3segpzrlZwA8H+k0TIhSI0xlbnXN7nXNr/fv1eB9wvU0aOOjMbDzwIbzp\nR+KCmY0A3gPcCeCca3XOHQi2qogUINPMUoAsYE8QRTjnnsM7ETTahXTPK3YPcNGgFkXvdTnnnnDO\ntfsPX8abCmfQ9fE7A/gx8FW8y1APuj7q+gxwU9f0QM65yoF+30QJhf7O2BoYMysG5gKvBFtJxE/w\n/iA6gy4kymSgCvi1f1jrV2aWHXRRzrndeN/Y3gH2ArXOuSeCreoQo51ze/375cDoIIvpwyeBx4Iu\noouZXQjsds69EXQtPZwALDazV8zsWTM7daDfIFFCIa6ZWQhYAdzgnKuLg3rOByqdc2uCrqWHFGAe\n8HPn3FygkWAOhRzCP0Z/IV5ojQWyzeyjwVbVO+cNN4yrIYdm9k28Q6n3BV0LgJllAd8Avh10Lb1I\nAUbhHW7+CvCAmdlAvkGihMJRZ2wNipml4gXCfc65h4Kux7cIWGZmO/EOtb3fzH4bbEmA18Irc851\ntaYexAuJoH0A+Kdzrso51wY8BJwZcE3RKromn/RvB/yQw/Eys6uA84ErXPyMj5+KF/Bv+H8D44G1\nZlYUaFWeMuAh53kVryU/oJ3giRIKR52xNQh+wt8JbHbO/Sjoero4577unBvvnCvG+139wzkX+Ddf\n51w5sMvMTvRXnQ30vLxrEN4BTjezLP/f9GzioAM8ysPAlf79K4G/BFhLhJmdh3eIcplz7mDQ9XRx\nzr3pnAs754r9v4EyYJ7//y9ofwbeB2BmJwBpDPDEfQkRCn5nVteMrZuBB5xzG4OtCvC+kX8M75v4\nOn9ZGnRRce7zwH1mth6YA3w/4HrwWy4PAmuBN/H+rgI5I9bM7gdeAk40szIz+xRwE3COmW3Fa9Xc\nFCd13QrkAE/6//dvH+y6jlBb4Pqo6y5gij9M9ffAlQPdwtIZzSIiEpEQLQUREekfhYKIiEQoFERE\nJEKhICIiEQoFERGJUCiIDCIzWxJPs86K9KRQEBGRCIWCSC/M7KNm9qp/UtUv/GtLNJjZj/157J82\ns0J/3zlm9nLUdQHy/PUlZvaUmb1hZmvNbKr/8qGoa0LcN9Bz14i8GwoFkR7MbAawHFjknJsDdABX\nANnAan8e+2eB//Cfci/wNf+6AG9Grb8PuM05dwreXEhdM5XOBW7Au7bHFLwz20XiQkrQBYjEobOB\n+cBr/pf4TLxJ5DqBP/j7/BZ4yL/Gw0jn3LP++nuAP5pZDjDOOfcnAOdcM4D/eq8658r8x+uAYmBV\n7H8skaNTKIgczoB7nHOHXAnMzP5fj/2Od46Ylqj7HejvUOKIDh+JHO5p4BIzC0PkGseT8P5eLvH3\n+QiwyjlXC+w3s8X++o8Bz/pX0iszs4v810j35+kXiWv6hiLSg3Nuk5l9C3jCzJKANuBavIv6LPS3\nVeL1O4A3HfXt/of+DuAT/vqPAb8ws+/5r3HpIP4YIsdFs6SK9JOZNTjnQkHXIRJLOnwkIiIRaimI\niEiEWgoiIhKhUBARkQiFgoiIRCgUREQkQqEgIiIRCgUREYn4/6LDb3qW9olQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fee41e99c88>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(model.history.history['categorical_crossentropy'], label='train');\n",
    "plt.plot(model.history.history['val_categorical_crossentropy'], label='val');\n",
    "plt.legend();\n",
    "plt.xlabel('epoch');\n",
    "plt.ylabel('logloss');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8XXWd//HXJ1uTNmmT7kvSNkDpCl0IBQQVRLCAUBhk\nExAcFecnCCIzvx/OOIrMOKLjuM0giqw6bAUBqxbZZBEB6Q40bWnpljQpTZdsbfb7+f1xbtLbNGlv\nS27OTe77+Xjcx73n3HPv/aRpzvt+z/ec79fcHREREYC0sAsQEZHkoVAQEZEOCgUREemgUBARkQ4K\nBRER6aBQEBGRDgoFERHpoFAQEZEOCgUREemQEXYBh2v48OE+ceLEsMsQEelTli5dusPdRxxquz4X\nChMnTmTJkiVhlyEi0qeY2eZ4ttPhIxER6aBQEBGRDgoFERHp0Of6FLrS0tJCeXk5jY2NYZeSUNnZ\n2RQWFpKZmRl2KSLST/WLUCgvLycvL4+JEydiZmGXkxDuzs6dOykvL6e4uDjsckSkn+oXh48aGxsZ\nNmxYvw0EADNj2LBh/b41JCLh6hehAPTrQGiXCj+jiISrXxw+EhHpS1raIjS0tNHY0kZjc4TG1jYa\nmoPlYH0k5vG+dWdOGcnMovyE1qZQ6AHV1dU8/PDDfOUrXzms15177rk8/PDD5Ocn9pcs0p+0tkXY\n09RGbWMLdY2t1De1Ut/UQmNLhObW4NbUFvO4ta3jcXPbgds0tUZobt+mLUJLq+MEc9cb+1rnXTXU\n21vv1sU2ZuAOza2R/Xb0jS1ttEb8iH72kXkD+nYomNk84KdAOnCPu9/R6fkJwH3ACGAXcJW7lyey\npkSorq7m5z//+QGh0NraSkZG9//EixYtSnRpIkesLeLsqG+iorqBbTWNVNY0sr2uCYDMdCM9zchM\nTyM9zchov6Wndbpvfy6N9HQjMy0t+jrDzNjb3Ep9Yyt1ja3UNrZQ3xQ8rm9spa4p2OkHt30BsLe5\n7bB/low0IysjLbilB/cDMtLIykgPHqenMTArg/yMNDLTjTQzPGa/3R4S+6+j07p9T8Zul5WRRk5m\nOtlZ6WRnpJOTlRa9T2dAZnrwXGawTU5mp3Udr0lnQEZarxxCTlgomFk6cCdwFlAOLDazhe5eGrPZ\nD4Ffu/uDZvYJ4HvA1YmqKVFuvfVW3n//fWbNmkVmZibZ2dkUFBSwZs0a3nvvPS688ELKyspobGzk\npptu4rrrrgP2DdlRX1/POeecw2mnncbrr7/OuHHj+N3vfkdOTk7IP5n0V13t8CtrGqL3jWyraeSD\n2sYDvtFmphuG0RqJcIRfdg9pUFY6edmZ5GZnkJedweCcTMbl55CXnUHugAzysjODx9kZDM4OlgcN\nyCA7M9jhD8hM33/Hn55GWpr64+KVyJbCXGC9u28AMLNHgflAbChMA74effwS8PSH/dDv/H4VpRW1\nH/Zt9jNt7GC+ff70bp+/4447ePfdd1mxYgUvv/wy5513Hu+++27HqaP33XcfQ4cOpaGhgRNPPJGL\nL76YYcOG7fce69at45FHHuFXv/oVl156Kb/97W+56qqrevTnkNRTvbeZ36+sYMuuvYfc4Q/ISGPM\nkGzGDMnhpOKhjMnPZvSQHMYOyWb0kGzGDskhf2Bmx7fVSMRpjTitkUhw3xZ93Oa0RZyWtkj0Plju\nvF1bxBk0oH1HH+zccwdkkK4deKgSGQrjgLKY5XLgpE7brAT+juAQ00VAnpkNc/edCawr4ebOnbvf\ntQQ/+9nPeOqppwAoKytj3bp1B4RCcXExs2bNAuCEE05g06ZNvVav9D/Ve5u597WN3P/XTdQ3tR72\nDj8eaWlGVpqR1X9OYhTC72j+R+B/zOxa4FVgK3DAAUMzuw64DmD8+PEHfcODfaPvLYMGDep4/PLL\nL/PCCy/wxhtvMHDgQE4//fQurzUYMGBAx+P09HQaGhp6pVbpX6r3NnNfNAzqmlo597jR3HDGJKaO\nydMpzRKXRIbCVqAoZrkwuq6Du1cQtBQws1zgYnev7vxG7n43cDdASUlJgo5kHrm8vDzq6uq6fK6m\npoaCggIGDhzImjVrePPNN3u5OkkFNXtbuPe1DfuFwY1nTmLK6MFhlyZ9TCJDYTEwycyKCcLgcuCz\nsRuY2XBgl7tHgG8QnInU5wwbNoxTTz2VGTNmkJOTw6hRozqemzdvHr/4xS+YOnUqkydP5uSTTw6x\nUulvOofBOTOCMJg6RmEgR8bcE/fF28zOBX5CcErqfe7+XTO7HVji7gvN7DMEZxw5weGj69296WDv\nWVJS4p0n2Vm9ejVTp05NyM+QbFLpZ5Xu1ext4d6/buT+1zYqDCQuZrbU3UsOtV1C+xTcfRGwqNO6\nb8U8fgJ4IpE1iPQnCgNJtLA7mkUkDjUNLdz32kbu++tG6hpbmTc9CINpYxUG0rMUCiJJTGEgvU2h\nIJKEahpauP+vG7n3NYWB9C6FgkgSqGtsYWVZDcu37GbZlt0s3rSb+qZWPjV9FDeeOYnpY4eEXaKk\nCIWCSC+LRJwNO+pZtrma5WW7Wba5mve213UMojZpZC6fPn4MV58yQWEgvU6hEILc3Fzq6+vDLkN6\nSU1DCyvKqlm2eTfLy6pZsWU3tY2tAAzOzmD2+ALOOW40c8YXMLMonyE5moNbwqNQEOlBbRFn/fZ6\nlm3ZHT0UVM367cEXADOYPCqP844fy+zx+cwZX8BRwwdpBE9JKgqFHnDrrbdSVFTE9ddfD8Btt91G\nRkYGL730Ert376alpYV///d/Z/78+SFXKj2tpqEl2PlvDgJgRVk19U1BK6BgYCazxxdw4ayxzB5f\nwPGFQ8jLVitAklv/C4VnboVt7/Tse44+Ds65o9unL7vsMr72ta91hMKCBQt49tlnufHGGxk8eDA7\nduzg5JNP5oILLtCgZH2Yu7Nxxx6Wbg46g5du3s267fW4Q5rBlNGDuXD2WOaML2D2+AImDhuo37f0\nOf0vFEIwe/Zstm/fTkVFBVVVVRQUFDB69GhuvvlmXn31VdLS0ti6dSsffPABo0ePDrtciVNDcxsr\ny6uDEIgGwe69LUDQFzBnQgHnHz+WEyYEfQGDBujPSfq+/ve/+CDf6BPpkksu4YknnmDbtm1cdtll\nPPTQQ1RVVbF06VIyMzOZOHFil0NmS/KoqG5g6ebdHS2B0orajolojh4xiLOmjWLO+AJOmFDA0SNy\n1Rcg/VL/C4WQXHbZZXzpS19ix44dvPLKKyxYsICRI0eSmZnJSy+9xObNm8MuUWI0trSxqqI2OCso\n2idQWROEdk5mOjOLhvDljx/FCRMKmF1UQMGgrJArFukdCoUeMn36dOrq6hg3bhxjxozhyiuv5Pzz\nz+e4446jpKSEKVOmhF1iymqLOOu217GyrJqV5TWsLKtm7ba6jlbAuPwcTpw4lBMmBK2AKaPzyEjX\nbGKSmhQKPeidd/Z1cA8fPpw33nijy+10jULiuDvluxtYUVbN2+XVrCyr4d2KGvY2BxP65WVnMLMw\nny9//ChmFuYzsyifUYOzQ65aJHkoFKRP21nfxNvlNawoq2ZleTVvl9ewa08zAFkZaUwfO5hLS4qY\nWTSEmYX5TBym6wJEDkahIH1K2a69PLtqG8vLqllZVk357mAuazM4dmQeZ04ZycyifGYV5XPsqDyy\nMnQYSORw9JtQcPd+f054ImfJS2aNLW08u2obC5aU8df1OwEoLMhhZmE+nztlAscX5jNj3BBydUqo\nyIfWL/6KsrOz2blzJ8OGDeu3weDu7Ny5k+zs1Dj+7e68XV7DgiVlLFxZQV1jK0VDc/j6Wcfyd3PG\nUVgwMOwSRfqlfhEKhYWFlJeXU1VVFXYpCZWdnU1hYWHYZSTUzvomnlq+lceXlLP2gzoGZKRx7nFj\nuKSkkJOLh6k/QCTB+kUoZGZmUlxcHHYZcoRa2yK8uq6KBYvLeXHNB7S0OTOL8vnuRTM4f+ZYBmu8\nIJFe0y9CQfqmDVX1PL60nN8uLWd7XRPDBmVxzSkTuaSkiMmj88IuTyQlKRSkV+1pauWP71Ty+JIy\nFm/aTXqaccbkEVxSUsQZk0fqbCGRkCkUJOHcnaWbd7NgSRl/eLuSvc1tHDViELeeM4W/mz2Okbp4\nTCRpKBQkYdoizrOrtnHXy+/zztYaBmWlc/7xY7n0xELmjC/ot2eKifRlCgXpcU2tbTy9fCu/fGUD\nG3bsoXj4IP7jouOYP2ushpcWSXL6C5UeU9/UyiN/28I9r23gg9omZowbzJ2fncO8GaNJ16mkIn2C\nQkE+tJ31TTz4+iYefGMzNQ0tfOToYfzwkpmcdsxwHSIS6WMUCnLEtlY38KtXN/Do4i00tkT41PRR\n/MPHj2b2+IKwSxORI6RQkMO27oM67nrlfRauqADgwtnj+IePH8UxI3VtgUhfp1CQuC3bspu7Xn6f\n50s/ICcznc+dMpEvfrSYsfk5YZcmIj0koaFgZvOAnwLpwD3ufken58cDDwL50W1udfdFiaxJDo+7\n8+q6Hdz18nre3LCL/IGZ3HTmJK75yESGaopKkX4nYaFgZunAncBZQDmw2MwWuntpzGbfBBa4+11m\nNg1YBExMVE0Sv7aI88y7ldz18vusqqhlzJBs/vXT07j8xCKdVirSjyXyr3susN7dNwCY2aPAfCA2\nFBwYHH08BKhIYD0SB3fn5feq+P4za1izrY6jRgziB585ngtnjdMQFCIpIJGhMA4oi1kuB07qtM1t\nwHNm9lVgEPDJrt7IzK4DrgMYP358jxcqgbfLq/neojW8sWEnE4YN5L+vmM15x43RcNUiKSTs4wBX\nAA+4+3+Z2SnAb8xshrtHYjdy97uBuwFKSkpSc/qxBNqycy//+dxafr+ygmGDsvjOBdO5Yu54tQxE\nUlAiQ2ErUBSzXBhdF+sLwDwAd3/DzLKB4cD2BNYlUTvrm/jvP6/nob9tJiMtjRs/cQxf+thR5Gn+\nApGUlchQWAxMMrNigjC4HPhsp222AGcCD5jZVCAb6N/TpyWBhuY27n1tA794ZQMNLW1cWlLEzZ+c\npNFKRSRxoeDurWZ2A/Aswemm97n7KjO7HVji7guBW4BfmdnNBJ3O13qqzk7fC1rbIjyxtJwfPf8e\n2+uaOHvaKP7vvCkcMzI37NJEJEkktE8hes3Bok7rvhXzuBQ4NZE1SHBG0Qurt/P9P61h/fZ65ozP\n5+dXzqFk4tCwSxORJBN2R7Mk2LItu/neotUs3rSbo0YM4hdXncCnpo/SQHUi0iWFQj+1oaqeH/xp\nLX9atY0ReQP47kUzuKykiIx0nVEkIt1TKPQzVXVN/PTF93jkrTKyM9L4+lnH8oXTinUVsojERXuK\nfuSp5eX8y1Pv0twa4cqTxnPjmZMYnjsg7LJEpA9RKPQD7s6dL63nh8+9x0nFQ7nj4uMpHj4o7LJE\npA9SKPRxrW0R/vV37/LIW2VcOGssP/jMTF2JLCJHTKHQh+1pauWGh5fx0toqrj/jaP7x7Mk6q0hE\nPhSFQh+1va6RLzywhFUVNfzHRcfx2ZM0UKCIfHgKhT5o/fZ6rr3/LXbWN3PPNSV8YsqosEsSkX5C\nodDHvLVxF1/69RIy043Hvnwyxxfmh12SiPQjCoU+5A9vV/D1x1ZSODSHBz8/l6KhA8MuSUT6GYVC\nH+Du3POXjXx30WpKJhTwq8+VUKD5kUUkARQKSa4t4vzbH0p54PVNnHfcGP7r0plkZ6aHXZaI9FMK\nhSTW0NzGTY8u57nSD/jiacX887lTNTWmiCSUQiFJ7axv4ou/XsKKsmq+ff40Pn9qcdgliUgKUCgk\noU079nDt/W9RWdPIXVfOYd6MMWGXJCIpQqGQZJZt2c0XH1yCu/Pwl07mhAkFYZckIilEoZBEnlu1\njRsfXc6owdk88Pm5GtRORHqdQiFJPPj6Jm77/SqOL8zn3mtKNOS1iIRCoRCySMS5409ruPvVDXxy\n6ij++4rZ5GTplFMRCYdCIWQ/eeE97n51A587ZQLfPn866TrlVERCpFAI0Vsbd/E/L63n4jmFfOeC\n6Rr2WkRCp9lYQlLT0MLNj62gaOhAvjNfgSAiyUEthZB863fvsq22kcf/4RRyB+jXICLJQS2FEDy9\nfCu/W1HBTWdOYs54XYcgIslDodDLynbt5V+ffpeSCQV85fSjwy5HRGQ/CoVe1NoW4ebHVgDw48tm\nkZGuf34RSS46mN2Lfv7y+yzZvJufXDZLE+SISFLSV9VesmzLbn764jrmzxrLhbPHhV2OiEiXEhoK\nZjbPzNaa2Xozu7WL539sZiuit/fMrDqR9YSlvqmVmx9bwejB2dw+f0bY5YiIdCuuw0dm9iRwL/CM\nu0fifE06cCdwFlAOLDazhe5e2r6Nu98cs/1XgdmHUXuf8Z2FqyjbtZdHrzuFITmZYZcjItKteFsK\nPwc+C6wzszvMbHIcr5kLrHf3De7eDDwKzD/I9lcAj8RZT5/xx7creXxpOdefcQxzi4eGXY6IyEHF\nFQru/oK7XwnMATYBL5jZ62b2eTPr7qvvOKAsZrk8uu4AZjYBKAb+HG/hfUFFdQPfePJtZhblc+OZ\nk8IuR0TkkOLuUzCzYcC1wBeB5cBPCULi+R6o43LgCXdv6+azrzOzJWa2pKqqqgc+LvHaIs7XF6yg\nNeL89LJZZOr0UxHpA+LaU5nZU8BfgIHA+e5+gbs/5u5fBXK7edlWoChmuTC6riuXc5BDR+5+t7uX\nuHvJiBEj4ik5dL/6ywbe3LCL286fzkRNliMifUS81yn8zN1f6uoJdy/p5jWLgUlmVkwQBpcT9Evs\nx8ymAAXAG3HWkvTeKa/hv55byzkzRnNJSWHY5YjEr6ke6j/Yd6v7YP/lht2QPwFGTo3epkHBREgL\neQ4Qd2htguZ6aKrbd99UD83t9/XBfVPtvscd93X7tm9pgIEFMLgQBo+FIeNgcPttLAwphIHDIa2X\nW//uwS3BnxtvKEwzs+XuXg1gZgXAFe7+8+5e4O6tZnYD8CyQDtzn7qvM7HZgibsvjG56OfCou/uR\n/xjJY29zKzc9tpxhgwbwvb87TqOfSvgibbBnB9Rvg/rtULctupPf3mnddmjZc+Dr0zIgd1Rwyx4M\nFctg1ZP7ns/IhuHHBgExckr0fioMKYKe+v/vHvwM1ZuD2+7NUL0lurwFqsugrSm+98rIgQF5MCAX\nsnKDx7mjYdikYF1GDuzdAbUVUL4YVi+Etub93yM9C/LGBEExJBoWnUOkc3C0NgeB1FgTvT/I46aa\nYLmxdv/nz/svOOGanvk37YbFsy82sxXuPqvTuuXu3uunkJaUlPiSJUt6+2Pj9s9PvcMjb23hoS+c\nxEeOGR52OZJq2lpg++pgx12xHLYug+2lEGk9cNvsIft29u23vPbHI4MdZe4oyCk48Ntp8x6oWht8\n1vZSqFoTPK6NOUKclQsjpsS0KqIti9xRB4aFe9AKad/JH7DT3wIte/d/zcBhkD8+aLkMKQzqHJAX\n3LJyozv96M6/fV1WLqQf5kAOkQjs3Qm15UFQ1GwNfs7aiuh99HFXwZE7KljfWAutDYf+rKxcGDA4\n+N1kD44+Hrxv3dTzobC7gzMHZ2ZLD3Jkp0O8/zrpZmbt3+aj1yBkHVFl/dhzq7bx8N+28OWPHaVA\nkMSLRGDn+iAAtkZDYNvb0NoYPJ89BMbOhlNuCHaaeaNjAmAkZOYc+WdnDYJxc4JbrIbqaFiUBiFR\ntRre+xMs/82+bbLzg3AYdhTs3bUvBJrr9n+v7CHBTn/YMXD0mVAwYV8I5BcFO/rekJYGuSOC29hu\nvgd3Do7aCqgpD1pgGQOiO/bojj57SKedfcy6sA/DEX9L4T+BCcAvo6u+DJS5+y0JrK1LydpS2F7b\nyKd+8ipj83N48isfYUBG+L/cfqmhOtjZjJwSfDNMFe7BzjO2BVC5Mji0AJA5EMbMhLHRHfXY2TD0\nqJ47fPNh7dkRbVWs3hcYuzcGh1jyx3fa4Y8Pbjn5YVfdr/R0S+H/EQTB/4kuPw/cc4S19TuRiHPL\n4ytpaGnjp5fPUiD0pEgEKlfA+hdh/QvBMd72M5eHHwuFc4PmdNHc4FBFWN+09u6CXRsBD3bElnaI\n2yG2aW2Ebe/sC4CKZcE3UYC0TBg9A467ZF8ADJ98+IdFetOg4VD80eAmSS2u/0XRoS3uit6kk/tf\n38Rf1u3g3y+cwTEje6lJ25/VV8H7fw5C4P0XoztDC3Z+H70luN9eGgTEe8/Aiv8NXpeVB4UnBEFR\nFA2Lnm5N7N217/h51Zro4zWwZ3vPfk47SwvC7thzYOysIARGzQgOSYgkQLxjH00CvgdMA7Lb17v7\nUQmqq89YXVnL959ZwyenjuTKk8aHXU7f1NYKW5fAuueDIKgM5pxg4HA45pPB7ehPBN822005N7h3\nh10bgoAoewvK34K//BDah+g60tbE3l2ddvyrg2PlsTv/rDwYMRkmnR0czhp2THCmjkcOcvNDPB/d\nxiw47j76+KCjVKSXxNvevB/4NvBj4Azg82jYbRpb2rjp0eUMzsnk+xcfr9NPD0fN1qAVsP4FeP/l\n4BQ8Sw923J/4ZhAEo2ce+pxsMxh2dHCbeXmwrqk+ONxS9lYQFmsXdd+aGDElOFZ/ODv/EVOD5SGF\nyXPMXqSHxBsKOe7+YvQMpM3AbWa2FPhWAmtLenc8s4b3Pqjngc+fyLBcNecPqrUJtrwRhMD6F4PD\nPwB5Y2HaBTDpLCj+eM90Lg7IheKPBTfY15poD4nOrYl22vmLxB0KTWaWRjBK6g0EVyindJv2pTXb\neeD1TVz7kYmcPnlk2OX0DPfgsMnujcFOtKY8OMe6rTnYqbe1RJdb9q0/6OOY1zTWBJ2n6Vkw/hQ4\n69+C1sDIqYnf4ca2JmZdEaxrb03seA/yJ2rnLxIVbyjcRDDu0Y3AvxEcQkrsZXVJbEd9E//0xEom\nj8rj1nOmhF3O4YlEgqtYd20IzpbZtWFfCOzaFBzG6SwtI9iZp2dG7zvfMvfdZw7Z9zhjwL7HWbkw\n8aMw8bTkOEbeuTUhIkAcoRC9UO0yd/9HoJ6gPyGlLVhSxo76Zn7zhZPIzkzC00/bWqGmLGaHv3H/\nAGi/uAmCHX7+eCgoDo6zDz0KhhYH90MKg0v+e3uMFxEJzSFDwd3bzOy03iimr1hVUUvR0Bymjhkc\ndinB5fOVK/ady77tnaDjNHZYg4zsYKc/9Cg45sxgp9++PKQouc9vF5FeFe/eYLmZLQQeBzpGzHL3\nJ7t/Sf+1uqKWaWEEQktDsNNvD4Cty2Dnun3P508IrmqdduH+3/hzR+vbvojEJd5QyAZ2Ap+IWedA\nyoXCnqZWNu7cw/xZXU4i13PaWoIzdNoDoGJ5cLpkewsgd3RwIdPxlwZDG4ydDYOGJbYmEen34r2i\nOeX7Edqt2VaHO0wb24MthQMGNoseBuoY2Cw/CIBTz943ts3gsT33+SIiUfFe0Xw/QctgP+7+9z1e\nUZIrrQwGIPvQoeAenLe/7New5o8xA5sNCoYzOPGLwbf/cXOC4/86VVJEekG8h4/+EPM4G7gIqOj5\ncpJfaUUtQ3IyGTsk+9Abd2XPDljxcBAGO9cFF0xNvzA4d3/cnGBYhiQYPldEUlO8h49+G7tsZo8A\nryWkoiRXWhl0Mh/WkBaRCGx4aV+rINICRSfBaT8PAiFLcziLSHI40nMRJwH95DLe+LW2RVhTWctV\nJ0+I7wU1W2HFQ7DsN1CzBXKGwtzrYM7VwZW8IiJJJt4+hTr271PYRjDHQkrZtHMPTa2Rg5+O2tYK\n656FpQ/C+ueD8XWKPw5n3QZTPq0hj0UkqcV7+EiTBBBctAbddDLv2hC0CFY8HAwjkTsaTrsZZl8d\nXC8gItIHxNtSuAj4s7vXRJfzgdPd/elEFpdsSitryUpP4+gR0bF7Wptg9e9h2YOw8dVgQpRJZ8Oc\na4J7XSksIn1MvHutb7v7U+0L7l5tZt8GUisUKmqZNCqXrMad8NqPYOUj0LA7GDvojG/C7Ct1/YCI\n9GnxhkJXYySk1Ndgd6e0opaLjna4fx7s3gxTzoMTroHi0zWMhIj0C/Hu2JeY2Y+AO6PL1wNLE1NS\ncqqqa2Lg3jJu2vID8D1w7R9g/MlhlyUi0qPi/Xr7VaAZeAx4FGgkCIaUsXHtch7Pup3sSANcs1CB\nICL9UrxnH+0Bbk1wLclr2zsc99wV7CFC01W/J3PszLArEhFJiLhaCmb2fPSMo/blAjN7NnFlJZHy\npfDAeTR5BjflfJfc8QoEEem/4j18NNzdq9sX3H03qXBF86a/wq/nQ04BX8n6LnnjdBWyiPRv8YZC\nxMzGty+Y2US6GDW1X1n/IvzvxTB4DHuv/D1v7s5l2pghYVclIpJQ8Z599C/Aa2b2CmDAR4HrElZV\n2Nb8ER6/FoZPhqufYvXODNxh6hhd2C0i/VtcLQV3/xNQAqwFHgFuARoO9Tozm2dma81svZl12VFt\nZpeaWamZrTKzhw+j9sR45wl47GoYfVxwllHuCFb31BwKIiJJLt5hLr4I3AQUAiuAk4E32H96zs6v\nSSe4ruEsoBxYbGYL3b00ZptJwDeAU919t5mF20+x7Dew8Ksw4SNwxaOQHYRAaWUtg7MzGJefE2p5\nIiKJFm+fwk3AicBmdz8DmA1UH/wlzAXWu/sGd28muL5hfqdtvgTcGe24xt23x115T/vb3bDwBjj6\nDLjyiY5AgGB4i2ljD3MOBRGRPijeUGh090YAMxvg7muAyYd4zTigLGa5PLou1rHAsWb2VzN708zm\nxVlPz3rtJ/DMP8Hk84IWQtbAjqfaIs6abbXqZBaRlBBvR3N59DqFp4HnzWw3sLmHPn8ScDrBoalX\nzey42NNfAczsOqId2+PHj+/8HkfOHV76D3j1BzDjYrjol5Ceud8mG3fsobElov4EEUkJ8V7RfFH0\n4W1m9hIwBPjTIV62FSiKWS6MrotVDvzN3VuAjWb2HkFILO70+XcDdwOUlJT0zKmw7vDcN+GN/4HZ\nV8H5P+tybuTS9k7mg02sIyLSTxz20J7u/oq7L4z2ExzMYmCSmRWbWRZwObCw0zZPE7QSMLPhBIeT\nNhxuTYdWac4VAAAMhUlEQVQtEoE/fj0IhLnXwfn/3WUgQNCfkJluHDMyN+FliYiELWHjPbt7K3AD\n8CywGljg7qvM7HYzuyC62bPATjMrBV4C/snddyaqJiCYLvN3X4El98GpX4NzfnDQYa9LK2uZNDKP\nrAwNjS0i/V9C50Rw90XAok7rvhXz2IGvR2+J19oMT34JSp8OJsX52D/CIc4oKq2o5fTJI3qlPBGR\nsKXORDktjbDgc7DuWTj7u/CRGw75ku11jeyob1J/goikjNQJhVd/AOueg0//GEr+Pq6XlFboSmYR\nSS2pEwofvQWKToJjPxX3S9rPPJqqloKIpIjU6T3NGnRYgQBBS6GwIIchOZmH3lhEpB9InVA4AqWV\ntepPEJGUolDoxt7mVjbu2KP+BBFJKQqFbqzZVoe7rmQWkdSiUOiGzjwSkVSkUOiG5lAQkVSkUOiG\n5lAQkVSkUOiC5lAQkVSlUOiC5lAQkVSlUOiC5lAQkVSlUOjC6krNoSAiqUmh0IXSCs2hICKpSXu9\nLpRW1qo/QURSkkKhk+11jVTVaQ4FEUlNCoVOVlfWARouW0RSk0Khk47hLRQKIpKCFAqdlFbWMi4/\nhyEDNYeCiKQehUInpRU16mQWkZSlUIixt7mVDTv26NCRiKQshUKMte1zKKilICIpSqEQQ8NbiEiq\nUyjEKK2oJS87g8ICzaEgIqlJoRCjtLKWaWM0h4KIpC6FQlRbxFlTWaf+BBFJaQqFqE0799DQ0qb+\nBBFJaQqFqI4rmdVSEJEUplCIKo3OoTBpZF7YpYiIhCahoWBm88xsrZmtN7Nbu3j+WjOrMrMV0dsX\nE1nPwZRW1HKM5lAQkRSXkag3NrN04E7gLKAcWGxmC929tNOmj7n7DYmqI16llbV8bNKIsMsQEQlV\nIr8WzwXWu/sGd28GHgXmJ/DzjljHHArqTxCRFJfIUBgHlMUsl0fXdXaxmb1tZk+YWVFXb2Rm15nZ\nEjNbUlVV1eOFts+hoDOPRCTVhX0A/ffARHc/HngeeLCrjdz9bncvcfeSESN6/hCP5lAQEQkkMhS2\nArHf/Auj6zq4+053b4ou3gOckMB6urVacyiIiACJDYXFwCQzKzazLOByYGHsBmY2JmbxAmB1Auvp\nVmllrfoTRERI4NlH7t5qZjcAzwLpwH3uvsrMbgeWuPtC4EYzuwBoBXYB1yaqnu40NLexoaqe844b\nc+iNRUT6uYSFAoC7LwIWdVr3rZjH3wC+kcgaDmXtB3VENIeCiAgQfkdz6NTJLCKyj0KhskZzKIiI\nRCkUKjSHgohIu5QOhbaIs2ab5lAQEWmX0qGweece9jZrDgURkXYpHQqllUEn81SFgogIkOqhUFFL\nRpoxaVRu2KWIiCSF1A6FylqOGZnLgIz0sEsREUkKqR0KFRreQkQkVsqGQlVdE9vrmtTJLCISI2VD\nYXW0k1ktBRGRfVI2FNrPPFJLQURkn9QNhYpgDoX8gVlhlyIikjRSNxQqa3V9gohIJykZCu1zKKg/\nQURkfykZCh1zKKilICKyn5QMhfY5FKarpSAisp/UDIXKGvIGaA4FEZHOUjIUVlfWMXWs5lAQEeks\n5UIhEnFWV9aqP0FEpAspFwqbd+0N5lBQf4KIyAFSLhTaO5nVUhAROVDqhUJljeZQEBHpRuqFQoXm\nUBAR6U7qhUKl5lAQEelOSoXCjvomPqjVHAoiIt1JqVDQHAoiIgeXUqGgM49ERA4utUKhUnMoiIgc\nTEJDwczmmdlaM1tvZrceZLuLzczNrCSR9ZRWaA4FEZGDSVgomFk6cCdwDjANuMLMpnWxXR5wE/C3\nRNUC0NjSxvuaQ0FE5KAS2VKYC6x39w3u3gw8CszvYrt/A74PNCawFtZu0xwKIiKHkshQGAeUxSyX\nR9d1MLM5QJG7/zGBdQBBfwIoFEREDia0jmYzSwN+BNwSx7bXmdkSM1tSVVV1RJ83bFAWZ00bpTkU\nREQOIiOB770VKIpZLoyua5cHzABejs5rMBpYaGYXuPuS2Ddy97uBuwFKSkr8SIo5e/pozp4++khe\nKiKSMhLZUlgMTDKzYjPLAi4HFrY/6e417j7c3Se6+0TgTeCAQBARkd6TsFBw91bgBuBZYDWwwN1X\nmdntZnZBoj5XRESOXCIPH+Hui4BFndZ9q5ttT09kLSIicmgpdUWziIgcnEJBREQ6KBRERKSDQkFE\nRDooFEREpIO5H9G1YKExsypg8xG+fDiwowfL6Smq6/CorsOXrLWprsPzYeqa4O4jDrVRnwuFD8PM\nlrh7QofnPhKq6/CorsOXrLWprsPTG3Xp8JGIiHRQKIiISIdUC4W7wy6gG6rr8Kiuw5estamuw5Pw\nulKqT0FERA4u1VoKIiJyECkTCmY2z8zWmtl6M7s17HoAzKzIzF4ys1IzW2VmN4VdUywzSzez5Wb2\nh7BraWdm+Wb2hJmtMbPVZnZK2DUBmNnN0d/hu2b2iJllh1THfWa23czejVk31MyeN7N10fuCJKnr\nP6O/x7fN7Ckzy0+GumKeu8XM3MyGJ0tdZvbV6L/ZKjP7QSI+OyVCwczSgTuBc4BpwBVmNi3cqgBo\nBW5x92nAycD1SVJXu5sIhj1PJj8F/uTuU4CZJEF9ZjYOuBEocfcZQDrB/CFheACY12ndrcCL7j4J\neDG63Nse4MC6ngdmuPvxwHvAN3q7KLquCzMrAs4GtvR2QVEP0KkuMzuDYJ77me4+HfhhIj44JUIB\nmAusd/cN7t4MPErwjxsqd69092XRx3UEO7hxB39V7zCzQuA84J6wa2lnZkOAjwH3Arh7s7tXh1tV\nhwwgx8wygIFARRhFuPurwK5Oq+cDD0YfPwhc2KtF0XVd7v5cdN4VCCbZKkyGuqJ+DPxfIJRO127q\n+j/AHe7eFN1meyI+O1VCYRxQFrNcTpLsfNuZ2URgNvC3cCvp8BOCP4pI2IXEKAaqgPujh7XuMbNB\nYRfl7lsJvrVtASqBGnd/Ltyq9jPK3Sujj7cBo8Ispht/DzwTdhEAZjYf2OruK8OupZNjgY+a2d/M\n7BUzOzERH5IqoZDUzCwX+C3wNXevTYJ6Pg1sd/elYdfSSQYwB7jL3WcDewjnUMh+osfo5xOE1lhg\nkJldFW5VXfPgdMOkOuXQzP6F4FDqQ0lQy0Dgn4EuJwMLWQYwlOBQ8z8BCyw6wX1PSpVQ2AoUxSwX\nRteFzswyCQLhIXd/Mux6ok4FLjCzTQSH2j5hZv8bbklA0MIrd/f21tQTBCERtk8CG929yt1bgCeB\nj4RcU6wPzGwMQPQ+IYcdjoSZXQt8GrjSk+P8+KMJwn1l9P9/IbDMzEaHWlWgHHjSA28RtOJ7vBM8\nVUJhMTDJzIrNLIugE3BhyDURTfl7gdXu/qOw62nn7t9w90J3n0jwb/Vndw/9m6+7bwPKzGxydNWZ\nQGmIJbXbApxsZgOjv9MzSYIO8BgLgWuij68BfhdiLR3MbB7BIcoL3H1v2PUAuPs77j7S3SdG//+X\nA3Oi//fC9jRwBoCZHQtkkYBB+1IiFKKdWTcAzxL8sS5w91XhVgUE38ivJvgmviJ6OzfsopLcV4GH\nzOxtYBbwHyHXQ7Tl8gSwDHiH4O8qlCtizewR4A1gspmVm9kXgDuAs8xsHUGr5o4kqet/gDzg+ej/\n/V8kSV2h66au+4CjoqepPgpck4jWla5oFhGRDinRUhARkfgoFEREpINCQUREOigURESkg0JBREQ6\nKBREepGZnZ5Mo86KdKZQEBGRDgoFkS6Y2VVm9lb0oqpfRueWqDezH0fHsn/RzEZEt51lZm/GzAtQ\nEF1/jJm9YGYrzWyZmR0dffvcmDkhHkrE+DUiR0qhINKJmU0FLgNOdfdZQBtwJTAIWBIdy/4V4NvR\nl/wa+H/ReQHeiVn/EHCnu88kGAupfaTS2cDXCOb2OIrgynaRpJARdgEiSehM4ARgcfRLfA7BIHIR\n4LHoNv8LPBmd4yHf3V+Jrn8QeNzM8oBx7v4UgLs3AkTf7y13L48urwAmAq8l/scSOTSFgsiBDHjQ\n3febCczM/rXTdkc6RkxTzOM29HcoSUSHj0QO9CLwGTMbCR1zHE8g+Hv5THSbzwKvuXsNsNvMPhpd\nfzXwSnQmvXIzuzD6HgOiY/WLJDV9QxHpxN1LzeybwHNmlga0ANcTTOozN/rcdoJ+BwiGo/5FdKe/\nAfh8dP3VwC/N7Pboe1zSiz+GyBHRKKkicTKzenfPDbsOkUTS4SMREemgloKIiHRQS0FERDooFERE\npINCQUREOigURESkg0JBREQ6KBRERKTD/wf6ODlAu7M3jQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fee3f1c2b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(model.history.history['accuracy'], label='train');\n",
    "plt.plot(model.history.history['val_accuracy'], label='val');\n",
    "plt.legend();\n",
    "plt.xlabel('epoch');\n",
    "plt.ylabel('accuracy');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VdW5//HPk5lAgBACCAESEZlEQYPa2lavU7HWoZND\n1drevuqvgx3scK/91Wutt731jr1tr221vfy0rUOpWuV6adU61ooKqIhMgiRAwpCBIZCQ8Ty/P/ZO\nOISEbCQn55Dzfb9e53X2XnvIcwLZz1l7rb2WuTsiIiJ9yUh2ACIicmxQwhARkUiUMEREJBIlDBER\niUQJQ0REIlHCEBGRSJQwREQkEiUMERGJRAlDREQiyUp2AP1p9OjRXlpamuwwRESOKcuXL69z9+K+\n9htUCaO0tJRly5YlOwwRkWOKmW2Ksp9uSYmISCRKGCIiEokShoiIRDKo2jB60tbWRlVVFc3NzckO\nJeHy8vIoKSkhOzs72aGIyCA06BNGVVUVBQUFlJaWYmbJDidh3J36+nqqqqooKytLdjgiMggN+ltS\nzc3NFBUVDepkAWBmFBUVpUVNSkSSY9AnDGDQJ4tO6fI5RSQ5Bv0tKREZnNydlvYYLW0xmts7aG7r\noLktFr530NweLLe0x3B3MjOMrAwjMyODrAwjo2vdeljPIDODHveNudPW4bS1x2iPxWhtd9o6YrR1\nxGjtiNHeEb8et1+43NYRoz3mtIZxHSTuS5/1XBxus0O2HV88lA+fPL6ff8sHU8IYALt37+b+++/n\ni1/84hEd96EPfYj777+fkSNHJigySUdtHTFWbW1gacVOXtu8iz3723CHmDtOcCGOX48FhcQcnM5t\nB/ZzvGvdzMgwyDA7aDnD6LZuWLicmXFgOf7Yto5Y14W/pS0uIYTJIUgESf5lHqX4RHC0n+WCmWOV\nMAaD3bt387Of/eyQhNHe3k5WVu//BIsXL050aJIG9rW089qmXSyr3MnSyl28vmUXzW0xACYX5TOm\nIBez4DtrhkFGRkbXBRwOXOiD7cHFvXN/s7iy8FtvzD18BUkk5nRbd2KxoCz45t3z9uxMIzc7kxFD\nsskryCUvO5O87IzwPZO8rAxyO5ezM8jLyuy2Twa5WcG7mdERc9o7PPy5TkcsqA10uAfbYk5HR/B+\nyD6xYL/2Dicjw8jJNLIzM8jKzOha7nzlZAU1lM7l+G3ZB+1rkW4jd6+FxK8OdL5UwhgAN998M++8\n8w5z5swhOzubvLw8CgsLWbt2LW+//TaXX345W7Zsobm5ma9+9avccMMNwIGhTvbt28dFF13E+973\nPl566SUmTJjAY489xpAhQ5L8ySQV1TQ0s7RyF0srd7Js005Wb20g5kEymDV+BFefPol5paMoLy1k\nTEFessOVPnRPKslsqkyrhPG9/1nF6q0N/XrOmeOH891LZh12nzvuuIO33nqLN954g+eee46LL76Y\nt956q6v764IFCxg1ahT79+9n3rx5fOxjH6OoqOigc6xfv54HHniAX/7yl1xxxRU8/PDDXHvttf36\nWeTY4+68U9vYVXtYWrmTzTubABiSncncSSO58dypzCstZO6kQoblptWfvPQz/e9JgtNPP/2gZyV+\n8pOf8Ic//AGALVu2sH79+kMSRllZGXPmzAHgtNNOo7KycsDileSLxZyG5jbq9rVSt6+FlVV7whrE\nLnY2tgJQNDSH8tJCPvWeyZSXjmLW+OFkZ6ZFR0gZIGmVMPqqCQyUoUOHdi0/99xz/PnPf2bJkiXk\n5+dzzjnn9PgsRW5ubtdyZmYm+/fvH5BYJTHcnX0t7dTva6W+sZX6fS3UN7ayszFICPX74pYbW9nV\n2Ep77OA71qVF+Zw7fQzzSguZVzqKstFD1bVaEiqtEkayFBQUsHfv3h637dmzh8LCQvLz81m7di0v\nv/zyAEcn/am9I0btvhZ2NLSwfU8zOxo6Xy3hxb+lK0m0tsd6PMew3CyKhuVQNDSHksJ85kwcSdGw\nHEYNzWX0sBxGDc1h2rgCtT/IgFPCGABFRUWcddZZnHTSSQwZMoSxY8d2bZs/fz6/+MUvmDFjBtOm\nTePMM89MYqTSG3enYX872xua2d6ZBPY0s2NvM9v3tHQlhrp9LXSrCJCZYYwpyKW4IJfiYblMGzuc\n0cNywqSQy6hhOYwemhsmhRzysjOT8yFF+mCHPDhyDCsvL/fuEyitWbOGGTNmJCmigZdun7e/tXXE\nWLOtgdc372ZF1W6qdu3vSgadXVHjjczPZtzwPMYOz2Ps8FzGDc9jzPA8xg3PY9yIPMYMz2X00Fwy\nMnSrSFKXmS139/K+9lMNQ9Latj37eX3zbl7fvIvXN+9mZfUeWsJbRcUFuZQVDeXkkpGMG54bJoUg\nEYwtCJKBagOSTpQwJG3sb+1gZfUeXt+8ize27Ob1zbvZ3hB0MMjJymD2hBFce+Zk5k4aydxJhYwf\nkadGZJE4CU8YZjYf+DGQCfzK3e/otn0ysAAoBnYC17p7VbitA1gZ7rrZ3S9NdLwyOLg7lfVNXTWH\n17fsYs22vXSEDQyTi/I54/hRzJ0YJIcZxw0nJ0tdUEUOJ6EJw8wygTuBC4AqYKmZLXL31XG7/Rvw\na3e/18zOBX4IXBdu2+/ucxIZowwONXubeat6DyurGnhjyy5e37Kb3U1tQNDr6JSJI/jC2VOYO2lk\n2Osot48zikh3ia5hnA5scPeNAGb2IHAZEJ8wZgJfD5efBR5NcExyjItPDiur9/BW9Z6uW0tmcOKY\nAubPGtd1a2lK8TAy1egsctQSnTAmAFvi1quAM7rtswL4KMFtq48ABWZW5O71QJ6ZLQPagTvcXckk\nzfSVHKYUD+M9U4o4acIIZk8YwczxwzX8hUiCpMJf1jeB/zKzTwMvANVAR7htsrtXm9nxwDNmttLd\n34k/2MxuAG4AmDRp0sBFnUDDhg1j3759yQ5jwCk5iKS2RP+1VQMT49ZLwrIu7r6VoIaBmQ0DPubu\nu8Nt1eH7RjN7DpgLvNPt+LuBuyF4DiMhn0ISoqahmWfW1vDculre2LL7oORw/OihSg4iKSbRf4FL\ngalmVkaQKK4CPhm/g5mNBna6ewz4NkGPKcysEGhy95Zwn7OAf0lwvAlx8803M3HiRL70pS8BcNtt\nt5GVlcWzzz7Lrl27aGtr4/vf/z6XXXZZkiNNLHfnreoGnl67g6fX1LCyeg8AE0YO4czjRzG7ZKSS\ng0gKS+hfpbu3m9mNwBME3WoXuPsqM7sdWObui4BzgB+amRPckvpSePgM4C4zixHMPX5Ht95VR+6P\nN8P2lX3vdyTGzYaL7jjsLldeeSVf+9rXuhLGwoULeeKJJ/jKV77C8OHDqaur48wzz+TSSy8ddP3+\n97d28OKGOp5Zu4Nn1tawo6EFMzh1UiHf+uA0zp8xlhPHDht0n1tkMEr41zh3Xwws7lZ2a9zyQ8BD\nPRz3EjA70fENhLlz51JTU8PWrVupra2lsLCQcePGcdNNN/HCCy+QkZFBdXU1O3bsYNy4cckO96ht\n3b2fZ9bW8PSaHbz0Tj0t7TGG5WZx9onFnDt9DOdMK1a3VpFjUHrV+/uoCSTSJz7xCR566CG2b9/O\nlVdeyX333UdtbS3Lly8nOzub0tLSHoc1PxbEYs6Kqt1hkqhh9bZgkqpJo/L55BmTOH/GWOaVjtKD\ncSLHuPRKGEl05ZVX8rnPfY66ujqef/55Fi5cyJgxY8jOzubZZ59l06ZNyQ7xiDS2tPOX9XU8vWYH\nz66roW5fKxkG5aWj+PZF0zlvxlimFGt+BpHBRAljgMyaNYu9e/cyYcIEjjvuOK655houueQSZs+e\nTXl5OdOnT092iH1qbuvgmbU1LHpjK8+sq6G1PcbwvCzOmTaG82aM4ewTixmZn5PsMEUkQZQwBtDK\nlQca3EePHs2SJUt63C+VnsFo64jx4vo6Fq3YypOrttPY2kFxQS6fPH0S808ax2mTCzUNqEiaUMKQ\nQ8RizquVO1m0Yit/XLmNXU1tjBiSzSWnjOeSU8Zz5vFFGmpDJA0pYQgQPCPxZtUeFq3YyuNvbmVH\nQwtDsjO5YOZYLj1lPB84sViN1iJpLi0ShrunRePru5k9cf2OvSxasZX/WbGVyvomcjIzOHtaMZee\nMp7zZowhPyct/ouISASD/mqQl5dHfX09RUVFgzppuDv19fXk5eX1ue+WnU1dSWLt9r1kGLx3ymi+\neM4JfHDWOEbkZw9AxCJyrBn0CaOkpISqqipqa2uTHUrC5eXlUVJS0uO2lvYOFi7dwiOvV/P65t0A\nnDa5kO9dOouLZo9jTEHfiUZE0tugTxjZ2dmUlZUlO4ykeuHtWm5btIqNdY3MOG44fz9/Oh8++Tgm\njspPdmgicgwZ9AkjnW3dvZ/v/+9qFq/cTmlRPvd8Zh7nTBuT7LBE5BilhDEItbbH+O8XK/jJ0+tx\nnG9eeCKf+8Dx5GZlJjs0ETmGKWEMMn/dUMetj73FO7WNXDhzLP/w4Zm69SQi/UIJY5DYvqeZ7//v\nah5/cxuTRuWz4NPlnDt9bLLDEpFBRAnjGNfWEeOev1byn39+m7aY87Xzp/L5s6eQl63bTyLSv5Qw\njmFL3qnn1sfeYn3NPs6dPobbLpnFpCLdfhKRxFDCOAbVNDTzT4vX8OgbW5kwcgi//FQ5F8zU7ScR\nSSwljGNIe0eMe5ds4kdPvU1re4yvnHsCXzjnBIbk6PaTiCSeEsYx4tWKndz62Fus3b6XD5xYzPcu\nnUXZ6KHJDktE0kjChx81s/lmts7MNpjZzT1sn2xmT5vZm2b2nJmVxG273szWh6/rEx1rKqrd28LX\nF77BFXctoWF/G7+49jTu/cw8JQsRGXAJrWGYWSZwJ3ABUAUsNbNF7r46brd/A37t7vea2bnAD4Hr\nzGwU8F2gHHBgeXjsrkTGnEo21OzlYz9fQlNrO188Zwo3nnuCRo8VkaRJ9NXndGCDu28EMLMHgcuA\n+IQxE/h6uPws8Gi4/EHgKXffGR77FDAfeCDBMaeExpZ2Pv/b18jONP741fdzwpiCZIckImku0bek\nJgBb4tarwrJ4K4CPhssfAQrMrCjisYOSu3PzIyvZWLuPn1w9V8lCRFJCKkyh9k3gbDN7HTgbqAY6\noh5sZjeY2TIzWzZYhjC/96VK/mfFVr5x4TTeO2V0ssMREQESnzCqgYlx6yVhWRd33+ruH3X3ucB3\nwrLdUY4N973b3cvdvby4uLi/4x9wyzft4geL13D+jDF84ewpyQ5HRKRLohPGUmCqmZWZWQ5wFbAo\nfgczG21mnXF8G1gQLj8BXGhmhWZWCFwYlg1a9ftauPH+1xg3Io9//8QcMjIG7wyBInLsSWjCcPd2\n4EaCC/0aYKG7rzKz283s0nC3c4B1ZvY2MBb4QXjsTuAfCZLOUuD2zgbwwagj5nz1wTeob2zl59ec\npmlSRSTlmLsnO4Z+U15e7suWLUt2GO/Kvz+5jp8+s4F//thsrpw3KdnhiEgaMbPl7l7e136p0Oid\n9p5Zu4OfPrOBK8pLlCxEJGUpYSTZlp1N3PS7Fcw8bji3X3ZSssMREemVEkYSNbd18MX7XiPmzs+v\nPVVzWIhIStM4E0l0++OrWVm9h7uvO43JRRobSkRSm2oYSfLw8iruf2Uznz97ChfOGpfscERE+qSE\nkQRrtjXwnUdXcubxo/jmhScmOxwRkUh0S2qANTS38YXfLmd4XjY/vfpUsjKVs2WQcAc7Bh82jXVA\ny15oaQjf90JzQ7jeELcebrMMyBsOuQWQG753rY84eD1n2LH5O+mFEsYAcne+9fsVbNm1nwdvOJPi\ngtxkhyTpqmknbHkVqpbC/p3Q0Qod7eF7K3S0HXiPtXUr61zuVh5rg4LxcNwpcNzJMO7k4H3ExIG/\naLY1Q/16qFkDtWthX00PF//wvXVf3+ezjAMJwmPBOVr3RgjEekgqcevZQyE7D7LCV/YQyMqFrCEH\nl2flhevh9uwhB8ozB+4yroQxgH75l408sWoHt1w8g3mlo5IdjqQLd9hVAZtfgc1LYMsrwUUUICML\nhhRCZk6wnJkTvrIPvGcPgbwRh98nMzvYtmsTbH8T1j8RXFghOH9n8hgXJpOiEyCjH3oFdrRB/YYg\nMdSsgdrwfefGAz8/IwuGFh98oR4xIVzv7WI+4uD1nKGHJr1YLEga3WsgLXt6TkwtDUFZU33w79Hc\nAG37oX0/xNrf/e8gIytIJNPmw8d+9e7PE4ESxgB5ZWM9//yndVx00jg++76yZIcjg1lHW3DR3vzy\ngVdjTbAtdwRMPB1mfxwmngkTToOc/P6PobUJdqyC7Stg25tBPK/cFdREALLzYeysoDbSmUzGzAy+\nPfck1gE7K6BmdZDsalZDzdqgFtF5sbUMGHU8FE+HWR+BMTOgeEaQnLJy+v8zZmQEiSVvBIw4ynN1\ntEN7c/Bq2w/tLUEiaWvuVt653Bxsb285UD468e2hGhpkANQ0NHPxT19kWG4Wi248i4I8jROVdmId\nwQUtEbdmmvfAlqUHag/Vy6GtKdg2chJMeg9MPAMmnRlcQDOS1G7W0Qa164Lk0ZlEtr154NZORlZw\nse9MIG1NQVKoWQN1b0NHy4FzjZwcJJgx04PPNGYGjJ4a1IbkiEUdGkQ1jARr74hx4wOvs7e5jd98\n9nQli8GmtRH2bod9O4LX3h1xy9uDe+f7tkNjXbB/ztDglZ0fNIjm5MeVhe854bbszm1x+3Xe865d\nF9QctrwSfJPHwTJh3Gw49VNBcph4Jgw/Lqm/noNkZsO4k4LXnE8GZbFYcHsmPolseApW3B9sH14S\nJIXjzz6QIEZPg9xhyfscaUwJI8H+9Yl1vFqxkx9deQrTxw1PdjjSnXvYaNsS3gZoCZdbg2p+U11w\n0e8tKfTUYJqRBcPGBq8RJVByGgwdE2xrbYS2xuC989XcEJy/dV9Y1hTcbuhLTgFMnAczLoVJZ8CE\n8mPvQpqRAUVTgtesjwRl7sHvPDsvuN0jKUMJI4GeWLWdu17YyDVnTOIjc0sGPoBYDHashI3PQ8UL\nwbfRjrbgfm5WHmTmBsud71l5QSNmVu6B9x7LwmX3oGdMrCO4jxxrD3vVdK63he8dYXn7oa+O9oPv\nQZsF7xC3bN22WQ/LcevxCaAjvPC3tx5IBN2TQ1Q5BTBsDBSMC+69DxsLBWNh2LgD5cPGBY28R3vb\nJ9YR3JJp7ZZc2sKEUlgatAH0R8NxqjELfq+ScpQwEqSyrpFvLlzBySUjuPWSmQPzQ92DHiMbnwsS\nROVfYP+uYNvoE+GkjwXfQNvjL6bx36pbgotr8+6DL7DtzQf2PdwFNiOut0z8KzN+PTu4yGVkHdg3\nMzu4SLiHr1j4csAPLHvs4HX84GM61zt773QmvLwRYZLL7SHxxSfNHsryi8KkMDa4JTRQMjLDXjqa\nz11ShxJGAuxv7eDzv11OZqbxs2tOJTcrgd8C91QFyaGzFrF3a1A+vASmfQjKzoay98Pw8f3z8zpv\n4bQ3H7jwZ2QnryFVRAaMEkY/c3f+4bG3WLdjLws+PY+Swn7usthYD5UvHEgSO98JyvOLoOwD4evs\noHthInrkmIW3rxLQTVFEUlqkhGFms919ZaKDGQzeqW3koeVVfOGcKfzNtDFHf8KWvbBpCVQ8HySI\nHeE/Q04BlJ4F8z4bJIgxM/UtX0QSKmoN42dmlgvcA9zn7nui/gAzmw/8GMgEfuXud3TbPgm4FxgZ\n7nOzuy82s1KCecDXhbu+7O6fj/pzk2VDTdBr5qKTIoxA29EGDVuhoRr2VENDVfheHdxqaqgOngqF\n4H76pDPg3Fug7BwYPye4Vy8iMkAiJQx3f7+ZTQX+FlhuZq8C/8/dnzrccWaWCdwJXABUAUvNbJG7\nr47b7RZgobv/3MxmAouB0nDbO+4+54g+UZJtqm8EYHJhXnjx39pzIthTHXTLpNuDk7kjgmELhk+A\nCaeG3TLnBQ9e6aEkEUmiyG0Y7r7ezG4BlgE/AeaamQH/190f6eWw04EN7r4RwMweBC4D4hOGA50P\nKIwAth7ZR0gtLVVv8HTerYz4t23gHQdvzB56IBlMnRE0THeujygJGqbVK0ZEUlTUNoyTgc8AFwNP\nAZe4+2tmNh5YAvSWMCYAW+LWq4Azuu1zG/CkmX0ZGAqcH7etzMxeBxqAW9z9L1HiTZqtr/O3G75C\nc0YenPW1uEQwIUgMeSMH1VDHIpJeotYwfgr8iqA20fUIqrtvDWsdR+Nq4B53/3czew/wGzM7CdgG\nTHL3ejM7DXjUzGa5e0P8wWZ2A3ADwKRJk44ylKNQtQx+81EaPJ8FU37CLeddlLxYREQSIGq3mouB\n+zuThZllmFk+gLv/5jDHVQMT49ZLwrJ4nwUWhudaAuQBo929xd3rw/LlwDvAIcMxuvvd7l7u7uXF\nxcURP04/2/wy/PpyYvmj+HjzLYw47oTkxCEikkBRE8afgfgW1/ywrC9LgalmVmZmOcBVwKJu+2wG\nzgMwsxkECaPWzIrDRnPM7HhgKrAxYrwDp/JF+M1HoWAsGy/+HVsZzeTRA/hEsIjIAIl6SyrP3btG\nWXP3fZ01jMNx93YzuxF4gqDL7AJ3X2VmtwPL3H0R8A3gl2Z2E0ED+Kfd3c3sA8DtZtYGxIDPu/vO\nI/t4CbbxObj/qmAI6esXsWFT0OOprEgJQ0QGn6gJo9HMTnX31wDCNoUIw2mCuy8m6CobX3Zr3PJq\n4KwejnsYeDhifANvw5/hwWtg1BT41GMwrJiKuuCp69LRCZiQRkQkyaImjK8BvzezrYAB44ArExZV\nqlv3J1h4HRRPg+seg6FFQPAMxuhhOZrzQkQGpagP7i01s+nAtLBonbu3JS6sFLbmcfj9p4NJYK59\nBPIPzM1dUdfIZN2OEpFB6kgGH5wGzCRolD7VzHD3XycmrBS16lF4+LNw3By49mEYMvKgzZX1jbzv\nhCT11BIRSbCoD+59FziHIGEsBi4CXgTSJ2GsfAgeuSEYpuOa30PewbPnNbW2s6OhhTK1X4jIIBW1\nW+3HCbq+bnf3zwCnEAzjkR7eeAAe+RxMek9Qs8g7dKrVTfVNAJSqS62IDFJRE8Z+d48B7WY2HKjh\n4AfyBq/XfgOPfiGYZ+Ka3/c6Z3JlXTDoYKnaMERkkIrahrHMzEYCvwSWA/sIxpAa3JYtgMdvghPO\nhyt/e9jRYivCUWpVwxCRwarPhBGOSPtDd98N/MLM/gQMd/c3Ex5dMr1yF/zx7+DE+XDFr4O5ng+j\nsq6R0cNyGZarSQxFZHDq85aUuztxD965e+WgTxYv/VeQLKZ/GK74TZ/JAqCyvkkN3iIyqEVtw3jN\nzOYlNJJU8Zf/gCe/AzMvh0/cE3nu6ko9gyEig1zU+ydnANeY2SagkeBpb3f3kxMWWTI8/y/w7A9g\n9ifg8l9AZrRfT2NLOzV7WyhT+4WIDGJRE8YHExpFsrkHieKFf4VTPgmX/RdkZEY+vLJePaREZPCL\nmjC8712OYZuXBMni1E/Bh38MGVHv1AUq6zqfwVAbhogMXlETxv8SJA0jGBqkDFgHzEpQXANr8nuD\nB/KOP/eIkwUcqGGoDUNEBrOogw/Ojl83s1OBLyYkomQ54fy+9+lFZV0jxQXqUisig9uRf50Gwnkx\nzujnWI5ZlfWNmjRJRAa9qIMPfj1uNQM4FdiakIiOQRV1TZw7XaPUisjgFvUeSkHccjtBm0bqzoY3\ngPa1tFO3r0VDgojIoBe1DeN7iQ7kWKVBB0UkXURqwzCzp8LBBzvXC83siYjHzjezdWa2wcxu7mH7\nJDN71sxeN7M3zexDcdu+HR63zsxS8lkQPYMhIukiaqN3cTj4IADuvgsY09dBZpYJ3Ekw4dJM4Goz\nm9ltt1uAhe4+F7gK+Fl47MxwfRYwH/hZeL6U0lXD0DMYIjLIRU0YHWY2qXPFzCYT7WG+04EN7r7R\n3VuBB4HLuu3jQOeMRCM40Jh+GfCgu7e4ewWwITxfSqmsb2Ls8Fzyc9SlVkQGt6hXue8AL5rZ8wQP\n770fuCHCcROALXHrVRzaHfc24Ekz+zIwFOh8IGIC8HK3YydEjHfAaNBBEUkXkWoY7v4ngq60vyOo\nJZzm7pHaMCK4GrjH3UuADwG/MbPIz4eY2Q1mtszMltXW1vZTSNHpGQwRSRdRG70/ArS5++Pu/jjB\nVK2XRzi0moOnci0Jy+J9FlgI4O5LCIYeGR3xWNz9bncvd/fy4uKBfRZib3Mbdfta1aVWRNJC1G/y\n33X3PZ0rYQP4dyMctxSYamZlZpZD0Ii9qNs+m4HzAMxsBkHCqA33u8rMcs2sDJgKvBox3gGxqT4Y\ndFATJ4lIOojahtFTYunzWHdvN7MbgSeATGCBu68ys9uBZe6+CPgG8Eszu4mgAfzT4Sx/q8xsIbCa\n4GHBL7l7R8R4B0RFnQYdFJH0ETVhLDOz/yDoIgvwJWB5lAPdfTFxU7yGZbfGLa8Gzurl2B8AP4gY\n44DTQ3sikk6i3pL6MtBK0Oj9O6CFIGmktYr6RsYNz2NITso9HiIi0u+iDg3SCBzylHa621TfpAf2\nRCRtRB2tthj4O4KnrvM6y9393ATFdUyorGvkgpljkx2GiMiAiHpL6j5gLcFMe98DKgl6QKWthuY2\n6hvVpVZE0kfUhFHk7v9N8CzG8+7+t0Da1y5ADd4ikj6i9pJqC9+3mdnFBOM9jUpMSMeGzi61Zaph\niEiaiJowvm9mIwiemfgpwWCBNyUsqmNA50N7k0ap0VtE0kPUXlKPh4t7gL/pvt3Mvu3uP+zPwFJd\nZV0jx41Ql1oRSR+RB/nrwyf66TzHjIr6RrVfiEha6a+EYf10nmNGZV2jekiJSFrpr4QRZTKlQWNP\nUxu7mto06KCIpBXVMN6Fznm8NeigiKST/koYv++n8xwTOhOGutSKSDo5bMIws9Hd1q81s5+Es9x1\n1Src/Z8SFWAqqqhrxExdakUkvfRVw3iyc8HMbgGuIxjW/ALgPxIYV0rbVN/E+BFDyMtWl1oRSR99\nPYcR3zbxUeD97t5oZvcDryUurNRWUdfI5CLVLkQkvfRVwxhiZnPN7DQgMxzmHHdvA1Jq9ruBVFmv\nLrUikn4LySKdAAAQNklEQVT6qmFs48Ctp51mdpy7bzOzIoJpU9PO7qZWdje1UaYeUiKSZg6bMNz9\nkGFAQruAD/R/OKmvMhxDSjUMEUk3UQcfxMw+CryP4CG9F939DxGPmw/8GMgEfuXud3Tb/iMOjE+V\nD4xx95Hhtg5gZbhts7tfGjXeRDkwrLnaMEQkvUSdce9nwAnAA2HR/zGz8939sPN6m1kmcCdBr6oq\nYKmZLXL31Z37uPtNcft/GZgbd4r97j4n0icZIJ1daieqS62IpJmoNYxzgRnu7gBmdi+wKsJxpwMb\n3H1jeNyDwGXA6l72vxr4bsSYkqKyvlFdakUkLUV90nsDMClufWJY1pcJwJa49aqw7BBmNplgCthn\n4orzzGyZmb1sZpf3ctwN4T7LamtrI4R0dCrrm/SEt4ikpagJowBYY2bPmdlzBDWE4Wa2yMwW9VMs\nVwEPuXt8d93J7l4OfBL4TzOb0v0gd7/b3cvdvby4uLifQuldpZ7BEJE0FfWW1K3v8vzVBLWRTiVh\nWU+uAg5qE3H36vB9Y5io5gLvvMtYjtquxlb27G9TDUNE0lLUGfeeN7OxwLyw6FV3r4lw6FJgqpmV\nESSKqwhqCwcxs+lAIbAkrqwQaHL3lnBMq7OAf4kSb6JU1Hf2kFLCEJH0E+mWlJldAbxKMLPeFcAr\nZvbxvo5z93bgRuAJYA2w0N1XmdntZhbfRfYq4MHORvXQDGCZma0AngXuiO9dlQybOhOGahgikoai\n3pL6DjCvs1ZhZsXAn4GH+jrQ3RcDi7uV3dpt/bYejnsJmB0xvgFRUddEhsHEUUOSHYqIyICL2uid\n0e0WVP0RHDtoVNY1Mn7kEHKz1KVWRNJP1BrGH83sCQ48uHcl3WoN6aCyvlEN3iKStqLWEhy4Czg5\nfN2dsIhSlLtTUdeoBm8RSVtRaxgXuPvfA490FpjZ94C/T0hUKWhXUxt7m9vV4C0iaeuwCcPMvgB8\nETjezN6M21QA/DWRgaWaCg06KCJprq8axv3AH4EfAjfHle91950JiyoFdY1SqxqGiKSpvubD2APs\nIRgUMK1V1jcGXWoLVcMQkfSUdl1j363K+iZKCvPJydKvTETSk65+EWnQQRFJd0oYEbg7lXV6BkNE\n0psSRgT1ja3sbWnXMxgiktaUMCLoHHRQNQwRSWdKGBFU1DUBqA1DRNKaEkYElXWNZGYYE0cpYYhI\n+lLCiKCivpGSwiFkZ+rXJSLpS1fACDbVa9BBEREljD4EXWqbNIaUiKQ9JYw+1O1rZV+LRqkVEUl4\nwjCz+Wa2zsw2mNnNPWz/kZm9Eb7eNrPdcduuN7P14ev6RMfak0rN4y0iAkSfD+NdMbNM4E7gAqAK\nWGpmi9x9dec+7n5T3P5fBuaGy6OA7wLlBBM4LQ+P3ZXImLvrHKW2TG0YIpLmEl3DOB3Y4O4b3b0V\neBC47DD7X82BaWA/CDzl7jvDJPEUMD+h0fagsr6RrAyjpHDIQP9oEZGUkuiEMQHYErdeFZYdwswm\nA2XAM0d6bCJV1jVRUjiELHWpFZE0l0pXwauAh9y940gOMrMbzGyZmS2rra3t96Aq6hrVfiEiQuIT\nRjUwMW69JCzryVUcuB0V+Vh3v9vdy929vLi4+CjDPeTcVOoZDBERIPEJYykw1czKzCyHICks6r6T\nmU0HCoElccVPABeaWaGZFQIXhmUDpnZfC02tHRp0UESEBPeScvd2M7uR4EKfCSxw91VmdjuwzN07\nk8dVwIPu7nHH7jSzfyRIOgC3D/Q84pUadFBEpEtCEwaAuy8GFncru7Xb+m29HLsAWJCw4PrQ1aVW\nNQwRkZRq9E45FWGX2gkj1aVWREQJ4zA21TcyaVS+utSKiKCEcVgVdU1qvxARCSlh9MLdg2HN1X4h\nIgIoYfSqZq+61IqIxFPC6EVnDyk9tCciElDC6EXXsOZKGCIigBJGryrqmsjONMaPzEt2KCIiKUEJ\noxeVdY1MVJdaEZEuuhr2orK+UZMmiYjEUcLoQdCltonJShgiIl2UMHqwo6GF/W0dlI3WQ3siIp2U\nMHpQ0dmlVs9giIh0UcLowSZ1qRUROYQSRg8q6hvJycxgvEapFRHpooTRg6BL7RAyMyzZoYiIpAwl\njB5U1jVpDCkRkW6UMLqJxZzK+ka1X4iIdKOE0c2Ovc20tMfUQ0pEpJuEJwwzm29m68xsg5nd3Ms+\nV5jZajNbZWb3x5V3mNkb4WtRomOFuC61qmGIiBwkK5EnN7NM4E7gAqAKWGpmi9x9ddw+U4FvA2e5\n+y4zGxN3iv3uPieRMXZXWdcEQKke2hMROUiiaxinAxvcfaO7twIPApd12+dzwJ3uvgvA3WsSHNNh\nVdY3kpOVwfgR6lIrIhIv0QljArAlbr0qLIt3InCimf3VzF42s/lx2/LMbFlYfnlPP8DMbgj3WVZb\nW3vUAVfWNTJ5VD4Z6lIrInKQhN6SiigLmAqcA5QAL5jZbHffDUx292ozOx54xsxWuvs78Qe7+93A\n3QDl5eV+tMFU1jdq0EERkR4kuoZRDUyMWy8Jy+JVAYvcvc3dK4C3CRII7l4dvm8EngPmJjLYWCwY\npVaDDoqIHCrRCWMpMNXMyswsB7gK6N7b6VGC2gVmNprgFtVGMys0s9y48rOA1STQtgZ1qRUR6U1C\nb0m5e7uZ3Qg8AWQCC9x9lZndDixz90XhtgvNbDXQAXzL3evN7L3AXWYWI0hsd8T3rkqETWGXWk2c\nJCJyqIS3Ybj7YmBxt7Jb45Yd+Hr4it/nJWB2ouOLVxGOUjtZNQwRkUPoSe84lXWN5GZlcNzwvGSH\nIiKScpQw4lTUNTG5SF1qRUR6ooQRZ5MGHRQR6ZUSRigWczbtbFIPKRGRXihhhLbu2U9re0w1DBGR\nXihhhDTooIjI4SlhhCrDLrWaaU9EpGdKGKHKukbysjMYW6AutSIiPVHCCFXWNzJ51FB1qRUR6YUS\nRqiirlHtFyIih6GEAXTEnC0796tLrYjIYShhAFt376e1I6ZBB0VEDkMJA4i5c/Hs45g5fniyQxER\nSVmpMONe0k0uGsqd15ya7DBERFKaahgiIhKJEoaIiESihCEiIpEoYYiISCQJTxhmNt/M1pnZBjO7\nuZd9rjCz1Wa2yszujyu/3szWh6/rEx2riIj0LqG9pMwsE7gTuACoApaa2SJ3Xx23z1Tg28BZ7r7L\nzMaE5aOA7wLlgAPLw2N3JTJmERHpWaJrGKcDG9x9o7u3Ag8Cl3Xb53PAnZ2JwN1rwvIPAk+5+85w\n21PA/ATHKyIivUh0wpgAbIlbrwrL4p0InGhmfzWzl81s/hEcKyIiAyQVHtzLAqYC5wAlwAtmNjvq\nwWZ2A3BDuLrPzNYdRSyjgbqjOD5RFNeRUVxHRnEdmcEY1+QoOyU6YVQDE+PWS8KyeFXAK+7eBlSY\n2dsECaSaIInEH/tc9x/g7ncDd/dHsGa2zN3L++Nc/UlxHRnFdWQU15FJ57gSfUtqKTDVzMrMLAe4\nCljUbZ9HCRODmY0muEW1EXgCuNDMCs2sELgwLBMRkSRIaA3D3dvN7EaCC30msMDdV5nZ7cAyd1/E\ngcSwGugAvuXu9QBm9o8ESQfgdnffmch4RUSkdwlvw3D3xcDibmW3xi078PXw1f3YBcCCRMcYp19u\nbSWA4joyiuvIKK4jk7ZxWXC9FhEROTwNDSIiIpEoYRBt+JKBZmYTzezZuCFTvprsmOKZWaaZvW5m\njyc7lk5mNtLMHjKztWa2xszek+yYAMzspvDf8C0ze8DM8pIYywIzqzGzt+LKRpnZU+EQPE+FnUxS\nIa5/Df8t3zSzP5jZyFSIK27bN8zMw846KRGXmX05/J2tMrN/6e+fm/YJI274kouAmcDVZjYzuVEB\n0A58w91nAmcCX0qRuDp9FViT7CC6+THwJ3efDpxCCsRnZhOArwDl7n4SQeePq5IY0j0cOmLCzcDT\n7j4VeDpcH2j3cGhcTwEnufvJwNsEQwgNtHvoYYQJM5tI0HNz80AHFLqHbnGZ2d8QjKRxirvPAv6t\nv39o2icMog1fMuDcfZu7vxYu7yW4+KXEk+5mVgJcDPwq2bF0MrMRwAeA/wZw91Z3353cqLpkAUPM\nLAvIB7YmKxB3fwHo3tvwMuDecPle4PIBDYqe43L3J929PVx9meBZrKTHFfoR8HcE49wNuF7i+gJw\nh7u3hPvUHHLgUVLCOAaGIDGzUmAu8EpyI+nynwR/LLFkBxKnDKgF/l94q+xXZjY02UG5ezXBN73N\nwDZgj7s/mdyoDjHW3beFy9uBsckMphd/C/wx2UEAmNllQLW7r0h2LN2cCLzfzF4xs+fNbF5//wAl\njBRnZsOAh4GvuXtDCsTzYaDG3ZcnO5ZusoBTgZ+7+1ygkeTcWjlI2B5wGUFCGw8MNbNrkxtV78Ju\n7inVddLMvkNwi/a+FIglH/i/wK197ZsEWcAoglvY3wIWmpn15w9Qwog2fElSmFk2QbK4z90fSXY8\nobOAS82skuD23blm9tvkhgQENcMqd++shT1EkECS7Xygwt1rw+FvHgHem+SYutthZscBhO/9fivj\n3TKzTwMfBq7x1HgGYApB8l8R/g2UAK+Z2bikRhWoAh7xwKsEdwD6tUFeCSPa8CUDLvxm8N/AGnf/\nj2TH08ndv+3uJe5eSvC7esbdk/6N2d23A1vMbFpYdB6w+jCHDJTNwJlmlh/+m55HCjTGd7MI6Jyg\n7HrgsSTG0iUcufrvgEvdvSnZ8QC4+0p3H+PupeHfQBVwavj/L9keBf4GwMxOBHLo50ES0z5hhI1q\nncOXrAEWuvuq5EYFBN/kryP4Bv9G+PpQsoNKcV8G7jOzN4E5wD8lOR7CGs9DwGvASoK/uaQ9KWxm\nDwBLgGlmVmVmnwXuAC4ws/UENaI7UiSu/wIKgKfC//+/SJG4kq6XuBYAx4ddbR8Eru/vWpme9BYR\nkUjSvoYhIiLRKGGIiEgkShgiIhKJEoaIiESihCEiIpEoYYikCDM7J5VG/xXpTglDREQiUcIQOUJm\ndq2ZvRo+THZXODfIPjP7UTgPwdNmVhzuO8fMXo6b06EwLD/BzP5sZivM7DUzmxKefljcnB739fdY\nQCJHQwlD5AiY2QzgSuAsd58DdADXAEOBZeE8BM8D3w0P+TXw9+GcDivjyu8D7nT3UwjGluocLXYu\n8DWCuVmOJ3jiXyQlZCU7AJFjzHnAacDS8Mv/EILB+mLA78J9fgs8Es7RMdLdnw/L7wV+b2YFwAR3\n/wOAuzcDhOd71d2rwvU3gFLgxcR/LJG+KWGIHBkD7nX3g2Z/M7N/6Lbfux1zpyVuuQP9jUoK0S0p\nkSPzNPBxMxsDXfNhTyb4W/p4uM8ngRfdfQ+wy8zeH5ZfBzwfzqBYZWaXh+fIDedZEElp+vYicgTc\nfbWZ3QI8aWYZQBvwJYIJm04Pt9UQtHNAMFz4L8KEsBH4TFh+HXCXmd0enuMTA/gxRN4VjVYr0g/M\nbJ+7D0t2HCKJpFtSIiISiWoYIiISiWoYIiISiRKGiIhEooQhIiKRKGGIiEgkShgiIhKJEoaIiETy\n/wHko/4/rc6h4gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fee3f1f8278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(model.history.history['top_5_accuracy'], label='train');\n",
    "plt.plot(model.history.history['val_top_5_accuracy'], label='val');\n",
    "plt.legend();\n",
    "plt.xlabel('epoch');\n",
    "plt.ylabel('top5_accuracy');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 5120 images belonging to 256 classes.\n"
     ]
    }
   ],
   "source": [
    "val_generator_no_shuffle = data_generator.flow_from_directory(\n",
    "    data_dir + 'val', val_logits,\n",
    "    target_size=(299, 299),\n",
    "    batch_size=64, shuffle=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5.7739551007747654, 0.689453125, 0.87441406249999998, 1.2974167175590992, 5.501888936758041]\n"
     ]
    }
   ],
   "source": [
    "print(model.evaluate_generator(val_generator_no_shuffle, 80))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}