GPT_2_MediumTitles.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GPT_2_MediumTitles.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "dddf199313ce4d8ebba3af31b7e73fb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a47628e306bd48eb905ad166bdb76afb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_40a3c0a6c1d04cd6aa73415577433e5c",
              "IPY_MODEL_efd52da76b1e41f092819bfba7b8a55d"
            ]
          }
        },
        "a47628e306bd48eb905ad166bdb76afb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "40a3c0a6c1d04cd6aa73415577433e5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_25fba9e072a749048aeb94f71aa7ae65",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 125545,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 125545,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_52a7d7f62cf348d4a570cf41bf1680d0"
          }
        },
        "efd52da76b1e41f092819bfba7b8a55d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bdb40fcd64464aa8a2b14683f46f9ad3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 125545/125545 [00:03&lt;00:00, 41038.72it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c3f8aae2644743078f85f37b22bb5a3f"
          }
        },
        "25fba9e072a749048aeb94f71aa7ae65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "52a7d7f62cf348d4a570cf41bf1680d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bdb40fcd64464aa8a2b14683f46f9ad3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c3f8aae2644743078f85f37b22bb5a3f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "21a90963c9da4c98af071da3e4893496": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b56b1905d8aa40a1ac1d0681883c16f3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ed119209c0444996b0a331ae5c29b8d7",
              "IPY_MODEL_020914a46d424a06b2e30f52a16470c2"
            ]
          }
        },
        "b56b1905d8aa40a1ac1d0681883c16f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "ed119209c0444996b0a331ae5c29b8d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_926ab6d24cfe479fbd87ca92142e9f75",
            "_dom_classes": [],
            "description": "Loss: 0.200 — Avg: 0.213 — GPU Mem: 15929 MB: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c269c67a8d214bf697eef6c5bf8df2d5"
          }
        },
        "020914a46d424a06b2e30f52a16470c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_10e44e50e9a04e4a9771a46731a83b59",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5000/5000 [8:02:33&lt;00:00,  5.79s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8b64c47ad6aa455cb2a5e56c3916ab18"
          }
        },
        "926ab6d24cfe479fbd87ca92142e9f75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c269c67a8d214bf697eef6c5bf8df2d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "10e44e50e9a04e4a9771a46731a83b59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8b64c47ad6aa455cb2a5e56c3916ab18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LnCc_ue1_naH"
      },
      "source": [
        "# Generating Medium Titles with GPT-2\r\n",
        "Fine tuning the GPT-2 355M model on a dataset containing titles from Medium blog posts using aitextgen. \r\n",
        "\r\n",
        "## Ethics Disclaimer:\r\n",
        "While a malicious use case isn't clear at the time of writing, it's worth stating that you should under no circumstances use text generated with this notebook to decieve people. You must clearly indicate that the text you are presenting is generated using GPT-2 even if it seems to you like it should be obvious.\r\n",
        "\r\n",
        "\r\n",
        "*By Brian Lechthaler - 3/17/21*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BmcjYFfJDJ1J"
      },
      "source": [
        "## Setup\r\n",
        "Here we install necessary dependencies and import everything we need."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LCIbcNictzwR"
      },
      "source": [
        "!pip install -q aitextgen pandas"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UpaklcazokT0"
      },
      "source": [
        "from aitextgen import aitextgen as atg\r\n",
        "from aitextgen.colab import mount_gdrive as mnt\r\n",
        "from aitextgen.utils import build_gpt2_config\r\n",
        "from aitextgen.tokenizers import train_tokenizer\r\n",
        "import pandas as pd"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gC34gfnxDYWu"
      },
      "source": [
        "## Mount Google Drive\r\n",
        "You will need this if you want to save the trained model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-DSznUSXVlES",
        "outputId": "422126f3-25cf-4272-858a-dc4ad67253db"
      },
      "source": [
        "mnt()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j2SBdimaDqYB"
      },
      "source": [
        "## Query Current GPU's Info\r\n",
        "Here we use the `nvidia-smi` command to grab some helful info about the currently attached GPU."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-cVHPMXWuOW-",
        "outputId": "f960bd21-a05b-4b6d-cd3b-a8c17ecd0f47"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Wed Mar 17 06:06:53 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.56       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   63C    P0    31W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GMoXAbsxD-fl"
      },
      "source": [
        "## Define Function for Saving a File\r\n",
        "We need to be able to write to disk in order to transform our original dataset\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uB7kRIVAPb_P"
      },
      "source": [
        "def writeln(line, path):\r\n",
        "  line = str(line) + \"\\n\"\r\n",
        "  with open(path, 'a') as saveto:\r\n",
        "    saveto.write(line)\r\n",
        "    saveto.close()"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kQBwFFUlFIOr"
      },
      "source": [
        "## Define Function to Preprocess Data\r\n",
        "In order to use our dataset with `aitextgen`, we must first convert it to a CSV file with only 1 column. Because the source dataset is a CSV with multiple columns, we write the following function to preprocess our dataset so that it plays nicely with `aitextgen`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b3f99QOSPcsT"
      },
      "source": [
        "def preprocess(colname, output_filename, input_dataset):\r\n",
        "  dataingest = pd.read_csv(input_dataset)\r\n",
        "  print(\"Transforming dataset...\")\r\n",
        "  #dataingest = dataingest.sample(frac=sample_frac)\r\n",
        "  for index, row in dataingest.iterrows():\r\n",
        "    line = row[colname]\r\n",
        "    if line is not None:\r\n",
        "      writeln(line, output_filename)\r\n",
        "  return dataingest\r\n",
        "  print(\"Done!\")"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1bnD9DWGH8Ee"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lb1g6UF8Fydw"
      },
      "source": [
        "## Preprocess Dataset\r\n",
        "Here we use the function we just wrote to narrow our dataset from a multi-column CSV down to a single-column CSV. We use `wc -l` at the end to count the number of samples we will be re-training GPT-2 `355M` with.\r\n",
        "\r\n",
        "### Don't forget to upload the dataset!\r\n",
        "Please note that the dataset used for this notebook is not included with this notebook. You must [download it from Kaggle](https://www.kaggle.com/nulldata/medium-post-titles), unzip it, and upload the extracted CSV to this colab instance *before* running this cell."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SXfHFPSyPyya",
        "outputId": "f19202d4-f14b-4e4d-c158-2437b8388341"
      },
      "source": [
        "!rm -rf output.csv\r\n",
        "dataset = preprocess('title',\r\n",
        "                     'output.csv',\r\n",
        "                     'medium_post_titles.csv')\r\n",
        "!wc -l output.csv"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Transforming dataset...\n",
            "126418 output.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "62WoukvmeEUP"
      },
      "source": [
        "## Initialize `aitextgen`\r\n",
        "Here we initialize `aitextgen` with the `355M` GPT-2 model and GPU support."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KEfckdpcubBQ"
      },
      "source": [
        "ai = atg(tf_gpt2='355M',\r\n",
        "         to_gpu=True)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0JzXRmynEq6b"
      },
      "source": [
        "## Fine Tune GPT-2 `355M` using a GPU\r\n",
        "Now, for the fun part: using our dataset to train the `355M` GPT-2 model so we can generate Medium blog titles.\r\n",
        "\r\n",
        "### Abuse Deterrance\r\n",
        "As is the case with all other notebooks in this series, models generated here are not included and will never be offered to anyone, ever, for any reason under any circumstance. Additionally, you must upload the dataset after retrieving it yourself by downloading it [here](https://www.kaggle.com/nulldata/medium-post-titles), unzipping it, and uploading the CSV to this Colab instance. These two measures are taken to discourage abuse of the code contained in this notebook."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8e8VyLnYM0QY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "dddf199313ce4d8ebba3af31b7e73fb0",
            "a47628e306bd48eb905ad166bdb76afb",
            "40a3c0a6c1d04cd6aa73415577433e5c",
            "efd52da76b1e41f092819bfba7b8a55d",
            "25fba9e072a749048aeb94f71aa7ae65",
            "52a7d7f62cf348d4a570cf41bf1680d0",
            "bdb40fcd64464aa8a2b14683f46f9ad3",
            "c3f8aae2644743078f85f37b22bb5a3f",
            "21a90963c9da4c98af071da3e4893496",
            "b56b1905d8aa40a1ac1d0681883c16f3",
            "ed119209c0444996b0a331ae5c29b8d7",
            "020914a46d424a06b2e30f52a16470c2",
            "926ab6d24cfe479fbd87ca92142e9f75",
            "c269c67a8d214bf697eef6c5bf8df2d5",
            "10e44e50e9a04e4a9771a46731a83b59",
            "8b64c47ad6aa455cb2a5e56c3916ab18"
          ]
        },
        "outputId": "6021b1f9-1257-4aaf-a113-a4c01bff2f59"
      },
      "source": [
        "ai.train('output.csv',\r\n",
        "         line_by_line=True,\r\n",
        "         from_cache=False,\r\n",
        "         num_steps=5000,\r\n",
        "         generate_every=250,\r\n",
        "         save_every=1000,\r\n",
        "         save_gdrive=True,\r\n",
        "         learning_rate=1e-4,\r\n",
        "         fp16=False,\r\n",
        "         batch_size=9)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dddf199313ce4d8ebba3af31b7e73fb0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=125545.0), HTML(value='')), layout=Layout…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "pytorch_model.bin already exists in /trained_model and will be overwritten!\n",
            "GPU available: True, used: True\n",
            "TPU available: None, using: 0 TPU cores\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:477: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 4, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "21a90963c9da4c98af071da3e4893496",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=5000.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r\u001b[1m250 steps reached: generating sample texts.\u001b[0m\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/checkpoint.py:25: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
            "  warnings.warn(\"None of the inputs have requires_grad=True. Gradients will be None\")\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\r==========\n",
            "\rA quick guide to the best ways to share your designs with friends<|endoftext|>A quick guide to the best ways to share your design work<|endoftext|>A quick guide to the best ways to use the same design<|endoftext|>A quick guide to the best ways to use the same design<|endoftext|>A quick guide to the best ways to use the same design<|endoftext|>A quick guide to the best ways to use the same design<|endoftext|>A quick guide to the best ways to use the same design<|endoftext|>A quick guide to the best ways to use the same design<|endoftext|>A quick guide to the best ways to use the same design<|endoftext|>A quick guide to the best ways to use the same design<|endoftext|>A quick guide to the best ways to use the same design<|endoftext|>A quick guide to the best ways to use the same design<|endoftext|>A quick guide to the best ways to use the same design<|endoftext|>A quick guide to the best ways\n",
            "==========\n",
            "\u001b[1m500 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            " at Home<|endoftext|>The Power of A Good Story<|endoftext|>The Power of A Good Story<|endoftext|>The Power of Agile Teams<|endoftext|>The Power of A Good Story<|endoftext|>The Power of AIM-Focused Team Building<|endoftext|>The Power of Aims<|endoftext|>The Power of AI<|endoftext|>The Power of AI: The Unfinished Business Case<|endoftext|>The Power of Agile vs. Software<|endoftext|>The Power of Advice<|endoftext|>The Power of Awesome Product Management<|endoftext|>The Power of Boredom: The Lessons of “The Princess and the Pea”<|endoftext|>The Power of C# in the Javascript<|endoftext|>The Power of Classification: The Definitive Guide<|endoftext|>The Power of Consciousness in a Digital World<|endoftext|>The Power of Conversation: How to Ask Good Questions<|endoftext|>The Power of Consciousness<|end\n",
            "==========\n",
            "\u001b[1m750 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ">The Rise of the ‘Crazed Coder’<|endoftext|>The Rise of the ‘Shitty<|endoftext|>The Rise of “Crazy Ex-Girlfriend” and How The Myth of “Tidy” Has Put It at Risk<|endoftext|>The Rise of “Disruption”<|endoftext|>The Rise of “Extremely High-Quality<|endoftext|>The Rise of “Insecure”: Why the Rise of the Millennial Generation Is Driving the Next Generation<|endoftext|>The Rise of “Extremely Violent” Software<|endoftext|>The Rise of “Meta-Variations” in Neural Networks<|endoftext|>The Rise of “Meta-Transgressions” in Deep Learning<|endoftext|>The Rise of “Real” Software Engineering<|endoftext|>The Rise of “Social Media”: A Trend That Will Change the World<|endoftext|>The Rise of “Standard” Design in Software Development<|endoftext|\n",
            "==========\n",
            "\u001b[1m1,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m1,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "What is the Difference Between Aperture and Diflour?<|endoftext|>What is the Difference Between Blockchain and Distributed Ledger Technology?<|endoftext|>What is the Difference Between Data Science and Machine Learning?<|endoftext|>What is the Difference Between JavaScript and ECMAScript?<|endoftext|>What is the Difference Between Typescript and React?<|endoftext|>What is the Difference between a Contract and a Token?<|endoftext|>What is the Difference between a Software Engineer and a Developer?<|endoftext|>What is the Impact of the Internet on the Environment?<|endoftext|>What is the Impact of the Internet on the Economy?<|endoftext|>What is the Impact of the Internet: A Brief Overview?<|endoftext|>What is the Keyword Role in SEO?<|endoftext|>What is the Lightning Network?<|endoftext|>What is the Model of Business?<|endoftext|>What is the MERN Award?<|endoftext|>What is the Most Common Type of AI Training?<|\n",
            "==========\n",
            "\u001b[1m1,250 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "|endoftext|>How To: Add an App To Your Library<|endoftext|>How To: Edit a Podcast Episode<|endoftext|>How To: Generate A Random String Using A StringBuilder<|endoftext|>How To: Separate Influential and Non-Influential Clusters in a SVM Case Study<|endoftext|>How To: Stop Being a Social Media Victim<|endoftext|>How To: Write an Effective Introductory Note on “The Art of…<|endoftext|>How To: build a custom React form in TypeScript<|endoftext|>How To: use a cloud-native API in NodeJS<|endoftext|>How To: use the Google Cloud Pipelauncher with Cloud Functions<|endoftext|>How To: use the Google Cloud Storage API in Node.js<|endoftext|>How To: “Be a Good Friend”<|endoftext|>How To: “Follow Your Passion” Makes You A Better Writer<|endoftext|>How Top 1% Companies Influence the Top 100 Companies<|endoftext\n",
            "==========\n",
            "\u001b[1m1,500 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "<|endoftext|>How to Create a Digital Photography Portfolio<|endoftext|>How to Create a Custom Web App in Flutter<|endoftext|>How to Create a Custom Webapp in React Native<|endoftext|>How to Create a DIY E-commerce Website with a Static Website<|endoftext|>How to Create a Digital Native Chat App with React Native<|endoftext|>How to Create a Drive-Through Movie Experience<|endoftext|>How to Create a Dynamic UI Design for Mobile Applications<|endoftext|>How to Create a Firebase Dashboard in less than 10 Minutes<|endoftext|>How to Create a Habit Habit that Lasts (Even After a Breakup)<|endoftext|>How to Create a Home Recording Session<|endoftext|>How to Create a Home Recording Session<|endoftext|>How to Create a Job Search Strategy: Step-by-Step<|endoftext|>How to Create a List of Products Your Company Should Be Buying<|endoftext|>How to Create a Little Community in Your Wilderness World<|endoftext|>\n",
            "==========\n",
            "\u001b[1m1,750 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "The U.S. and China Are Both Decadent on Social Issues<|endoftext|>The U.S. and China Are Both Decentralizing Their Surveillance of Social Media<|endoftext|>The U.S. and India Are on a Collision Course — and Both Are Losing<|endoftext|>The U.S. and Iran Are Both Decadent on Social Issues<|endoftext|>The U.S. government is behind the global community on climate change<|endoftext|>The UAE Is Launching a Cryptocurrency Pilot Program<|endoftext|>The UAE: A Country of Cute Women<|endoftext|>The Ugly Side of Femininity<|endoftext|>The Ugly Side of Glamour<|endoftext|>The Ugly Racism You Never Learned About<|endoftext|>The Ugly Sex Talk We Say To Our Younger Partners<|endoftext|>The Ugly Story of the B-52 Stratified Bomber That Almost Never Made It Out of the Air<|endoftext|>The UAE’s new policy could lead to\n",
            "==========\n",
            "\u001b[1m2,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m2,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "|<|endoftext|>How to learn a new language by creating a “language kit”<|endoftext|>How to learn a new language? Start with a boring one<|endoftext|>How to learn a programming language?<|endoftext|>How to learn a valuable trade by analysing the market<|endoftext|>How to learn about Machine Learning<|endoftext|>How to leave a job<|endoftext|>How to live after a heartbreak<|endoftext|>How to live with and Without Social Media<|endoftext|>How to live with the post-partum depression that followed childbirth<|endoftext|>How to live with the guilt of a heartbreak<|endoftext|>How to live with the guilt of not coming home<|endoftext|>How to live life as if it were a Game of Thrones — and not get raped by the Faceless Man<|endoftext|>How to live with the guilt of “not coming home”<|endoftext|>How to live underground<|endoftext|>How\n",
            "==========\n",
            "\u001b[1m2,250 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "‘We Are Already Living in a ‘New India’’<|endoftext|>‘We Are All Queer’: The Challenges of LGBTQ Visibility in the Workplace<|endoftext|>‘We Are Going to Make Something Out of This’<|endoftext|>‘We Are Never Safe’: A Message for the Nation of Saudi Arabia<|endoftext|>‘We Are Here to Stay’ Taps the Future of the Democratic Party<|endoftext|>‘We Are Not All the Rage of a Revolution’: The Time I Walked With Black Activists in New York City<|endoftext|>‘We Are Not Defined To Be Equal’: A Conversation With Sister Lucinda Van Ness<|endoftext|>‘We Are The Children of Immigrants’<|endoftext|>‘We Are in an Epidemic of Incarceration’<|endoftext|>‘We Can’t Have Everything’: An Excerpt from ‘Faces’<|endoftext|>‘We Didn’\n",
            "==========\n",
            "\u001b[1m2,500 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "I’m Not A ‘Real’ Writer<|— Why Writers Fail At Writing<|endoftext|>I’m Not A ‘Writer’: My Story of Editing Fiction<|endoftext|>I’m Not An “Ayn Rand Republican”<|endoftext|>I’m Not Ashamed to Be Achieving My Goals<|endoftext|>I’m Not Crazy. I’m Just a Hufflepuff. And I’m Okay With It.<|endoftext|>I’m Not Coming to Dinner<|endoftext|>I’m Not Feeling Well<|endoftext|>I’m Not Going to Arrive in Time For The Holiday Season<|endoftext|>I’m Not Guilty! I’m Just Doing My Job!<|endoftext|>I’m Not Guilty! I’m Just a Hufflepuff!<|endoftext|>I’m Not Just A Hufflepuff: My Journey Through My Parents’ Divorce<|endoftext|>I’\n",
            "==========\n",
            "\u001b[1m2,750 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "The Art and Science of Bullet Journaling<|endoftext|>The Art and Science of Making Your Novel a Reality<|endoftext|>The Art and Science of Morning (Break)<|endoftext|>The Art and Science of Night (Break)<|endoftext|>The Art and Science of Off-the-Record<|endoftext|>The Art and Science of Post-Apocalyptic Fiction<|endoftext|>The Art and Science of Revising a Book<|endoftext|>The Art and Science of Writing in Braids<|endoftext|>The Art and Science of the Selfie<|endoftext|>The Art of Art Investment<|endoftext|>The Art of Artichoke<|endoftext|>The Art of Asking for Help and How It Improves Your Life<|endoftext|>The Art of Attraction<|endoftext|>The Art of Avoiding the 9–5<|endoftext|>The Art of Better Writing<|endoftext|>The Art of Better Scrum By Design<|endoftext|>The Art of Bug Reporting<|endof\n",
            "==========\n",
            "\u001b[1m3,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m3,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            " a theory of everything<|endoftext|>A thunk<|endoftext|>A time for Fine Tuning<|endoftext|>A time where love and pain met<|endoftext|>A time-loop analysis of the Bitcoin lightning network<|endoftext|>A timeline of Trump’s impeachable offenses<|endoftext|>A torch<|endoftext|>A tour of awesome features of spaCy (part 1/2)<|endoftext|>A trip to Lab.Oratory<|endoftext|>A trip to the Museum<|endoftext|>A troubled past and a promising future<|endoftext|>A town designed for people<|endoftext|>A town full of strangers is no way to make friends<|endoftext|>A troubled past: The legacy of Pearson v. Chicago<|endoftext|>A threadbare internet is a fragile ecosystem<|endoftext|>A treat for the broken<|endoftext|>A video game designed to make you a better designer<|endoftext|>A view of the early Earth — the most important date\n",
            "==========\n",
            "\u001b[1m3,250 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "<|endoftext|>A Guide to Being a Great Product Marketer<|endoftext|>A Guide to Better Meetings<|endoftext|>A Guide to CSS Animation<|endoftext|>A Guide to Caching CRUD Web Designs<|endoftext|>A Guide to Choosing Health Insurance<|endoftext|>A Guide to Decision Trees<|endoftext|>A Guide to EMF Waves in the 21st Century<|endoftext|>A Guide to Editing Your Content<|endoftext|>A Guide to Establishing a Custom Domain Name on Google Cloud<|endoftext|>A Guide to Getting Lunch with Stephen King<|endoftext|>A Guide to Grading Your Data Science Resume<|endoftext|>A Guide to GraphQL: From Zero to Hero<|endoftext|>A Guide to Having More Than One Opinion<|endoftext|>A Guide to Learning to Code by Writing Code<|endoftext|>A Guide to Lessing<|endoftext|>A Guide to Miami’s Fashion Scene — #TBT<|endoftext\n",
            "==========\n",
            "\u001b[1m3,500 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "“You Never Write Alone”<|endoftext|>“You Never Walk Out the Same Way”: How I Survived and Thrived in an Abusive Relationship<|endoftext|>“You Really Can’t Afford to Be Broke.”<|endoftext|>“You Should Have Asked!” — How Insight Drives Business Success in Startup Brands.<|endoftext|>“You are wrong<|endoftext|>“You are what you make”<|endoftext|>“You can’t make this stuff up” — the early days of wearable technology<|endoftext|>“You can’t love yourself “ until you love yourself;”<|endoftext|>“You don’t get it. You don’t know who I am.”<|endoftext|>“You don’t look ill\" — Sunset overdrive artist reflects on “seeing the real me”<|endoftext|>“You don’t understand.”\n",
            "==========\n",
            "\u001b[1m3,750 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "The Truth About Finding a Job You Want<|endoftext|>The Truth About Gettleman’s Final Exam<|endoftext|>The Truth About Human Nature<|endoftext|>The Truth About Immigration: Asylum<|endoftext|>The Truth About Immigration: We Don’t Need Any More<|endoftext|>The Truth About Instagram’s New Anti-Bullying Reset<|endoftext|>The Truth About Invisible Tech<|endoftext|>The Truth About Ischemotherapy — Ishtar Review<|endoftext|>The Truth About Kids Getting “Hyped Up” on Sugar<|endoftext|>The Truth About Loneliness<|endoftext|>The Truth About Mental Health and Addiction<|endoftext|>The Truth About Me<|endoftext|>The Truth About Morning and Night People<|endoftext|>The Truth About Owning a Home<|endoftext|>The Truth About Post-Mortem — Someday We’ll Be Back to the ‘Dark Ages’<|endoftext|>\n",
            "==========\n",
            "\u001b[1m4,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m4,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "Kellogg’s Law<|endoftext|>Kellogg’s<|endoftext|>Ken Starr Recommended Impeachment. So Why Won’t Mueller?<|endoftext|>Kenya fights to save coral reef<|endoftext|>Kenya: The Rise of a Failed State<|endoftext|>Kenya: a model for social and regional transformation<|endoftext|>Kenya’s devastating earthquake 30 years on<|endoftext|>Kenya’s small and poor state is home to one of the worst humanitarian disasters in the world<|endoftext|>Kenya’s ‘waters of life’ could be one of the greatest examples of climate’s impact in…<|endoftext|>Kew Gardens: A Social Commentary on Poverty<|endoftext|>Keto Diet: It’s a Matter of Mind and Body<|endoftext|>Keto vs. Carnivore Diet<|endoftext|>Keto: The new kid on the block<|endoftext|>Keto: The truth about the effects of a low-\n",
            "==========\n",
            "\u001b[1m4,250 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "You Need To Be Critical When Collaborating With Other Authors<|endoftext|>You Need To Believe In Yourself<|endoftext|>You Need To Let Me Win<|endoftext|>You Need a Brand Language<|endoftext|>You Need a Different Approach to Writing About Mental Illness<|endoftext|>You Need a Critique if You Want to Be Critical<|endoftext|>You Need a Dose of Real Self-Worth<|endoftext|>You Need a Feedback Factory<|endoftext|>You Need a Good Social Media Rep<|endoftext|>You Need to Be Brave to Work in Tech<|endoftext|>You Need to Be Irresponsible<|endoftext|>You Need to Be Kind to Everyone<|endoftext|>You Need to Be More Than an Inbox<|endoftext|>You Need to Be More than an Inbox<|endoftext|>You Need to Celebrate Your Differences<|endoftext|>You Need to Explore Your Creative Streak<|endoftext|>You Need to Fail to Become the Best Writer You Can Be<|end\n",
            "==========\n",
            "\u001b[1m4,500 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "A Beginner’s Guide to Gauge Equations<|endoftext|>A Beginner’s Guide to GraphQL (with Python)<|endoftext|>A Beginner’s Guide to IPv4 Subnet Masking in Python<|endoftext|>A Beginner’s Guide to Learning Machine Learning<|endoftext|>A Beginner’s Guide to Lodash and Its Consequences<|endoftext|>A Beginner’s Guide to Machine Learning with TensorFlow<|endoftext|>A Beginner’s Guide to Sentiment Analysis for Stock Market<|endoftext|>A Beginner’s Guide to Setting Up a React App<|endoftext|>A Beginner’s Guide to Setting Up a React Native App<|endoftext|>A Beginner’s Guide to Text Classification<|endoftext|>A Beginner’s Guide to the React Handbook<|endoftext|>A Beginner’s Guide to the Use of Conditionals in React<|endoftext|>A Beginner’s Guide to the Visual Studio Code\n",
            "==========\n",
            "\u001b[1m4,750 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "How I Write My Own Voice<|endoftext|>How I Write a Book and Other Musings<|endoftext|>How I Wrote 24 Fancy-Filled Memoir in One Year<|endoftext|>How I Wrote My Published Memoir<|endoftext|>How I Wrote a Full-Length Story for a Movie Script I Just Can’t Release<|endoftext|>How I Wrote the Rest of My Memoir<|endoftext|>How I accidentally hedged and killed a Cryptocurrency<|endoftext|>How I added Dark Mode to my website<|endoftext|>How I added dark mode to my website<|endoftext|>How I am Automating My Productivity Using This Smart Home Setup<|endoftext|>How I am Changing as a Writer<|endoftext|>How I analyzed developer trends using data science<|endoftext|>How I analyzed the GitHub Issues from the most popular Front-End frameworks<|endoftext|>How I automated creating a new task each day without losing focus<|endoftext|>How I automated my Twitter stream with this Twitch\n",
            "==========\n",
            "\u001b[1m5,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m5,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "What Could Go Wrong?<|endoftext|>What Could Go Wrong?<|endoftext|>What Could Possibly Go Wrong? A Primer on Improving Customer Experience for Healthcare Products<|endoftext|>What Cracked the Pauli Existence?<|endoftext|>What Creators Should Know About Marketing Landing. Part I: Get the Right People in the Room<|endoftext|>What Creators Should Know About Marketing Landing. Part II: How To Make Your Own Call-to-Action<|endoftext|>What Creators Should Know About Marketing Landing. Part III: How To Optimize Your Chance Of Getting It…<|endoftext|>What Creators can learn from Billy McFarland and the MGsXR Podcast<|endoftext|>What Could President Trump’s Read As A Means of War?<|endoftext|>What Created a Bronze-Age Man with Feathery Tails Can Teach Today’s Children<|endoftext|>What Crypto Manipulation Tools Will We Find in the Future?<|endoftext|>What Crypto Wallets Truly Mean for Payments<|endoftext|\n",
            "==========\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}