infer-final-v1-final-7d-7dx-ens-no-tta.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# Final Inference Notebook with TTA"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "This solution/module/whatever makes use of the following libraries/modules:\n",
    "\n",
    "EfficientDet-PyTorch (https://github.com/rwightman/efficientdet-pytorch) licensed under Apache 2.0, Copyright Ross Wightman, License: third_party/effdet/LICENSE\n",
    "\n",
    "PyTorch (https://github.com/pytorch/pytorch) "
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "## Introduction\n",
    "\n",
    "This notebook aims to provide a working training script for the updated version of Effdet created by [@rwrightman](https://www.kaggle.com/rwightman). I realised the training code released by [@shonenkov](https://www.kaggle.com/shonenkov) does not work for the updated Effdet, hence, i made some changes accordingly.\n",
    "\n",
    "#### Main Changes:\n",
    "* **DetBenchTrain** forward() now takes in a dictionary object with key:value pair {bbox:, cls:, img_size:, img_scale:} as argument\n",
    "* **DetBenchPredict** forward() takes in (image, img_size, img_scale) as argumemnt\n",
    "\n",
    "(Im not sure what is the significance of img_scale and img_size, would appreciate if anyone can advise on this)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Download and Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: albumentations in /opt/conda/lib/python3.7/site-packages (0.4.5)\r\n",
      "Requirement already satisfied: opencv-python>=4.1.1 in /opt/conda/lib/python3.7/site-packages (from albumentations) (4.2.0.34)\r\n",
      "Requirement already satisfied: imgaug<0.2.7,>=0.2.5 in /opt/conda/lib/python3.7/site-packages (from albumentations) (0.2.6)\r\n",
      "Requirement already satisfied: PyYAML in /opt/conda/lib/python3.7/site-packages (from albumentations) (5.3.1)\r\n",
      "Requirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from albumentations) (1.4.1)\r\n",
      "Requirement already satisfied: numpy>=1.11.1 in /opt/conda/lib/python3.7/site-packages (from albumentations) (1.18.5)\r\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from imgaug<0.2.7,>=0.2.5->albumentations) (1.14.0)\r\n",
      "Requirement already satisfied: scikit-image>=0.11.0 in /opt/conda/lib/python3.7/site-packages (from imgaug<0.2.7,>=0.2.5->albumentations) (0.16.2)\r\n",
      "Requirement already satisfied: matplotlib!=3.0.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-image>=0.11.0->imgaug<0.2.7,>=0.2.5->albumentations) (3.2.1)\r\n",
      "Requirement already satisfied: networkx>=2.0 in /opt/conda/lib/python3.7/site-packages (from scikit-image>=0.11.0->imgaug<0.2.7,>=0.2.5->albumentations) (2.4)\r\n",
      "Requirement already satisfied: pillow>=4.3.0 in /opt/conda/lib/python3.7/site-packages (from scikit-image>=0.11.0->imgaug<0.2.7,>=0.2.5->albumentations) (5.4.1)\r\n",
      "Requirement already satisfied: imageio>=2.3.0 in /opt/conda/lib/python3.7/site-packages (from scikit-image>=0.11.0->imgaug<0.2.7,>=0.2.5->albumentations) (2.8.0)\r\n",
      "Requirement already satisfied: PyWavelets>=0.4.0 in /opt/conda/lib/python3.7/site-packages (from scikit-image>=0.11.0->imgaug<0.2.7,>=0.2.5->albumentations) (1.1.1)\r\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image>=0.11.0->imgaug<0.2.7,>=0.2.5->albumentations) (2.4.7)\r\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image>=0.11.0->imgaug<0.2.7,>=0.2.5->albumentations) (1.2.0)\r\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image>=0.11.0->imgaug<0.2.7,>=0.2.5->albumentations) (2.8.1)\r\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image>=0.11.0->imgaug<0.2.7,>=0.2.5->albumentations) (0.10.0)\r\n",
      "Requirement already satisfied: decorator>=4.3.0 in /opt/conda/lib/python3.7/site-packages (from networkx>=2.0->scikit-image>=0.11.0->imgaug<0.2.7,>=0.2.5->albumentations) (4.4.2)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install albumentations\n",
    "!pip install --no-deps '../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null\n",
    "!pip install --no-deps '../input/timm-0130/timm-0.1.30-py3-none-any.whl' > /dev/null\n",
    "import sys\n",
    "sys.path.insert(0,'../input/effidetpytorch/effidetpytorch') #add packages to system path to allow import\n",
    "sys.path.insert(0,'../input/torch-img-model')\n",
    "sys.path.insert(0,'../input/omegaconf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import torch\n",
    "import os\n",
    "from glob import glob\n",
    "import random\n",
    "from tqdm.notebook import tqdm\n",
    "import cv2\n",
    "import albumentations as A\n",
    "from torch.utils.data import Dataset,DataLoader\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from albumentations.pytorch.transforms import ToTensorV2\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as patches"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "IMG_SIZE = 1024"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Model Instantiation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import effdet\n",
    "from effdet import get_efficientdet_config, EfficientDet, DetBenchTrain\n",
    "from effdet.efficientdet import HeadNet\n",
    "\n",
    "#def get_net():\n",
    "#    config = get_efficientdet_config('tf_efficientdet_d7')\n",
    "#    net = EfficientDet(config, pretrained_backbone=False)\n",
    "#    checkpoint = torch.load('../input/efficientdet-model/eff_det_models/tf_efficientdet_d5-ef44aea8.pth') #d3-d7 ('efficientdet_model' folder) \n",
    "#    net.load_state_dict(checkpoint)\n",
    "#    config.num_classes = 1\n",
    "#    config.image_size = IMG_SIZE\n",
    "#    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n",
    "#    return DetBenchTrain(net, config)\n",
    "\n",
    "# net = get_net()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Run training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "## Inference\n",
    "For the inference, i have referred to [@shonenkov](https://www.kaggle.com/shonenkov) insightful use of Weighted Box Fusion(WBF) that uses information from all boxes to fix the overlapping bounding boxes issues as well as the Test Time Augmentation(TTA) template that he has kindly provided [here](https://www.kaggle.com/shonenkov/wbf-over-tta-single-model-efficientdet). Do head over and read about it!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "sys.path.insert(0, \"../input/weightedboxfusion\")\n",
    "\n",
    "import gc\n",
    "from effdet import DetBenchPredict\n",
    "from ensemble_boxes import *\n",
    "\n",
    "TEST_PATH = \"../input/global-wheat-detection/test/\""
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Test Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WheatData(Dataset):\n",
    "    def __init__(self, img_ids, transform=None):\n",
    "        self.img_ids = img_ids\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        img_id = self.img_ids[index]\n",
    "        image = cv2.imread(f'{TEST_PATH}/{img_id}.jpg', cv2.IMREAD_COLOR)\n",
    "        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n",
    "        image = image /255.0\n",
    "        \n",
    "        if self.transform:\n",
    "            sample = {'image' : image}\n",
    "            sample = self.transform(**sample)\n",
    "            image = sample['image']\n",
    "        \n",
    "        target = {}\n",
    "        target['img_scale'] = torch.tensor([1.])\n",
    "            \n",
    "        return image, img_id, target\n",
    "        \n",
    "    def __len__(self) -> int: #annotate parameters with their expected type\n",
    "        return self.img_ids.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid_transform():\n",
    "    return A.Compose([\n",
    "            A.Resize(height=IMG_SIZE, width=IMG_SIZE, p=1.0),\n",
    "            ToTensorV2(p=1.0)], \n",
    "            p=1.0)\n",
    "\n",
    "\n",
    "def collate_fn(batch):\n",
    "    return tuple(zip(*batch))\n",
    "\n",
    "\n",
    "test_dataset = WheatData(\n",
    "    img_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{TEST_PATH}/*.jpg')]),\n",
    "    transform=valid_transform())\n",
    "\n",
    "test_loader = DataLoader(test_dataset,\n",
    "                         batch_size = 4,\n",
    "                         shuffle = False,\n",
    "                         drop_last = False,\n",
    "                         collate_fn = collate_fn) "
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Load saved model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### All the weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#WEIGHTS = glob(\"/kaggle/input/*/*.bin\")\n",
    "WEIGHTS  = [\n",
    "#     '/kaggle/input/0723-effdet7valid-f0/best-checkpoint-043epoch.bin',\n",
    "#             '/kaggle/input/0804-effidetd7/f1_ep39_loss_0.404.bin',\n",
    "#             '/kaggle/input/0725-effidetd7/f2_ep_39_loss_0.39553.bin',\n",
    "#             '/kaggle/input/0725-effidetd7/f3_ep_33_loss_0.40588.bin',\n",
    "#             '/kaggle/input/0725-effidetd7/f4_ep_41_loss_0.39407.bin',\n",
    "            '/kaggle/input/0803-ramen-dx-3cups/f0-0387-ep64.bin',\n",
    "            '/kaggle/input/0803-ramen-dx-3cups/f1-0387-ep42.bin',\n",
    "            '/kaggle/input/0804ramendxf2/f2_ep31_loss_0.3883.bin']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pretrained_tag(path):\n",
    "    if \"ramen\" in path:\n",
    "        return 'tf_efficientdet_d7x'\n",
    "    else:\n",
    "        return 'tf_efficientdet_d7'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NET(torch.nn.Module):\n",
    "    def __init__(self,model):\n",
    "        super().__init__()\n",
    "        self.model = model\n",
    "        \n",
    "    def forward(self,*args,**kwargs):\n",
    "        return self.model(*args,**kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[7DX] from /kaggle/input/0803-ramen-dx-3cups/f0-0387-ep64.bin\n",
      "[7DX] from /kaggle/input/0803-ramen-dx-3cups/f1-0387-ep42.bin\n",
      "[7DX] from /kaggle/input/0804ramendxf2/f2_ep31_loss_0.3883.bin\n"
     ]
    }
   ],
   "source": [
    "def load_net(checkpoint_path,):\n",
    "    tag = get_pretrained_tag(checkpoint_path)\n",
    "    \n",
    "    \n",
    "    if tag == 'tf_efficientdet_d7x':\n",
    "        print(f'[7DX] from {checkpoint_path}')\n",
    "        config = get_efficientdet_config(tag)\n",
    "        model = EfficientDet(config, pretrained_backbone=False)\n",
    "    \n",
    "        config.num_classes = 1\n",
    "        config.image_size = IMG_SIZE\n",
    "        model.class_net = HeadNet(config, num_outputs=config.num_classes, \n",
    "                                norm_kwargs=dict(eps=.001, momentum=.01))\n",
    "    \n",
    "        net = NET(model)\n",
    "    \n",
    "        checkpoint = torch.load(checkpoint_path)\n",
    "        #net.load_state_dict(checkpoint['model_state_dict'])\n",
    "    \n",
    "        import collections\n",
    "        apex_model = collections.OrderedDict()\n",
    "        for key, value in checkpoint['model_state_dict'].items():\n",
    "            k = 'model.'+ key\n",
    "            apex_model[k] = value\n",
    "            #if 'anchor_labeler.anchors.boxes' in key:\n",
    "            #    pass\n",
    "            #elif 'model.' in key:\n",
    "            #    k = key[5:]\n",
    "            #    apex_model[k] = value\n",
    "            #\n",
    "            #else:\n",
    "            #    apex_model[key] = value\n",
    "        net.load_state_dict(apex_model)\n",
    "    \n",
    "        del checkpoint\n",
    "        del apex_model\n",
    "        gc.collect()\n",
    "    \n",
    "        \n",
    "    else:\n",
    "        print(f'[7D] from {checkpoint_path}')\n",
    "        config = get_efficientdet_config(tag)\n",
    "        model = EfficientDet(config, pretrained_backbone=False)\n",
    "    \n",
    "        config.num_classes = 1\n",
    "        config.image_size = IMG_SIZE\n",
    "        model.class_net = HeadNet(config, num_outputs=config.num_classes, \n",
    "                            norm_kwargs=dict(eps=.001, momentum=.01))\n",
    "    \n",
    "        net = NET(model)\n",
    "        checkpoint = torch.load(checkpoint_path)\n",
    "        \n",
    "        import collections\n",
    "        apex_model = collections.OrderedDict()\n",
    "        for key, value in checkpoint['model'].items():\n",
    "            if 'anchor_labeler.anchors.boxes' in key:\n",
    "                pass\n",
    "            else:\n",
    "                apex_model[key] = value\n",
    "        net.load_state_dict(apex_model)\n",
    "    \n",
    "        del checkpoint\n",
    "        del apex_model\n",
    "        gc.collect()\n",
    "    \n",
    "    net = DetBenchPredict(net, config)\n",
    "    \n",
    "    net.eval()\n",
    "    net = net.cuda()\n",
    "    return net.cuda()\n",
    "\n",
    "# load\n",
    "Nets = list(load_net(weight) for weight in WEIGHTS)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Testtime inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "class BaseWheatTTA:\n",
    "    \"\"\" author: @shonenkov \"\"\"\n",
    "    image_size = IMG_SIZE\n",
    "\n",
    "    def augment(self, image):\n",
    "        raise NotImplementedError\n",
    "    \n",
    "    def batch_augment(self, images):\n",
    "        raise NotImplementedError\n",
    "    \n",
    "    def deaugment_boxes(self, boxes):\n",
    "        raise NotImplementedError\n",
    "\n",
    "class TTAHorizontalFlip(BaseWheatTTA):\n",
    "    \"\"\" author: @shonenkov \"\"\"\n",
    "\n",
    "    def augment(self, image):\n",
    "        return image.flip(1)\n",
    "    \n",
    "    def batch_augment(self, images):\n",
    "        return images.flip(2)\n",
    "    \n",
    "    def deaugment_boxes(self, boxes):\n",
    "        boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]\n",
    "        return boxes\n",
    "\n",
    "class TTAVerticalFlip(BaseWheatTTA):\n",
    "    \"\"\" author: @shonenkov \"\"\"\n",
    "    \n",
    "    def augment(self, image):\n",
    "        return image.flip(2)\n",
    "    \n",
    "    def batch_augment(self, images):\n",
    "        return images.flip(3)\n",
    "    \n",
    "    def deaugment_boxes(self, boxes):\n",
    "        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n",
    "        return boxes\n",
    "    \n",
    "class TTARotate90(BaseWheatTTA):\n",
    "    \"\"\" author: @shonenkov \"\"\"\n",
    "    \n",
    "    def augment(self, image):\n",
    "        return torch.rot90(image, 1, (1, 2))\n",
    "\n",
    "    def batch_augment(self, images):\n",
    "        return torch.rot90(images, 1, (2, 3))\n",
    "    \n",
    "    def deaugment_boxes(self, boxes):\n",
    "        res_boxes = boxes.copy()\n",
    "        res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]\n",
    "        res_boxes[:, [1,3]] = boxes[:, [2,0]]\n",
    "        return res_boxes\n",
    "\n",
    "class TTACompose(BaseWheatTTA):\n",
    "    \"\"\" author: @shonenkov \"\"\"\n",
    "    def __init__(self, transforms):\n",
    "        self.transforms = transforms\n",
    "        \n",
    "    def augment(self, image):\n",
    "        for transform in self.transforms:\n",
    "            image = transform.augment(image)\n",
    "        return image\n",
    "    \n",
    "    def batch_augment(self, images):\n",
    "        for transform in self.transforms:\n",
    "            images = transform.batch_augment(images)\n",
    "        return images\n",
    "    \n",
    "    def prepare_boxes(self, boxes):\n",
    "        result_boxes = boxes.copy()\n",
    "        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n",
    "        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n",
    "        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n",
    "        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n",
    "        return result_boxes\n",
    "    \n",
    "    def deaugment_boxes(self, boxes):\n",
    "        for transform in self.transforms[::-1]:\n",
    "            boxes = transform.deaugment_boxes(boxes)\n",
    "        return self.prepare_boxes(boxes)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Weighted Box Fusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "def run_wbf(predictions, image_index, image_size=IMG_SIZE, iou_thr=0.44, \n",
    "            skip_box_thr=0.43, weights=None):\n",
    "    boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]\n",
    "    scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]\n",
    "    labels = [np.ones(prediction[image_index]['scores'].shape[0]).astype(int).tolist() for prediction in predictions]\n",
    "    boxes, scores, labels = ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n",
    "    boxes = boxes*(image_size-1)\n",
    "    return boxes, scores, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Running inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_prediction_string(boxes, scores):\n",
    "    pred_strings = []\n",
    "    for j in zip(scores, boxes):\n",
    "        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], \n",
    "                                                             j[1][2], j[1][3]))\n",
    "    return \" \".join(pred_strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "from itertools import product\n",
    "\n",
    "tta_transforms = []\n",
    "for tta_combination in product([TTAHorizontalFlip(), None], \n",
    "                               [TTAVerticalFlip(), None],\n",
    "                               [TTARotate90(), None]):\n",
    "    tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# WBF over TTA\n",
    "def predict_single(net,tta = True):\n",
    "    def predict_func(images, target, score_thres=0.5):\n",
    "        with torch.no_grad():\n",
    "            prediction = []\n",
    "            images = torch.stack(images).to(device).float()\n",
    "            img_scale = torch.tensor([1]*images.shape[0]).float().cuda()\n",
    "            img_size = torch.tensor([(IMG_SIZE, IMG_SIZE) for target in targets]).to(device)\n",
    "    \n",
    "            '''\n",
    "    \n",
    "            Within the forward function of the DetBenchPredict class, it takes in 3 arguments (image, image_scale, image_size)\n",
    "            The return object is as follows: \n",
    "            detections = torch.cat([boxes, scores, classes.float()], dim=1) \n",
    "            where the first 4 col will be the bboxes, 5th col the scores\n",
    "            Find out more at https://github.com/rwightman/efficientdet-pytorch/blob/master/effdet/bench.py\n",
    "    \n",
    "            '''\n",
    "            if tta:\n",
    "                for tta_transform in tta_transforms:\n",
    "                    result = []\n",
    "                    det = net(tta_transform.batch_augment(images.clone()),\n",
    "                              img_scales = img_scale,\n",
    "                              img_size = img_size)\n",
    "        \n",
    "                    for i in range(images.shape[0]):\n",
    "                        boxes = det[i].detach().cpu().numpy()[:,:4]    \n",
    "                        scores = det[i].detach().cpu().numpy()[:,4]\n",
    "                        indexes = np.where(scores > score_thres)[0]\n",
    "                        boxes = boxes[indexes]\n",
    "                        boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n",
    "                        boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n",
    "                        boxes = tta_transform.deaugment_boxes(boxes.copy())\n",
    "                        result.append({\n",
    "                            'boxes': boxes,\n",
    "                            'scores': scores[indexes],\n",
    "                        })\n",
    "        \n",
    "                    prediction.append(result)\n",
    "            else:\n",
    "                result = []\n",
    "                det = net(images.clone(),\n",
    "                              img_scales = img_scale,\n",
    "                              img_size = img_size)\n",
    "        \n",
    "                for i in range(images.shape[0]):\n",
    "                    boxes = det[i].detach().cpu().numpy()[:,:4]    \n",
    "                    scores = det[i].detach().cpu().numpy()[:,4]\n",
    "                    indexes = np.where(scores > score_thres)[0]\n",
    "                    boxes = boxes[indexes]\n",
    "                    boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n",
    "                    boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n",
    "                    result.append({\n",
    "                            'boxes': boxes,\n",
    "                            'scores': scores[indexes],\n",
    "                        })\n",
    "        \n",
    "                prediction.append(result)\n",
    "    \n",
    "        return prediction\n",
    "    return predict_func"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(images, target,tta=False, score_thres=0.5):\n",
    "    preds = []\n",
    "    for net in Nets:\n",
    "        preds.append(predict_single(net,tta=tta)(images, target, score_thres))\n",
    "    return list(chain(*preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "firstnet = Nets[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/conda-bld/pytorch_1591914880026/work/aten/src/ATen/native/BinaryOps.cpp:81: UserWarning: Integer division of tensors using div or / is deprecated, and in a future release div will perform true division as in Python 3. Use true_divide or floor_divide (// in Python) instead.\n",
      "../input/weightedboxfusion/ensemble_boxes/ensemble_boxes_wbf.py:85: UserWarning: Y2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.\n",
      "  warnings.warn('Y2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n",
      "../input/weightedboxfusion/ensemble_boxes/ensemble_boxes_wbf.py:73: UserWarning: X2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.\n",
      "  warnings.warn('X2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n",
      "../input/weightedboxfusion/ensemble_boxes/ensemble_boxes_wbf.py:73: UserWarning: X2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.\n",
      "  warnings.warn('X2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n",
      "../input/weightedboxfusion/ensemble_boxes/ensemble_boxes_wbf.py:85: UserWarning: Y2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.\n",
      "  warnings.warn('Y2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 33.7 s, sys: 15.5 s, total: 49.2 s\n",
      "Wall time: 50.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "results = []\n",
    "for images, image_ids, targets in test_loader:\n",
    "    predictions = predict(images, targets,tta = True)\n",
    "    for i, image in enumerate(images):\n",
    "        boxes, scores, labels = run_wbf(predictions, image_index=i)\n",
    "        boxes = (boxes*1024/1024).astype(np.int32).clip(min=0, max=1023)\n",
    "        image_id = image_ids[i]\n",
    "        \n",
    "        boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n",
    "        boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n",
    "\n",
    "        result = {\n",
    "            'image_id': image_id,\n",
    "            'PredictionString': format_prediction_string(boxes, scores)\n",
    "        }\n",
    "        results.append(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>PredictionString</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>348a992bb</td>\n",
       "      <td>0.8578 732 222 141 88 0.8136 598 444 122 98 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>796707dd7</td>\n",
       "      <td>0.8602 709 823 110 104 0.8564 895 331 113 94 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aac893a91</td>\n",
       "      <td>0.8652 559 532 121 188 0.8521 28 451 103 158 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f5a1f0358</td>\n",
       "      <td>0.8969 688 204 113 93 0.8917 943 435 79 185 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cb8d261a3</td>\n",
       "      <td>0.8743 753 489 120 91 0.8529 311 167 101 201 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cc3532ff6</td>\n",
       "      <td>0.9557 772 828 164 161 0.8581 909 124 112 96 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>51f1be19e</td>\n",
       "      <td>0.8374 608 81 160 180 0.8209 839 265 136 205 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>51b3e36ab</td>\n",
       "      <td>0.8760 495 359 315 131 0.8742 870 287 152 142 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>53f253011</td>\n",
       "      <td>0.8863 14 33 145 110 0.8830 621 100 119 146 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2fd875eaa</td>\n",
       "      <td>0.8874 106 584 141 85 0.8742 730 155 83 88 0.8...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    image_id                                   PredictionString\n",
       "0  348a992bb  0.8578 732 222 141 88 0.8136 598 444 122 98 0....\n",
       "1  796707dd7  0.8602 709 823 110 104 0.8564 895 331 113 94 0...\n",
       "2  aac893a91  0.8652 559 532 121 188 0.8521 28 451 103 158 0...\n",
       "3  f5a1f0358  0.8969 688 204 113 93 0.8917 943 435 79 185 0....\n",
       "4  cb8d261a3  0.8743 753 489 120 91 0.8529 311 167 101 201 0...\n",
       "5  cc3532ff6  0.9557 772 828 164 161 0.8581 909 124 112 96 0...\n",
       "6  51f1be19e  0.8374 608 81 160 180 0.8209 839 265 136 205 0...\n",
       "7  51b3e36ab  0.8760 495 359 315 131 0.8742 870 287 152 142 ...\n",
       "8  53f253011  0.8863 14 33 145 110 0.8830 621 100 119 146 0....\n",
       "9  2fd875eaa  0.8874 106 584 141 85 0.8742 730 155 83 88 0.8..."
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\n",
    "test_df.to_csv('submission.csv', index=False)\n",
    "test_df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}