Image PreProcessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import zipfile\n",
    "import os\n",
    "import csv\n",
    "import sys\n",
    "import imagehash\n",
    "from io import BytesIO\n",
    "from PIL import Image\n",
    "\n",
    "def create_image_hashes(zipfilename, csvfilename):\n",
    "    archive = zipfile.ZipFile(zipfilename, 'r')\n",
    "    map_hashes = {}\n",
    "    allfiles = archive.namelist()\n",
    "    filecount = len(allfiles)\n",
    "    for filename in archive.namelist():\n",
    "        if not filename.lower().endswith(\".jpg\"):\n",
    "            continue\n",
    "\n",
    "        imgdata = archive.read(filename)\n",
    "\n",
    "        try:            \n",
    "            im = Image.open(BytesIO(imgdata))\n",
    "\n",
    "            img_hash = imagehash.average_hash(im)\n",
    "            img_key = os.path.basename(os.path.splitext(filename)[0])\n",
    "            map_hashes[img_key] = img_hash\n",
    "        except BaseException as e:\n",
    "            print('Cannot process image: %s' % filename, e)\n",
    "#             break\n",
    "            continue\n",
    "    print('From %s %d images loaded' % (zipfilename, len(map_hashes)))\n",
    "    \n",
    "    with open(csvfilename, 'w+') as csvfile:\n",
    "        writer = csv.writer(csvfile, delimiter=',', quotechar='\"')\n",
    "\n",
    "        for key in map_hashes:\n",
    "            writer.writerow([key, map_hashes[key]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing file data/Images_0.zip\n",
      "Cannot process image: Images_0/6/9268406.jpg cannot identify image file <_io.BytesIO object at 0x7fef1170b348>\n",
      "From Images_0/9/9999909.jpg 1082012 images loaded\n",
      "Processing file data/Images_1.zip\n",
      "Cannot process image: Images_1/13/4515613.jpg cannot identify image file <_io.BytesIO object at 0x7fef41621dc8>\n",
      "Cannot process image: Images_1/14/9322814.jpg cannot identify image file <_io.BytesIO object at 0x7fef41621dc8>\n",
      "From Images_1/19/9999919.jpg 1082331 images loaded\n",
      "Processing file data/Images_2.zip\n",
      "From Images_2/29/9999829.jpg 1082285 images loaded\n",
      "Processing file data/Images_3.zip\n",
      "Cannot process image: Images_3/36/10684636.jpg cannot identify image file <_io.BytesIO object at 0x7fef31761048>\n",
      "From Images_3/39/9999939.jpg 1082022 images loaded\n",
      "Processing file data/Images_4.zip\n",
      "Cannot process image: Images_4/41/12953041.jpg cannot identify image file <_io.BytesIO object at 0x7fef3d2bedc8>\n",
      "Cannot process image: Images_4/41/13717141.jpg cannot identify image file <_io.BytesIO object at 0x7fef3d2bedc8>\n",
      "From Images_4/49/9999749.jpg 1082644 images loaded\n",
      "Processing file data/Images_5.zip\n",
      "Cannot process image: Images_5/54/9126754.jpg cannot identify image file <_io.BytesIO object at 0x7fef8e7eac48>\n",
      "From Images_5/59/9999959.jpg 1082535 images loaded\n",
      "Processing file data/Images_6.zip\n",
      "Cannot process image: Images_6/61/12961761.jpg cannot identify image file <_io.BytesIO object at 0x7fef8e7aabe8>\n",
      "Cannot process image: Images_6/67/651267.jpg cannot identify image file <_io.BytesIO object at 0x7fef8e7aabe8>\n",
      "From Images_6/69/9999969.jpg 1082498 images loaded\n",
      "Processing file data/Images_7.zip\n",
      "Cannot process image: Images_7/74/11775274.jpg cannot identify image file <_io.BytesIO object at 0x7fef8e7bae88>\n",
      "From Images_7/79/9999979.jpg 1082865 images loaded\n",
      "Processing file data/Images_8.zip\n",
      "Cannot process image: Images_8/81/14366881.jpg cannot identify image file <_io.BytesIO object at 0x7fef8e7c71c8>\n",
      "Cannot process image: Images_8/81/9651181.jpg cannot identify image file <_io.BytesIO object at 0x7fef8e7c71c8>\n",
      "Cannot process image: Images_8/84/9943584.jpg cannot identify image file <_io.BytesIO object at 0x7fef8e7c71c8>\n",
      "From Images_8/89/999989.jpg 1082611 images loaded\n",
      "Processing file data/Images_9.zip\n",
      "Cannot process image: Images_9/95/13949795.jpg cannot identify image file <_io.BytesIO object at 0x7fef8e7fd588>\n",
      "From Images_9/99/999999.jpg 1082514 images loaded\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    zipfilename = 'data/Images_' + str(i) + '.zip'\n",
    "    csvfilename = 'processed_data/hashes' + str(i) + '.csv'\n",
    "    print('Processing file %s' % zipfilename)\n",
    "    create_image_hashes(zipfilename, csvfilename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example of usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "e7e7ff21b1b131ff\n",
      "ff00ff0080ff00ff\n",
      "False\n",
      "20\n"
     ]
    }
   ],
   "source": [
    "from PIL import Image\n",
    "import imagehash\n",
    "hash = imagehash.average_hash(Image.open('test.png'))\n",
    "print(hash)\n",
    "otherhash = imagehash.average_hash(Image.open('other.png'))\n",
    "print(otherhash)\n",
    "print(hash == otherhash)\n",
    "print(imagehash.hex_to_hash(str(hash)) - otherhash)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}