mining-openai/mining-facebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "424dad2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "from tqdm import tqdm\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "20f6ee01",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = []\n",
    "\n",
    "with open('ada-002-facebook.jsonl') as fopen:\n",
    "    for x in fopen:\n",
    "        data.append(json.loads(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "47d96b98",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "192885"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "033fecab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18602215170"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "math.comb(len(data),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0ed02911",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'v': [-0.033773023635149,\n",
       "  -0.03641154244542122,\n",
       "  -0.011886521242558956,\n",
       "  -0.03778357058763504,\n",
       "  -0.037546105682849884,\n",
       "  0.02902369201183319,\n",
       "  -0.0076780859380960464,\n",
       "  0.018746666610240936,\n",
       "  -0.013258550316095352,\n",
       "  -0.035197824239730835,\n",
       "  -0.010758555494248867,\n",
       "  0.015554061159491539,\n",
       "  -0.0008855524356476963,\n",
       "  -0.019076481461524963,\n",
       "  0.006744710262864828,\n",
       "  -0.008555392734706402,\n",
       "  0.025065915659070015,\n",
       "  -0.005042866803705692,\n",
       "  0.031398359686136246,\n",
       "  -0.007137190084904432,\n",
       "  0.004251311533153057,\n",
       "  0.009716341271996498,\n",
       "  0.012374646961688995,\n",
       "  0.005098935216665268,\n",
       "  -0.002503293566405773,\n",
       "  -0.01372029073536396,\n",
       "  0.0069393012672662735,\n",
       "  -0.017941920086741447,\n",
       "  -0.02440628781914711,\n",
       "  0.009063308127224445,\n",
       "  0.02474929392337799,\n",
       "  0.012110795825719833,\n",
       "  0.0031431340612471104,\n",
       "  0.011193910613656044,\n",
       "  -0.019010519608855247,\n",
       "  -0.01094325166195631,\n",
       "  0.015474905259907246,\n",
       "  0.016042185947299004,\n",
       "  -0.013667521066963673,\n",
       "  0.010626628994941711,\n",
       "  0.011695228517055511,\n",
       "  -0.015461713075637817,\n",
       "  0.0013373985420912504,\n",
       "  -0.02551446482539177,\n",
       "  -0.024261169135570526,\n",
       "  0.023179376497864723,\n",
       "  -0.006899723317474127,\n",
       "  -0.019089674577116966,\n",
       "  -0.008693914860486984,\n",
       "  0.020699169486761093,\n",
       "  0.01841685175895691,\n",
       "  0.02343003638088703,\n",
       "  -0.02042212523519993,\n",
       "  -0.019815266132354736,\n",
       "  -0.010527685284614563,\n",
       "  0.013271743431687355,\n",
       "  -0.008364100940525532,\n",
       "  -0.00039454083889722824,\n",
       "  0.018113423138856888,\n",
       "  -0.02679414488375187,\n",
       "  0.019762495532631874,\n",
       "  0.002369718626141548,\n",
       "  -0.028416832908988,\n",
       "  0.003426774637773633,\n",
       "  0.005956453271210194,\n",
       "  -0.003423476591706276,\n",
       "  -0.006813971325755119,\n",
       "  0.00944589264690876,\n",
       "  -0.018496008589863777,\n",
       "  -0.0198812298476696,\n",
       "  0.04184688627719879,\n",
       "  0.0209498293697834,\n",
       "  -0.012810003012418747,\n",
       "  -0.0030326461419463158,\n",
       "  0.01641157828271389,\n",
       "  -0.016490735113620758,\n",
       "  0.005755266174674034,\n",
       "  -0.0013489420525729656,\n",
       "  -0.011728210374712944,\n",
       "  0.009313967078924179,\n",
       "  0.02730865590274334,\n",
       "  -0.003007909981533885,\n",
       "  -0.042031582444906235,\n",
       "  0.02341684326529503,\n",
       "  0.012572536244988441,\n",
       "  0.006866741925477982,\n",
       "  0.0028792822267860174,\n",
       "  0.0021833733189851046,\n",
       "  0.007051438093185425,\n",
       "  -0.03153028339147568,\n",
       "  0.015118705108761787,\n",
       "  -0.005811335053294897,\n",
       "  0.0397624596953392,\n",
       "  0.01134562585502863,\n",
       "  -0.003235482145100832,\n",
       "  0.016147727146744728,\n",
       "  -0.014195224270224571,\n",
       "  -0.0025659583043307066,\n",
       "  0.014643772505223751,\n",
       "  -0.033139780163764954,\n",
       "  0.011404992081224918,\n",
       "  -0.009736130014061928,\n",
       "  0.002467013895511627,\n",
       "  -0.007506582420319319,\n",
       "  -0.019841652363538742,\n",
       "  -0.017427409067749977,\n",
       "  0.00119145552162081,\n",
       "  -0.005316612776368856,\n",
       "  0.02986801788210869,\n",
       "  -0.008984152227640152,\n",
       "  -0.010349584743380547,\n",
       "  0.048443183302879333,\n",
       "  0.010211062617599964,\n",
       "  -0.040105465799570084,\n",
       "  -0.017889149487018585,\n",
       "  -0.02568596787750721,\n",
       "  0.026530293747782707,\n",
       "  -0.0139973359182477,\n",
       "  -0.02234824374318123,\n",
       "  -0.01577833481132984,\n",
       "  -0.010501299984753132,\n",
       "  0.01519786100834608,\n",
       "  0.039393067359924316,\n",
       "  -0.008047478273510933,\n",
       "  0.018904978409409523,\n",
       "  0.0190237108618021,\n",
       "  0.010554070584475994,\n",
       "  -0.005333103705197573,\n",
       "  -0.013707098551094532,\n",
       "  0.02688649296760559,\n",
       "  0.03999992460012436,\n",
       "  0.0018205770757049322,\n",
       "  0.013238761574029922,\n",
       "  0.008792859502136707,\n",
       "  -0.019802074879407883,\n",
       "  0.01893136277794838,\n",
       "  0.002971630310639739,\n",
       "  0.006846953183412552,\n",
       "  -0.021345606073737144,\n",
       "  -0.021319221705198288,\n",
       "  0.01606857217848301,\n",
       "  0.0397360734641552,\n",
       "  -0.009439296089112759,\n",
       "  -0.0006476736161857843,\n",
       "  0.002447224920615554,\n",
       "  0.02183373272418976,\n",
       "  0.023693887516856194,\n",
       "  0.022044813260436058,\n",
       "  -0.02274402044713497,\n",
       "  -9.502785542281345e-05,\n",
       "  0.013172798790037632,\n",
       "  -0.012559343129396439,\n",
       "  -0.019841652363538742,\n",
       "  0.0011015810305252671,\n",
       "  -0.008218982256948948,\n",
       "  0.02868068590760231,\n",
       "  -0.000725592311937362,\n",
       "  0.011530322022736073,\n",
       "  -0.03786272555589676,\n",
       "  0.007150382734835148,\n",
       "  0.0209498293697834,\n",
       "  0.025197843089699745,\n",
       "  -0.002597290789708495,\n",
       "  -0.005507905501872301,\n",
       "  0.012025043368339539,\n",
       "  0.03477565944194794,\n",
       "  0.013238761574029922,\n",
       "  -0.0019558011554181576,\n",
       "  -0.00250494247302413,\n",
       "  0.012968313880264759,\n",
       "  0.0069920714013278484,\n",
       "  0.02291552536189556,\n",
       "  -0.04366746544837952,\n",
       "  0.01765168271958828,\n",
       "  -0.007420830428600311,\n",
       "  0.007810011971741915,\n",
       "  -0.010831114836037159,\n",
       "  0.009498663246631622,\n",
       "  -0.01568598672747612,\n",
       "  -0.018350889906287193,\n",
       "  -0.016266461461782455,\n",
       "  0.03897090256214142,\n",
       "  0.020408933982253075,\n",
       "  0.02675456739962101,\n",
       "  -0.029551396146416664,\n",
       "  -0.005277035292237997,\n",
       "  -0.002636868506669998,\n",
       "  -0.020184658467769623,\n",
       "  0.019076481461524963,\n",
       "  -0.004584424197673798,\n",
       "  0.001711738295853138,\n",
       "  0.017559334635734558,\n",
       "  -0.008568585850298405,\n",
       "  -0.016899704933166504,\n",
       "  -0.6184685230255127,\n",
       "  -0.0028430025558918715,\n",
       "  0.006424790248274803,\n",
       "  0.025976205244660378,\n",
       "  0.005906980950385332,\n",
       "  0.022374628111720085,\n",
       "  0.0003656820335891098,\n",
       "  -0.010976232588291168,\n",
       "  -0.02240101434290409,\n",
       "  -0.023601539433002472,\n",
       "  -0.038997288793325424,\n",
       "  0.024168821051716805,\n",
       "  0.030448492616415024,\n",
       "  -0.011154333129525185,\n",
       "  0.0026962351985275745,\n",
       "  -0.024907605722546577,\n",
       "  -0.0021322520915418863,\n",
       "  -0.007935341447591782,\n",
       "  -0.0034894393756985664,\n",
       "  -0.003997353836894035,\n",
       "  0.0017282289918512106,\n",
       "  0.01143137738108635,\n",
       "  -0.0015723914839327335,\n",
       "  -0.0009506908245384693,\n",
       "  -0.007770434021949768,\n",
       "  -0.008608163334429264,\n",
       "  -0.007176767569035292,\n",
       "  -0.018667511641979218,\n",
       "  0.0057981424033641815,\n",
       "  0.014353535138070583,\n",
       "  -0.026807337999343872,\n",
       "  0.041134487837553024,\n",
       "  0.009155656211078167,\n",
       "  0.032849542796611786,\n",
       "  0.062374554574489594,\n",
       "  -0.005428749602288008,\n",
       "  -0.008139826357364655,\n",
       "  0.010949848219752312,\n",
       "  -0.004706455860286951,\n",
       "  0.02303425781428814,\n",
       "  -0.030712343752384186,\n",
       "  -0.012928735464811325,\n",
       "  -0.019986771047115326,\n",
       "  -0.02406327985227108,\n",
       "  0.0032882525119930506,\n",
       "  0.006698536220937967,\n",
       "  0.007579141762107611,\n",
       "  -0.008027689531445503,\n",
       "  -0.021530302241444588,\n",
       "  0.012084410525858402,\n",
       "  -0.0006550944526679814,\n",
       "  0.012493380345404148,\n",
       "  -0.0033674079459160566,\n",
       "  -0.0006126307998783886,\n",
       "  0.0104683181270957,\n",
       "  0.015435327775776386,\n",
       "  -0.004567933268845081,\n",
       "  -0.02675456739962101,\n",
       "  0.015593638643622398,\n",
       "  0.01358836516737938,\n",
       "  0.002503293566405773,\n",
       "  -0.009907633066177368,\n",
       "  -0.0126121137291193,\n",
       "  0.0015946540515869856,\n",
       "  -0.021121332421898842,\n",
       "  0.016556696966290474,\n",
       "  -0.027361426502466202,\n",
       "  -0.010481511242687702,\n",
       "  0.010402355343103409,\n",
       "  -0.027335042133927345,\n",
       "  -0.016292845830321312,\n",
       "  0.0241556279361248,\n",
       "  -0.010620033368468285,\n",
       "  0.02610813081264496,\n",
       "  -0.005623340606689453,\n",
       "  0.005181388929486275,\n",
       "  0.009558029472827911,\n",
       "  4.676669050240889e-06,\n",
       "  0.00015645584790036082,\n",
       "  0.0023565259762108326,\n",
       "  -0.007189960218966007,\n",
       "  -0.0048086983151733875,\n",
       "  -0.017163556069135666,\n",
       "  -0.009234811179339886,\n",
       "  0.021728191524744034,\n",
       "  -0.002191618550568819,\n",
       "  -0.0036972227972000837,\n",
       "  -0.02496037632226944,\n",
       "  0.003466352354735136,\n",
       "  -0.00799470767378807,\n",
       "  0.010507895611226559,\n",
       "  0.02654348686337471,\n",
       "  0.0051286183297634125,\n",
       "  0.0026995332445949316,\n",
       "  -8.71432275744155e-05,\n",
       "  0.007572545204311609,\n",
       "  -0.013410265557467937,\n",
       "  -0.003209096845239401,\n",
       "  0.013627942651510239,\n",
       "  0.0042381188832223415,\n",
       "  -0.021147718653082848,\n",
       "  -0.004567933268845081,\n",
       "  0.030237410217523575,\n",
       "  0.003360811620950699,\n",
       "  0.014960394240915775,\n",
       "  0.005580464377999306,\n",
       "  -0.03369386866688728,\n",
       "  -0.023799428716301918,\n",
       "  0.04427432268857956,\n",
       "  -0.01812661625444889,\n",
       "  0.0025873961858451366,\n",
       "  0.006543523631989956,\n",
       "  0.006362125277519226,\n",
       "  -0.005910279229283333,\n",
       "  0.024393094703555107,\n",
       "  -0.031820520758628845,\n",
       "  0.017335060983896255,\n",
       "  0.021477533504366875,\n",
       "  0.013476228341460228,\n",
       "  -0.006576505023986101,\n",
       "  0.001924468670040369,\n",
       "  0.010375970043241978,\n",
       "  0.025633197277784348,\n",
       "  -0.014327149838209152,\n",
       "  0.010567262768745422,\n",
       "  0.03751971945166588,\n",
       "  -0.008133230730891228,\n",
       "  -0.029947174713015556,\n",
       "  -0.0004324695037212223,\n",
       "  -0.0018007882172241807,\n",
       "  -0.006477560382336378,\n",
       "  -0.049815211445093155,\n",
       "  0.0007899061893112957,\n",
       "  -0.02833767794072628,\n",
       "  0.004422815050929785,\n",
       "  -0.031002581119537354,\n",
       "  0.05055399611592293,\n",
       "  -0.00019768267520703375,\n",
       "  0.018707089126110077,\n",
       "  -0.025079108774662018,\n",
       "  -0.0011452814796939492,\n",
       "  -0.0139973359182477,\n",
       "  -0.02234824374318123,\n",
       "  0.015224246308207512,\n",
       "  -0.014287572354078293,\n",
       "  -0.029076462611556053,\n",
       "  -0.0375988744199276,\n",
       "  -0.008588374592363834,\n",
       "  0.0054320478811860085,\n",
       "  0.015039550140500069,\n",
       "  0.0021949168294668198,\n",
       "  -0.006012521684169769,\n",
       "  -0.0027753906324505806,\n",
       "  -0.010507895611226559,\n",
       "  0.015382557176053524,\n",
       "  -0.02217674069106579,\n",
       "  0.012777021154761314,\n",
       "  -0.03253292292356491,\n",
       "  -0.03440626710653305,\n",
       "  0.0021932676900178194,\n",
       "  0.004271100275218487,\n",
       "  0.02064639888703823,\n",
       "  -0.0252110343426466,\n",
       "  0.015870682895183563,\n",
       "  0.016767779365181923,\n",
       "  -0.014498653821647167,\n",
       "  -0.0033838986419141293,\n",
       "  -0.0008340189233422279,\n",
       "  0.0018073845421895385,\n",
       "  -0.011193910613656044,\n",
       "  0.026174094527959824,\n",
       "  -0.026517100632190704,\n",
       "  -0.01416883897036314,\n",
       "  0.01576514169573784,\n",
       "  -0.03303423896431923,\n",
       "  -0.0045052687637507915,\n",
       "  0.01970972679555416,\n",
       "  0.0062334975227713585,\n",
       "  -0.007282308302819729,\n",
       "  -0.0011254926212131977,\n",
       "  -0.0016416526632383466,\n",
       "  0.02572554536163807,\n",
       "  -0.008700511418282986,\n",
       "  -0.004775716923177242,\n",
       "  0.012664884328842163,\n",
       "  0.0007734154351055622,\n",
       "  -0.014221609570086002,\n",
       "  0.035065896809101105,\n",
       "  -0.03295508399605751,\n",
       "  0.009386526420712471,\n",
       "  -0.01523743849247694,\n",
       "  0.04266482964158058,\n",
       "  0.020408933982253075,\n",
       "  -0.012368051335215569,\n",
       "  -0.0019937297329306602,\n",
       "  0.024050086736679077,\n",
       "  0.00980868935585022,\n",
       "  0.013357494957745075,\n",
       "  -0.006523734424263239,\n",
       "  -0.006078484933823347,\n",
       "  -0.008080460131168365,\n",
       "  -0.0051912833005189896,\n",
       "  0.004739437252283096,\n",
       "  0.0023383861407637596,\n",
       "  0.029841633513569832,\n",
       "  -0.028311293572187424,\n",
       "  -0.006131255067884922,\n",
       "  -0.016556696966290474,\n",
       "  0.02389177680015564,\n",
       "  0.0028941240161657333,\n",
       "  -0.0002430322056170553,\n",
       "  -0.03300785273313522,\n",
       "  0.013476228341460228,\n",
       "  -0.008258559741079807,\n",
       "  0.039129216223955154,\n",
       "  0.005514501594007015,\n",
       "  0.008324522525072098,\n",
       "  0.031081736087799072,\n",
       "  -0.0022707742173224688,\n",
       "  -0.014999971725046635,\n",
       "  -0.005616744048893452,\n",
       "  -0.010626628994941711,\n",
       "  -0.007335078902542591,\n",
       "  -0.0028545460663735867,\n",
       "  0.01833769679069519,\n",
       "  -0.0008476237417198718,\n",
       "  0.007579141762107611,\n",
       "  -0.017928726971149445,\n",
       "  0.00517149455845356,\n",
       "  -0.028786227107048035,\n",
       "  -0.02444586530327797,\n",
       "  0.01970972679555416,\n",
       "  0.00799470767378807,\n",
       "  0.03295508399605751,\n",
       "  0.0016837039729580283,\n",
       "  0.010587051510810852,\n",
       "  0.02248016931116581,\n",
       "  -0.0211609099060297,\n",
       "  0.038575127720832825,\n",
       "  -0.008634548634290695,\n",
       "  -0.019010519608855247,\n",
       "  0.01461738720536232,\n",
       "  0.01692608930170536,\n",
       "  0.0009811987401917577,\n",
       "  0.004934027791023254,\n",
       "  0.023139799013733864,\n",
       "  0.02291552536189556,\n",
       "  -0.008937978185713291,\n",
       "  -0.040158238261938095,\n",
       "  0.025237420573830605,\n",
       "  -0.027889130637049675,\n",
       "  0.022374628111720085,\n",
       "  -0.02167542092502117,\n",
       "  0.022836368530988693,\n",
       "  0.015593638643622398,\n",
       "  -0.019947191700339317,\n",
       "  -0.014182032085955143,\n",
       "  0.007163574919104576,\n",
       "  0.033482786267995834,\n",
       "  -0.0037829745560884476,\n",
       "  0.0021141122560948133,\n",
       "  0.0076780859380960464,\n",
       "  0.004858170635998249,\n",
       "  -0.012401032261550426,\n",
       "  0.010032963007688522,\n",
       "  -0.0030491368379443884,\n",
       "  0.019551414996385574,\n",
       "  0.00500988494604826,\n",
       "  -0.016266461461782455,\n",
       "  0.0010265482123941183,\n",
       "  -0.025633197277784348,\n",
       "  -0.00046421418664976954,\n",
       "  0.005375979468226433,\n",
       "  0.003575191367417574,\n",
       "  0.008588374592363834,\n",
       "  -0.0032222894951701164,\n",
       "  0.0014998322585597634,\n",
       "  -0.001036442699842155,\n",
       "  0.013225569389760494,\n",
       "  -0.028152981773018837,\n",
       "  -0.003133239457383752,\n",
       "  -0.051899638026952744,\n",
       "  0.015606830827891827,\n",
       "  0.027546122670173645,\n",
       "  0.015356171876192093,\n",
       "  -0.026688605546951294,\n",
       "  -0.03701839968562126,\n",
       "  0.014366728253662586,\n",
       "  0.007110804785043001,\n",
       "  0.004234820604324341,\n",
       "  0.00842346716672182,\n",
       "  0.00889180414378643,\n",
       "  0.017585719004273415,\n",
       "  -0.005507905501872301,\n",
       "  -0.015369364991784096,\n",
       "  0.005989434663206339,\n",
       "  0.02538253925740719,\n",
       "  -0.027387812733650208,\n",
       "  0.05350913479924202,\n",
       "  0.012209739536046982,\n",
       "  0.01318599097430706,\n",
       "  -0.003144782967865467,\n",
       "  -0.002801775699481368,\n",
       "  -0.0019343631574884057,\n",
       "  0.01633242331445217,\n",
       "  -0.0023664203472435474,\n",
       "  -0.041688576340675354,\n",
       "  0.007381252944469452,\n",
       "  -0.005606849677860737,\n",
       "  0.001057880581356585,\n",
       "  -0.002892474876716733,\n",
       "  -0.011233488097786903,\n",
       "  -0.01782318577170372,\n",
       "  0.014366728253662586,\n",
       "  0.012849580496549606,\n",
       "  0.02034297026693821,\n",
       "  -0.01627965271472931,\n",
       "  -0.010666207410395145,\n",
       "  -0.0016812303801998496,\n",
       "  -0.0005767634138464928,\n",
       "  -0.00895776692777872,\n",
       "  -0.023073835298419,\n",
       "  -0.014643772505223751,\n",
       "  0.013759869150817394,\n",
       "  0.10110799223184586,\n",
       "  0.009755918756127357,\n",
       "  -0.012447206303477287,\n",
       "  0.007829801179468632,\n",
       "  -0.005296824034303427,\n",
       "  -0.024261169135570526,\n",
       "  -0.018865400925278664,\n",
       "  -0.022664865478873253,\n",
       "  0.0062697771936655045,\n",
       "  -0.009208425879478455,\n",
       "  0.01744060032069683,\n",
       "  -0.024353517219424248,\n",
       "  0.014986779540777206,\n",
       "  -0.011312643997371197,\n",
       "  0.018825823441147804,\n",
       "  -0.02055405080318451,\n",
       "  -0.0020135187078267336,\n",
       "  -0.004871362820267677,\n",
       "  -0.0011997009860351682,\n",
       "  0.014841660857200623,\n",
       "  -0.011952484026551247,\n",
       "  0.004027037415653467,\n",
       "  -0.0075461603701114655,\n",
       "  0.0066358717158436775,\n",
       "  -0.004366746637970209,\n",
       "  -0.0013497666222974658,\n",
       "  0.015791527926921844,\n",
       "  0.03902367502450943,\n",
       "  -0.035488061606884,\n",
       "  -0.024010509252548218,\n",
       "  -0.01791553385555744,\n",
       "  0.010310007259249687,\n",
       "  0.012711058370769024,\n",
       "  0.010270429775118828,\n",
       "  -0.01679416373372078,\n",
       "  0.010382566601037979,\n",
       "  0.0058509125374257565,\n",
       "  0.01406329870223999,\n",
       "  0.013667521066963673,\n",
       "  -0.02671498991549015,\n",
       "  0.028311293572187424,\n",
       "  0.028021056205034256,\n",
       "  0.005474924109876156,\n",
       "  -0.003766483860090375,\n",
       "  0.0015773387858644128,\n",
       "  -0.027968285605311394,\n",
       "  0.0024307342246174812,\n",
       "  0.0029765774961560965,\n",
       "  -0.013865409418940544,\n",
       "  0.005356190726161003,\n",
       "  0.05878617241978645,\n",
       "  0.01734825223684311,\n",
       "  -0.0060389069840312,\n",
       "  -0.01151712890714407,\n",
       "  -0.013601558282971382,\n",
       "  0.004726244602352381,\n",
       "  -0.040026310831308365,\n",
       "  -0.0069920714013278484,\n",
       "  0.011246681213378906,\n",
       "  0.0329023152589798,\n",
       "  -0.03005271404981613,\n",
       "  -0.006418193690478802,\n",
       "  0.022770406678318977,\n",
       "  -0.0073944455944001675,\n",
       "  -0.019947191700339317,\n",
       "  -0.008852226659655571,\n",
       "  -0.003875322639942169,\n",
       "  0.0008937978418543935,\n",
       "  -0.026688605546951294,\n",
       "  -0.006378616206347942,\n",
       "  -0.010422144085168839,\n",
       "  0.010369373485445976,\n",
       "  -0.03862789645791054,\n",
       "  -0.027704434469342232,\n",
       "  0.012130584567785263,\n",
       "  -0.001769455848261714,\n",
       "  -0.0017430706648156047,\n",
       "  -0.020277006551623344,\n",
       "  0.00016975149628706276,\n",
       "  0.01203163992613554,\n",
       "  -0.00936014112085104,\n",
       "  -0.01572556421160698,\n",
       "  -0.009848266839981079,\n",
       "  -0.015738757327198982,\n",
       "  -0.01875985972583294,\n",
       "  0.008548797108232975,\n",
       "  0.02585747092962265,\n",
       "  -0.00753296772018075,\n",
       "  0.009076500311493874,\n",
       "  -0.009003940969705582,\n",
       "  0.021121332421898842,\n",
       "  -0.000751152983866632,\n",
       "  0.015870682895183563,\n",
       "  -0.01602899469435215,\n",
       "  -0.010850903578102589,\n",
       "  -0.0074736010283231735,\n",
       "  0.0070910160429775715,\n",
       "  -0.01718994230031967,\n",
       "  -0.00030281112412922084,\n",
       "  0.009241407737135887,\n",
       "  0.01348282489925623,\n",
       "  -0.025659583508968353,\n",
       "  -0.01761210523545742,\n",
       "  -0.007282308302819729,\n",
       "  0.00129287363961339,\n",
       "  -0.002018465893343091,\n",
       "  0.002046500099822879,\n",
       "  0.018153000622987747,\n",
       "  0.009373333305120468,\n",
       "  -0.010936655104160309,\n",
       "  0.036306001245975494,\n",
       "  -0.002694586059078574,\n",
       "  0.037123940885066986,\n",
       "  0.01829811930656433,\n",
       "  -0.0009036922710947692,\n",
       "  0.0188126303255558,\n",
       "  -0.00543864443898201,\n",
       "  0.04005269706249237,\n",
       "  0.015263823792338371,\n",
       "  -0.008317926898598671,\n",
       "  -0.001650722580961883,\n",
       "  -0.01593664661049843,\n",
       "  0.02240101434290409,\n",
       "  0.01932714134454727,\n",
       "  -0.013047468848526478,\n",
       "  0.01265169121325016,\n",
       "  0.017875956371426582,\n",
       "  -0.02846960350871086,\n",
       "  0.01576514169573784,\n",
       "  -0.009802092798054218,\n",
       "  -0.008726896718144417,\n",
       "  0.008113441057503223,\n",
       "  -0.021701807156205177,\n",
       "  -0.0052077737636864185,\n",
       "  -0.040580399334430695,\n",
       "  0.002683042548596859,\n",
       "  -0.002686340594664216,\n",
       "  -0.013403668999671936,\n",
       "  0.010903673246502876,\n",
       "  -0.015342979691922665,\n",
       "  -0.0196569561958313,\n",
       "  0.009307370521128178,\n",
       "  0.01982845924794674,\n",
       "  -0.002097621327266097,\n",
       "  0.015079127624630928,\n",
       "  -0.03390495106577873,\n",
       "  -0.025949819013476372,\n",
       "  0.0011312643764540553,\n",
       "  -0.02709757536649704,\n",
       "  0.034089647233486176,\n",
       "  -0.004053422715514898,\n",
       "  -0.006678747478872538,\n",
       "  -0.014116068370640278,\n",
       "  0.011273066513240337,\n",
       "  0.01624007523059845,\n",
       "  -0.05290227755904198,\n",
       "  -0.017203133553266525,\n",
       "  -0.004277696367353201,\n",
       "  0.008304733783006668,\n",
       "  0.014300765469670296,\n",
       "  0.036780934780836105,\n",
       "  -0.019907614216208458,\n",
       "  0.005695899948477745,\n",
       "  -0.02324534021317959,\n",
       "  0.004277696367353201,\n",
       "  0.021121332421898842,\n",
       "  -0.03005271404981613,\n",
       "  -0.004999990575015545,\n",
       "  -0.028865382075309753,\n",
       "  -0.000801449699793011,\n",
       "  0.0035455080214887857,\n",
       "  0.02696564979851246,\n",
       "  0.02778358943760395,\n",
       "  0.009353544563055038,\n",
       "  -0.0012087707873433828,\n",
       "  0.025290191173553467,\n",
       "  -0.011437973938882351,\n",
       "  0.00626318110153079,\n",
       "  -0.01151712890714407,\n",
       "  -0.017889149487018585,\n",
       "  -0.005775055382400751,\n",
       "  -0.014999971725046635,\n",
       "  -0.014445883221924305,\n",
       "  -0.00022612919565290213,\n",
       "  -0.008120037615299225,\n",
       "  -0.002028360264375806,\n",
       "  0.03060680255293846,\n",
       "  -0.006451175548136234,\n",
       "  0.014881239272654057,\n",
       "  0.008390485309064388,\n",
       "  0.002780337817966938,\n",
       "  -0.017875956371426582,\n",
       "  0.01412926148623228,\n",
       "  -0.00035393238067626953,\n",
       "  0.02461736835539341,\n",
       "  -0.024670138955116272,\n",
       "  -0.01102900318801403,\n",
       "  0.007156978826969862,\n",
       "  0.006724921520799398,\n",
       "  0.005755266174674034,\n",
       "  0.0048812576569616795,\n",
       "  -0.005326507147401571,\n",
       "  0.00275230361148715,\n",
       "  0.011081773787736893,\n",
       "  0.007123997434973717,\n",
       "  -0.007664893288165331,\n",
       "  -0.03469650447368622,\n",
       "  -0.002790232188999653,\n",
       "  0.013245358131825924,\n",
       "  -0.025738738477230072,\n",
       "  -0.010369373485445976,\n",
       "  -0.007467004470527172,\n",
       "  -0.024722909554839134,\n",
       "  -0.04641152173280716,\n",
       "  0.0007429075776599348,\n",
       "  -0.008403678424656391,\n",
       "  -0.03862789645791054,\n",
       "  0.01782318577170372,\n",
       "  -0.013852217234671116,\n",
       "  -0.03683370351791382,\n",
       "  0.010079137049615383,\n",
       "  0.004957114811986685,\n",
       "  0.02786274440586567,\n",
       "  0.00299801561050117,\n",
       "  0.005369383376091719,\n",
       "  0.013067258521914482,\n",
       "  -0.01562002394348383,\n",
       "  -0.005382575560361147,\n",
       "  0.017717644572257996,\n",
       "  -0.004835083149373531,\n",
       "  0.013773061335086823,\n",
       "  0.005953154992312193,\n",
       "  0.011668844148516655,\n",
       "  -0.013251954689621925,\n",
       "  -0.019604185596108437,\n",
       "  0.035276979207992554,\n",
       "  0.01465696468949318,\n",
       "  -0.023139799013733864,\n",
       "  -0.04163580760359764,\n",
       "  0.006540225353091955,\n",
       "  0.01897094026207924,\n",
       "  0.011470954865217209,\n",
       "  0.009525048546493053,\n",
       "  0.0077770305797457695,\n",
       "  -0.018060652539134026,\n",
       "  -0.013298128731548786,\n",
       "  0.00624009408056736,\n",
       "  -0.015699179843068123,\n",
       "  -0.026767760515213013,\n",
       "  -0.02337726578116417,\n",
       "  -0.02757250890135765,\n",
       "  0.006959090009331703,\n",
       "  -0.03166221082210541,\n",
       "  0.014525039121508598,\n",
       "  0.004465690813958645,\n",
       "  0.004877959378063679,\n",
       "  -0.014182032085955143,\n",
       "  -0.00835750438272953,\n",
       "  -0.030791498720645905,\n",
       "  0.0008063969435170293,\n",
       "  -0.006945897359400988,\n",
       "  0.014630579389631748,\n",
       "  0.0046569835394620895,\n",
       "  -0.004000652115792036,\n",
       "  -0.01209760271012783,\n",
       "  -0.011780980974435806,\n",
       "  -0.01748017780482769,\n",
       "  -0.011576496064662933,\n",
       "  0.0016705114394426346,\n",
       "  0.023311302065849304,\n",
       "  -0.008726896718144417,\n",
       "  0.011959080584347248,\n",
       "  0.0030458387918770313,\n",
       "  0.011378606781363487,\n",
       "  0.0303429514169693,\n",
       "  -0.008608163334429264,\n",
       "  0.00182717340067029,\n",
       "  -0.022625287994742393,\n",
       "  0.002961735939607024,\n",
       "  -0.0065369270741939545,\n",
       "  0.008403678424656391,\n",
       "  0.031134506687521935,\n",
       "  0.004383237101137638,\n",
       "  0.005715688690543175,\n",
       "  -0.02517145685851574,\n",
       "  -0.009122674353420734,\n",
       "  0.005052761174738407,\n",
       "  0.006678747478872538,\n",
       "  -0.014313957653939724,\n",
       "  -0.03184690698981285,\n",
       "  0.0010842657648026943,\n",
       "  0.002559361979365349,\n",
       "  -0.005329805426299572,\n",
       "  -0.015224246308207512,\n",
       "  -0.016530312597751617,\n",
       "  -0.002434032503515482,\n",
       "  0.0019228196470066905,\n",
       "  0.019815266132354736,\n",
       "  -0.029630551114678383,\n",
       "  0.004884555470198393,\n",
       "  -0.014116068370640278,\n",
       "  -0.0019475558074191213,\n",
       "  -0.019498644396662712,\n",
       "  -0.003095310879871249,\n",
       "  0.008100248873233795,\n",
       "  -0.006629275158047676,\n",
       "  0.023865390568971634,\n",
       "  -0.028179368004202843,\n",
       "  -0.02671498991549015,\n",
       "  -0.022387821227312088,\n",
       "  0.01885220780968666,\n",
       "  0.011866732500493526,\n",
       "  -0.008608163334429264,\n",
       "  -0.03651708364486694,\n",
       "  0.025448501110076904,\n",
       "  4.8286932724295184e-05,\n",
       "  0.004581125918775797,\n",
       "  -0.008522411808371544,\n",
       "  -0.021094948053359985,\n",
       "  -0.0010743713937699795,\n",
       "  -0.007189960218966007,\n",
       "  0.0006703483522869647,\n",
       "  0.007499985862523317,\n",
       "  -0.009076500311493874,\n",
       "  -0.006022416055202484,\n",
       "  -0.016728201881051064,\n",
       "  0.0029881210066378117,\n",
       "  0.0010587051510810852,\n",
       "  -0.03923475742340088,\n",
       "  -0.006503945682197809,\n",
       "  -0.012559343129396439,\n",
       "  0.03269123286008835,\n",
       "  -0.01361475046724081,\n",
       "  -0.004221627954393625,\n",
       "  0.0008191772503778338,\n",
       "  -0.02461736835539341,\n",
       "  -0.014999971725046635,\n",
       "  0.03379940986633301,\n",
       "  -0.016860127449035645,\n",
       "  -0.026992034167051315,\n",
       "  0.024076472967863083,\n",
       "  0.006048801355063915,\n",
       "  0.014472268521785736,\n",
       "  0.004000652115792036,\n",
       "  -0.02060682140290737,\n",
       "  0.007420830428600311,\n",
       "  -0.011873329058289528,\n",
       "  0.0017925428692251444,\n",
       "  -0.04846956580877304,\n",
       "  -0.012229529209434986,\n",
       "  -0.012176758609712124,\n",
       "  0.023878583684563637,\n",
       "  -0.007255923002958298,\n",
       "  -0.017242712900042534,\n",
       "  -0.029947174713015556,\n",
       "  -0.020738746970891953,\n",
       "  -0.05055399611592293,\n",
       "  -0.03274400159716606,\n",
       "  -0.013027680106461048,\n",
       "  0.029947174713015556,\n",
       "  0.02448544278740883,\n",
       "  0.011075177229940891,\n",
       "  0.0019673446658998728,\n",
       "  -0.01510551292449236,\n",
       "  0.01461738720536232,\n",
       "  0.010685996152460575,\n",
       "  -0.02141156978905201,\n",
       "  -0.0018040863797068596,\n",
       "  -0.0024571195244789124,\n",
       "  -0.01906328834593296,\n",
       "  0.017981497570872307,\n",
       "  0.005026375874876976,\n",
       "  -0.03218991309404373,\n",
       "  -0.01650392636656761,\n",
       "  0.00420843530446291,\n",
       "  -0.0072427308186888695,\n",
       "  0.017071207985281944,\n",
       "  0.005049462895840406,\n",
       "  -0.014485461637377739,\n",
       "  -0.014960394240915775,\n",
       "  0.014999971725046635,\n",
       "  -0.009709744714200497,\n",
       "  0.041055332869291306,\n",
       "  -0.004666877910494804,\n",
       "  0.02902369201183319,\n",
       "  -0.029419470578432083,\n",
       "  0.0067875864915549755,\n",
       "  0.00472294632345438,\n",
       "  -0.00576845882460475,\n",
       "  -0.006266478914767504,\n",
       "  0.0047724186442792416,\n",
       "  0.007922149263322353,\n",
       "  -0.0026154303923249245,\n",
       "  -0.0008026864961721003,\n",
       "  0.01513189822435379,\n",
       "  0.017110785469412804,\n",
       "  -0.030633188784122467,\n",
       "  -0.008812648244202137,\n",
       "  0.02530338242650032,\n",
       "  0.023667503148317337,\n",
       "  0.0037928689271211624,\n",
       "  0.021728191524744034,\n",
       "  -0.00013677003153134137,\n",
       "  -0.003796167206019163,\n",
       "  0.011668844148516655,\n",
       "  -0.008819244801998138,\n",
       "  -0.003967670723795891,\n",
       "  -0.022981487214565277,\n",
       "  0.02585747092962265,\n",
       "  -0.008601566776633263,\n",
       "  0.025118686258792877,\n",
       "  0.01748017780482769,\n",
       "  0.016424771398305893,\n",
       "  -0.0052803331054747105,\n",
       "  0.0018486113986000419,\n",
       "  0.0211609099060297,\n",
       "  -0.010131907649338245,\n",
       "  -0.009393122047185898,\n",
       "  0.024116050451993942,\n",
       "  0.007348271552473307,\n",
       "  -0.028364064171910286,\n",
       "  0.0037862726021558046,\n",
       "  -0.000530589371919632,\n",
       "  0.004172155633568764,\n",
       "  -0.014116068370640278,\n",
       "  -0.02346961386501789,\n",
       "  0.029287545010447502,\n",
       "  -0.02730865590274334,\n",
       "  -0.026649026200175285,\n",
       "  0.005408960860222578,\n",
       "  -0.006543523631989956,\n",
       "  -0.011134544387459755,\n",
       "  -0.0252110343426466,\n",
       "  -0.011424780823290348,\n",
       "  -0.005220966413617134,\n",
       "  0.017506564036011696,\n",
       "  0.2090761363506317,\n",
       "  -0.010890481062233448,\n",
       "  -0.03068595938384533,\n",
       "  0.016477541998028755,\n",
       "  -0.020738746970891953,\n",
       "  0.008924786001443863,\n",
       "  0.008205789141356945,\n",
       "  0.009419507347047329,\n",
       "  0.003133239457383752,\n",
       "  0.019802074879407883,\n",
       "  -0.0026154303923249245,\n",
       "  0.000592017371673137,\n",
       "  -0.004297485575079918,\n",
       "  0.0016111447475850582,\n",
       "  0.01139179989695549,\n",
       "  -0.03815296292304993,\n",
       "  -0.035065896809101105,\n",
       "  -0.0083377156406641,\n",
       "  -0.005267140455543995,\n",
       "  -0.0354616753757,\n",
       "  0.0026780953630805016,\n",
       "  0.00416885782033205,\n",
       "  0.0166754312813282,\n",
       "  -0.02158307284116745,\n",
       "  0.014485461637377739,\n",
       "  0.02751973830163479,\n",
       "  -0.003872024593874812,\n",
       "  0.0044063241221010685,\n",
       "  0.01654350571334362,\n",
       "  0.0003743396664503962,\n",
       "  -0.009525048546493053,\n",
       "  -0.03398410603404045,\n",
       "  -0.004445902071893215,\n",
       "  -0.024973567575216293,\n",
       "  0.008588374592363834,\n",
       "  0.0060389069840312,\n",
       "  0.027071189135313034,\n",
       "  -0.0038621299900114536,\n",
       "  0.007229538168758154,\n",
       "  0.013364091515541077,\n",
       "  0.00782320462167263,\n",
       "  -0.00784299336373806,\n",
       "  -0.003990757744759321,\n",
       "  -0.010659610852599144,\n",
       "  -0.022638481110334396,\n",
       "  0.04245374724268913,\n",
       "  ...],\n",
       " 'cleaned_text': 'haii mat mlm.. Noraty Lesty... aku x pernah bergambar... nanti tanye boss aku kalau dia onzz... boleh',\n",
       " 'raw_text': 'haii mat mlm.. Noraty Lesty... aku x pernah bergambar... nanti tanye boss aku kalau dia onzz... boleh'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ff164585",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors, texts = [], []\n",
    "\n",
    "for d in data:\n",
    "    vectors.append(d['v'])\n",
    "    texts.append(d['cleaned_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f3a5c420",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'haii mat mlm.. Noraty Lesty... aku x pernah bergambar... nanti tanye boss aku kalau dia onzz... boleh'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0]['cleaned_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "88d80634",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'haii mat mlm.. Noraty Lesty... aku x pernah bergambar... nanti tanye boss aku kalau dia onzz... boleh'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b904d1b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial import KDTree\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2c269e5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(192885, 1536)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat = np.array(vectors)\n",
    "concat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8536adf2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.55 s, sys: 24.1 ms, total: 2.57 s\n",
      "Wall time: 2.57 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "kd_tree = KDTree(concat, leafsize = 40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "665346e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.03377302, -0.03641154, -0.01188652, ..., -0.02255932,\n",
       "        0.00241424, -0.03153028])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "61511caa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 192885/192885 [2:04:28<00:00, 25.83it/s]  \n",
      "100%|██████████| 192885/192885 [00:00<00:00, 1150382.97it/s]\n"
     ]
    }
   ],
   "source": [
    "dist_list = []\n",
    "ind_list = []\n",
    "indices = []\n",
    "\n",
    "def compute_neighbors(args):\n",
    "    idx, x = args\n",
    "    dist, ind = kd_tree.query(x, k=len(concat),workers=50)\n",
    "    return idx, dist, ind\n",
    "\n",
    "with ThreadPoolExecutor(max_workers=30) as executor:\n",
    "    futures = list(tqdm(executor.map(compute_neighbors, enumerate(concat)), total=len(concat)))\n",
    "    \n",
    "    for future in tqdm(futures, total=len(concat)):\n",
    "        if future:\n",
    "            dist_list.append(future[1])\n",
    "            ind_list.append(future[2])\n",
    "            indices.append(future[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "42862c8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(192885, 192885, 192885)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dist_list),len(ind_list),len(indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aeb21649",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0b5b8485",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " 1,\n",
       " 2,\n",
       " 3,\n",
       " 4,\n",
       " 5,\n",
       " 6,\n",
       " 7,\n",
       " 8,\n",
       " 9,\n",
       " 10,\n",
       " 11,\n",
       " 12,\n",
       " 13,\n",
       " 14,\n",
       " 15,\n",
       " 16,\n",
       " 17,\n",
       " 18,\n",
       " 19,\n",
       " 20,\n",
       " 21,\n",
       " 22,\n",
       " 23,\n",
       " 24,\n",
       " 25,\n",
       " 26,\n",
       " 27,\n",
       " 28,\n",
       " 29,\n",
       " 30,\n",
       " 31,\n",
       " 32,\n",
       " 33,\n",
       " 34,\n",
       " 35,\n",
       " 36,\n",
       " 37,\n",
       " 38,\n",
       " 39,\n",
       " 40,\n",
       " 41,\n",
       " 42,\n",
       " 43,\n",
       " 44,\n",
       " 45,\n",
       " 46,\n",
       " 47,\n",
       " 48,\n",
       " 49,\n",
       " 50,\n",
       " 51,\n",
       " 52,\n",
       " 53,\n",
       " 54,\n",
       " 55,\n",
       " 56,\n",
       " 57,\n",
       " 58,\n",
       " 59,\n",
       " 60,\n",
       " 61,\n",
       " 62,\n",
       " 63,\n",
       " 64,\n",
       " 65,\n",
       " 66,\n",
       " 67,\n",
       " 68,\n",
       " 69,\n",
       " 70,\n",
       " 71,\n",
       " 72,\n",
       " 73,\n",
       " 74,\n",
       " 75,\n",
       " 76,\n",
       " 77,\n",
       " 78,\n",
       " 79,\n",
       " 80,\n",
       " 81,\n",
       " 82,\n",
       " 83,\n",
       " 84,\n",
       " 85,\n",
       " 86,\n",
       " 87,\n",
       " 88,\n",
       " 89,\n",
       " 90,\n",
       " 91,\n",
       " 92,\n",
       " 93,\n",
       " 94,\n",
       " 95,\n",
       " 96,\n",
       " 97,\n",
       " 98,\n",
       " 99,\n",
       " 100,\n",
       " 101,\n",
       " 102,\n",
       " 103,\n",
       " 104,\n",
       " 105,\n",
       " 106,\n",
       " 107,\n",
       " 108,\n",
       " 109,\n",
       " 110,\n",
       " 111,\n",
       " 112,\n",
       " 113,\n",
       " 114,\n",
       " 115,\n",
       " 116,\n",
       " 117,\n",
       " 118,\n",
       " 119,\n",
       " 120,\n",
       " 121,\n",
       " 122,\n",
       " 123,\n",
       " 124,\n",
       " 125,\n",
       " 126,\n",
       " 127,\n",
       " 128,\n",
       " 129,\n",
       " 130,\n",
       " 131,\n",
       " 132,\n",
       " 133,\n",
       " 134,\n",
       " 135,\n",
       " 136,\n",
       " 137,\n",
       " 138,\n",
       " 139,\n",
       " 140,\n",
       " 141,\n",
       " 142,\n",
       " 143,\n",
       " 144,\n",
       " 145,\n",
       " 146,\n",
       " 147,\n",
       " 148,\n",
       " 149,\n",
       " 150,\n",
       " 151,\n",
       " 152,\n",
       " 153,\n",
       " 154,\n",
       " 155,\n",
       " 156,\n",
       " 157,\n",
       " 158,\n",
       " 159,\n",
       " 160,\n",
       " 161,\n",
       " 162,\n",
       " 163,\n",
       " 164,\n",
       " 165,\n",
       " 166,\n",
       " 167,\n",
       " 168,\n",
       " 169,\n",
       " 170,\n",
       " 171,\n",
       " 172,\n",
       " 173,\n",
       " 174,\n",
       " 175,\n",
       " 176,\n",
       " 177,\n",
       " 178,\n",
       " 179,\n",
       " 180,\n",
       " 181,\n",
       " 182,\n",
       " 183,\n",
       " 184,\n",
       " 185,\n",
       " 186,\n",
       " 187,\n",
       " 188,\n",
       " 189,\n",
       " 190,\n",
       " 191,\n",
       " 192,\n",
       " 193,\n",
       " 194,\n",
       " 195,\n",
       " 196,\n",
       " 197,\n",
       " 198,\n",
       " 199,\n",
       " 200,\n",
       " 201,\n",
       " 202,\n",
       " 203,\n",
       " 204,\n",
       " 205,\n",
       " 206,\n",
       " 207,\n",
       " 208,\n",
       " 209,\n",
       " 210,\n",
       " 211,\n",
       " 212,\n",
       " 213,\n",
       " 214,\n",
       " 215,\n",
       " 216,\n",
       " 217,\n",
       " 218,\n",
       " 219,\n",
       " 220,\n",
       " 221,\n",
       " 222,\n",
       " 223,\n",
       " 224,\n",
       " 225,\n",
       " 226,\n",
       " 227,\n",
       " 228,\n",
       " 229,\n",
       " 230,\n",
       " 231,\n",
       " 232,\n",
       " 233,\n",
       " 234,\n",
       " 235,\n",
       " 236,\n",
       " 237,\n",
       " 238,\n",
       " 239,\n",
       " 240,\n",
       " 241,\n",
       " 242,\n",
       " 243,\n",
       " 244,\n",
       " 245,\n",
       " 246,\n",
       " 247,\n",
       " 248,\n",
       " 249,\n",
       " 250,\n",
       " 251,\n",
       " 252,\n",
       " 253,\n",
       " 254,\n",
       " 255,\n",
       " 256,\n",
       " 257,\n",
       " 258,\n",
       " 259,\n",
       " 260,\n",
       " 261,\n",
       " 262,\n",
       " 263,\n",
       " 264,\n",
       " 265,\n",
       " 266,\n",
       " 267,\n",
       " 268,\n",
       " 269,\n",
       " 270,\n",
       " 271,\n",
       " 272,\n",
       " 273,\n",
       " 274,\n",
       " 275,\n",
       " 276,\n",
       " 277,\n",
       " 278,\n",
       " 279,\n",
       " 280,\n",
       " 281,\n",
       " 282,\n",
       " 283,\n",
       " 284,\n",
       " 285,\n",
       " 286,\n",
       " 287,\n",
       " 288,\n",
       " 289,\n",
       " 290,\n",
       " 291,\n",
       " 292,\n",
       " 293,\n",
       " 294,\n",
       " 295,\n",
       " 296,\n",
       " 297,\n",
       " 298,\n",
       " 299,\n",
       " 300,\n",
       " 301,\n",
       " 302,\n",
       " 303,\n",
       " 304,\n",
       " 305,\n",
       " 306,\n",
       " 307,\n",
       " 308,\n",
       " 309,\n",
       " 310,\n",
       " 311,\n",
       " 312,\n",
       " 313,\n",
       " 314,\n",
       " 315,\n",
       " 316,\n",
       " 317,\n",
       " 318,\n",
       " 319,\n",
       " 320,\n",
       " 321,\n",
       " 322,\n",
       " 323,\n",
       " 324,\n",
       " 325,\n",
       " 326,\n",
       " 327,\n",
       " 328,\n",
       " 329,\n",
       " 330,\n",
       " 331,\n",
       " 332,\n",
       " 333,\n",
       " 334,\n",
       " 335,\n",
       " 336,\n",
       " 337,\n",
       " 338,\n",
       " 339,\n",
       " 340,\n",
       " 341,\n",
       " 342,\n",
       " 343,\n",
       " 344,\n",
       " 345,\n",
       " 346,\n",
       " 347,\n",
       " 348,\n",
       " 349,\n",
       " 350,\n",
       " 351,\n",
       " 352,\n",
       " 353,\n",
       " 354,\n",
       " 355,\n",
       " 356,\n",
       " 357,\n",
       " 358,\n",
       " 359,\n",
       " 360,\n",
       " 361,\n",
       " 362,\n",
       " 363,\n",
       " 364,\n",
       " 365,\n",
       " 366,\n",
       " 367,\n",
       " 368,\n",
       " 369,\n",
       " 370,\n",
       " 371,\n",
       " 372,\n",
       " 373,\n",
       " 374,\n",
       " 375,\n",
       " 376,\n",
       " 377,\n",
       " 378,\n",
       " 379,\n",
       " 380,\n",
       " 381,\n",
       " 382,\n",
       " 383,\n",
       " 384,\n",
       " 385,\n",
       " 386,\n",
       " 387,\n",
       " 388,\n",
       " 389,\n",
       " 390,\n",
       " 391,\n",
       " 392,\n",
       " 393,\n",
       " 394,\n",
       " 395,\n",
       " 396,\n",
       " 397,\n",
       " 398,\n",
       " 399,\n",
       " 400,\n",
       " 401,\n",
       " 402,\n",
       " 403,\n",
       " 404,\n",
       " 405,\n",
       " 406,\n",
       " 407,\n",
       " 408,\n",
       " 409,\n",
       " 410,\n",
       " 411,\n",
       " 412,\n",
       " 413,\n",
       " 414,\n",
       " 415,\n",
       " 416,\n",
       " 417,\n",
       " 418,\n",
       " 419,\n",
       " 420,\n",
       " 421,\n",
       " 422,\n",
       " 423,\n",
       " 424,\n",
       " 425,\n",
       " 426,\n",
       " 427,\n",
       " 428,\n",
       " 429,\n",
       " 430,\n",
       " 431,\n",
       " 432,\n",
       " 433,\n",
       " 434,\n",
       " 435,\n",
       " 436,\n",
       " 437,\n",
       " 438,\n",
       " 439,\n",
       " 440,\n",
       " 441,\n",
       " 442,\n",
       " 443,\n",
       " 444,\n",
       " 445,\n",
       " 446,\n",
       " 447,\n",
       " 448,\n",
       " 449,\n",
       " 450,\n",
       " 451,\n",
       " 452,\n",
       " 453,\n",
       " 454,\n",
       " 455,\n",
       " 456,\n",
       " 457,\n",
       " 458,\n",
       " 459,\n",
       " 460,\n",
       " 461,\n",
       " 462,\n",
       " 463,\n",
       " 464,\n",
       " 465,\n",
       " 466,\n",
       " 467,\n",
       " 468,\n",
       " 469,\n",
       " 470,\n",
       " 471,\n",
       " 472,\n",
       " 473,\n",
       " 474,\n",
       " 475,\n",
       " 476,\n",
       " 477,\n",
       " 478,\n",
       " 479,\n",
       " 480,\n",
       " 481,\n",
       " 482,\n",
       " 483,\n",
       " 484,\n",
       " 485,\n",
       " 486,\n",
       " 487,\n",
       " 488,\n",
       " 489,\n",
       " 490,\n",
       " 491,\n",
       " 492,\n",
       " 493,\n",
       " 494,\n",
       " 495,\n",
       " 496,\n",
       " 497,\n",
       " 498,\n",
       " 499,\n",
       " 500,\n",
       " 501,\n",
       " 502,\n",
       " 503,\n",
       " 504,\n",
       " 505,\n",
       " 506,\n",
       " 507,\n",
       " 508,\n",
       " 509,\n",
       " 510,\n",
       " 511,\n",
       " 512,\n",
       " 513,\n",
       " 514,\n",
       " 515,\n",
       " 516,\n",
       " 517,\n",
       " 518,\n",
       " 519,\n",
       " 520,\n",
       " 521,\n",
       " 522,\n",
       " 523,\n",
       " 524,\n",
       " 525,\n",
       " 526,\n",
       " 527,\n",
       " 528,\n",
       " 529,\n",
       " 530,\n",
       " 531,\n",
       " 532,\n",
       " 533,\n",
       " 534,\n",
       " 535,\n",
       " 536,\n",
       " 537,\n",
       " 538,\n",
       " 539,\n",
       " 540,\n",
       " 541,\n",
       " 542,\n",
       " 543,\n",
       " 544,\n",
       " 545,\n",
       " 546,\n",
       " 547,\n",
       " 548,\n",
       " 549,\n",
       " 550,\n",
       " 551,\n",
       " 552,\n",
       " 553,\n",
       " 554,\n",
       " 555,\n",
       " 556,\n",
       " 557,\n",
       " 558,\n",
       " 559,\n",
       " 560,\n",
       " 561,\n",
       " 562,\n",
       " 563,\n",
       " 564,\n",
       " 565,\n",
       " 566,\n",
       " 567,\n",
       " 568,\n",
       " 569,\n",
       " 570,\n",
       " 571,\n",
       " 572,\n",
       " 573,\n",
       " 574,\n",
       " 575,\n",
       " 576,\n",
       " 577,\n",
       " 578,\n",
       " 579,\n",
       " 580,\n",
       " 581,\n",
       " 582,\n",
       " 583,\n",
       " 584,\n",
       " 585,\n",
       " 586,\n",
       " 587,\n",
       " 588,\n",
       " 589,\n",
       " 590,\n",
       " 591,\n",
       " 592,\n",
       " 593,\n",
       " 594,\n",
       " 595,\n",
       " 596,\n",
       " 597,\n",
       " 598,\n",
       " 599,\n",
       " 600,\n",
       " 601,\n",
       " 602,\n",
       " 603,\n",
       " 604,\n",
       " 605,\n",
       " 606,\n",
       " 607,\n",
       " 608,\n",
       " 609,\n",
       " 610,\n",
       " 611,\n",
       " 612,\n",
       " 613,\n",
       " 614,\n",
       " 615,\n",
       " 616,\n",
       " 617,\n",
       " 618,\n",
       " 619,\n",
       " 620,\n",
       " 621,\n",
       " 622,\n",
       " 623,\n",
       " 624,\n",
       " 625,\n",
       " 626,\n",
       " 627,\n",
       " 628,\n",
       " 629,\n",
       " 630,\n",
       " 631,\n",
       " 632,\n",
       " 633,\n",
       " 634,\n",
       " 635,\n",
       " 636,\n",
       " 637,\n",
       " 638,\n",
       " 639,\n",
       " 640,\n",
       " 641,\n",
       " 642,\n",
       " 643,\n",
       " 644,\n",
       " 645,\n",
       " 646,\n",
       " 647,\n",
       " 648,\n",
       " 649,\n",
       " 650,\n",
       " 651,\n",
       " 652,\n",
       " 653,\n",
       " 654,\n",
       " 655,\n",
       " 656,\n",
       " 657,\n",
       " 658,\n",
       " 659,\n",
       " 660,\n",
       " 661,\n",
       " 662,\n",
       " 663,\n",
       " 664,\n",
       " 665,\n",
       " 666,\n",
       " 667,\n",
       " 668,\n",
       " 669,\n",
       " 670,\n",
       " 671,\n",
       " 672,\n",
       " 673,\n",
       " 674,\n",
       " 675,\n",
       " 676,\n",
       " 677,\n",
       " 678,\n",
       " 679,\n",
       " 680,\n",
       " 681,\n",
       " 682,\n",
       " 683,\n",
       " 684,\n",
       " 685,\n",
       " 686,\n",
       " 687,\n",
       " 688,\n",
       " 689,\n",
       " 690,\n",
       " 691,\n",
       " 692,\n",
       " 693,\n",
       " 694,\n",
       " 695,\n",
       " 696,\n",
       " 697,\n",
       " 698,\n",
       " 699,\n",
       " 700,\n",
       " 701,\n",
       " 702,\n",
       " 703,\n",
       " 704,\n",
       " 705,\n",
       " 706,\n",
       " 707,\n",
       " 708,\n",
       " 709,\n",
       " 710,\n",
       " 711,\n",
       " 712,\n",
       " 713,\n",
       " 714,\n",
       " 715,\n",
       " 716,\n",
       " 717,\n",
       " 718,\n",
       " 719,\n",
       " 720,\n",
       " 721,\n",
       " 722,\n",
       " 723,\n",
       " 724,\n",
       " 725,\n",
       " 726,\n",
       " 727,\n",
       " 728,\n",
       " 729,\n",
       " 730,\n",
       " 731,\n",
       " 732,\n",
       " 733,\n",
       " 734,\n",
       " 735,\n",
       " 736,\n",
       " 737,\n",
       " 738,\n",
       " 739,\n",
       " 740,\n",
       " 741,\n",
       " 742,\n",
       " 743,\n",
       " 744,\n",
       " 745,\n",
       " 746,\n",
       " 747,\n",
       " 748,\n",
       " 749,\n",
       " 750,\n",
       " 751,\n",
       " 752,\n",
       " 753,\n",
       " 754,\n",
       " 755,\n",
       " 756,\n",
       " 757,\n",
       " 758,\n",
       " 759,\n",
       " 760,\n",
       " 761,\n",
       " 762,\n",
       " 763,\n",
       " 764,\n",
       " 765,\n",
       " 766,\n",
       " 767,\n",
       " 768,\n",
       " 769,\n",
       " 770,\n",
       " 771,\n",
       " 772,\n",
       " 773,\n",
       " 774,\n",
       " 775,\n",
       " 776,\n",
       " 777,\n",
       " 778,\n",
       " 779,\n",
       " 780,\n",
       " 781,\n",
       " 782,\n",
       " 783,\n",
       " 784,\n",
       " 785,\n",
       " 786,\n",
       " 787,\n",
       " 788,\n",
       " 789,\n",
       " 790,\n",
       " 791,\n",
       " 792,\n",
       " 793,\n",
       " 794,\n",
       " 795,\n",
       " 796,\n",
       " 797,\n",
       " 798,\n",
       " 799,\n",
       " 800,\n",
       " 801,\n",
       " 802,\n",
       " 803,\n",
       " 804,\n",
       " 805,\n",
       " 806,\n",
       " 807,\n",
       " 808,\n",
       " 809,\n",
       " 810,\n",
       " 811,\n",
       " 812,\n",
       " 813,\n",
       " 814,\n",
       " 815,\n",
       " 816,\n",
       " 817,\n",
       " 818,\n",
       " 819,\n",
       " 820,\n",
       " 821,\n",
       " 822,\n",
       " 823,\n",
       " 824,\n",
       " 825,\n",
       " 826,\n",
       " 827,\n",
       " 828,\n",
       " 829,\n",
       " 830,\n",
       " 831,\n",
       " 832,\n",
       " 833,\n",
       " 834,\n",
       " 835,\n",
       " 836,\n",
       " 837,\n",
       " 838,\n",
       " 839,\n",
       " 840,\n",
       " 841,\n",
       " 842,\n",
       " 843,\n",
       " 844,\n",
       " 845,\n",
       " 846,\n",
       " 847,\n",
       " 848,\n",
       " 849,\n",
       " 850,\n",
       " 851,\n",
       " 852,\n",
       " 853,\n",
       " 854,\n",
       " 855,\n",
       " 856,\n",
       " 857,\n",
       " 858,\n",
       " 859,\n",
       " 860,\n",
       " 861,\n",
       " 862,\n",
       " 863,\n",
       " 864,\n",
       " 865,\n",
       " 866,\n",
       " 867,\n",
       " 868,\n",
       " 869,\n",
       " 870,\n",
       " 871,\n",
       " 872,\n",
       " 873,\n",
       " 874,\n",
       " 875,\n",
       " 876,\n",
       " 877,\n",
       " 878,\n",
       " 879,\n",
       " 880,\n",
       " 881,\n",
       " 882,\n",
       " 883,\n",
       " 884,\n",
       " 885,\n",
       " 886,\n",
       " 887,\n",
       " 888,\n",
       " 889,\n",
       " 890,\n",
       " 891,\n",
       " 892,\n",
       " 893,\n",
       " 894,\n",
       " 895,\n",
       " 896,\n",
       " 897,\n",
       " 898,\n",
       " 899,\n",
       " 900,\n",
       " 901,\n",
       " 902,\n",
       " 903,\n",
       " 904,\n",
       " 905,\n",
       " 906,\n",
       " 907,\n",
       " 908,\n",
       " 909,\n",
       " 910,\n",
       " 911,\n",
       " 912,\n",
       " 913,\n",
       " 914,\n",
       " 915,\n",
       " 916,\n",
       " 917,\n",
       " 918,\n",
       " 919,\n",
       " 920,\n",
       " 921,\n",
       " 922,\n",
       " 923,\n",
       " 924,\n",
       " 925,\n",
       " 926,\n",
       " 927,\n",
       " 928,\n",
       " 929,\n",
       " 930,\n",
       " 931,\n",
       " 932,\n",
       " 933,\n",
       " 934,\n",
       " 935,\n",
       " 936,\n",
       " 937,\n",
       " 938,\n",
       " 939,\n",
       " 940,\n",
       " 941,\n",
       " 942,\n",
       " 943,\n",
       " 944,\n",
       " 945,\n",
       " 946,\n",
       " 947,\n",
       " 948,\n",
       " 949,\n",
       " 950,\n",
       " 951,\n",
       " 952,\n",
       " 953,\n",
       " 954,\n",
       " 955,\n",
       " 956,\n",
       " 957,\n",
       " 958,\n",
       " 959,\n",
       " 960,\n",
       " 961,\n",
       " 962,\n",
       " 963,\n",
       " 964,\n",
       " 965,\n",
       " 966,\n",
       " 967,\n",
       " 968,\n",
       " 969,\n",
       " 970,\n",
       " 971,\n",
       " 972,\n",
       " 973,\n",
       " 974,\n",
       " 975,\n",
       " 976,\n",
       " 977,\n",
       " 978,\n",
       " 979,\n",
       " 980,\n",
       " 981,\n",
       " 982,\n",
       " 983,\n",
       " 984,\n",
       " 985,\n",
       " 986,\n",
       " 987,\n",
       " 988,\n",
       " 989,\n",
       " 990,\n",
       " 991,\n",
       " 992,\n",
       " 993,\n",
       " 994,\n",
       " 995,\n",
       " 996,\n",
       " 997,\n",
       " 998,\n",
       " 999,\n",
       " ...]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5558e862",
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_data = list(zip(indices, dist_list, ind_list))\n",
    "\n",
    "sorted_data = sorted(combined_data, key=lambda x: x[0])\n",
    "\n",
    "sorted_indices_list, sorted_dist_list, sorted_index_list = zip(*sorted_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4c498526",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([     0, 149242, 190042, ...,  30543,  20483,   9985])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_index_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0c49b43b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([    50,  43644,  78217,  15770,  91392, 172623,  90454, 168396])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_index_list[50][sorted_dist_list[50] <=0.45]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "05572787",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mohd Fuad Abdullah Sani kah'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "1b5bfaad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mohd Fuad Abdullah Sani takziah untuk kau sekeluarga'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[43644]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "64c217c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17,)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dist_list[1][dist_list[1] <=0.45].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94dccadc",
   "metadata": {},
   "outputs": [],
   "source": [
    "dist[dist > 0.8].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6294d1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "sns.distplot(dist[4:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "7399acf2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mohd Fuad Abdullah Sani kah'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "bc42d90a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mohd Fuad Abdullah Sani takziah untuk kau sekeluarga'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[43644]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "4cd2b3ab",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[24]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index = [x for x in ind_list[50][dist_list[50] < 0.4]]\n",
    "dist_index = [x for x in ind_list[50][dist_list[50] < 0.4]]\n",
    "\n",
    "index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "a16fdbe3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[50, 43644]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dist_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "d2bd8fd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 2)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dist_index),len(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99297e18",
   "metadata": {},
   "outputs": [],
   "source": [
    "[texts[x] for x in index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71ee016e",
   "metadata": {},
   "outputs": [],
   "source": [
    "index = [x for x in ind[19999][dist[19999] <= 0.45]]\n",
    "dist_index = [x for x in dist[19999][dist[19999] <= 0.45]]\n",
    "\n",
    "index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7caf9b23",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dist_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f27dc4a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "[texts[x] for x in index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4db6cce1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def construct_json(index):\n",
    "    result = {\n",
    "        'positive': [],\n",
    "        'negative': []\n",
    "    }\n",
    "    \n",
    "    result['positive'] = [x for x in sorted_index_list[index][sorted_dist_list[index] <= 0.45] if x != index]\n",
    "    result['negative'] = [x for x in sorted_index_list[index][sorted_dist_list[index] > 0.8]]\n",
    "    \n",
    "    return result\n",
    "\n",
    "\n",
    "result_json = {int(index): construct_json(index) for index in range(len(data))}\n",
    "\n",
    "print(result_json)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f385e618",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "192885"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "7fbd039e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert int64 to regular integers\n",
    "def convert_int64(obj):\n",
    "    if isinstance(obj, np.int64):\n",
    "        return int(obj)\n",
    "    raise TypeError\n",
    "\n",
    "json_result = {int(key): value for key, value in result_json.items()}\n",
    "\n",
    "with open('facebook-pair-mining.json', 'w') as json_file:\n",
    "    json.dump(json_result, json_file, indent=4, default = convert_int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44ee394f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[6]['cleaned_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f525334",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[57006]['cleaned_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "f25688a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('facebook-pair-mining.json') as json_file:\n",
    "    dat = json.load(json_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "9e8ee745",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'positive': [43644, 78217, 15770, 91392, 172623, 90454, 168396],\n",
       " 'negative': [159503,\n",
       "  145138,\n",
       "  63754,\n",
       "  181566,\n",
       "  10828,\n",
       "  167890,\n",
       "  127536,\n",
       "  29695,\n",
       "  106669,\n",
       "  39862,\n",
       "  107284,\n",
       "  29126,\n",
       "  15123,\n",
       "  131744,\n",
       "  42373,\n",
       "  2819,\n",
       "  118357,\n",
       "  69860,\n",
       "  101545,\n",
       "  20483,\n",
       "  30543,\n",
       "  9985]}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dat['50']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "fed17f11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Saifudin abdullah'"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[168396]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "78a723f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mohd Fuad Abdullah Sani kah'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7e2fa40",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}