iate_extract.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 1484: expected 7 fields, saw 8\\nSkipping line 15005: expected 7 fields, saw 8\\nSkipping line 17230: expected 7 fields, saw 8\\nSkipping line 18056: expected 7 fields, saw 8\\nSkipping line 19484: expected 7 fields, saw 8\\nSkipping line 21546: expected 7 fields, saw 8\\nSkipping line 23434: expected 7 fields, saw 8\\nSkipping line 24447: expected 7 fields, saw 8\\nSkipping line 25932: expected 7 fields, saw 8\\nSkipping line 26037: expected 7 fields, saw 8\\nSkipping line 28561: expected 7 fields, saw 8\\nSkipping line 33397: expected 7 fields, saw 8\\nSkipping line 34016: expected 7 fields, saw 8\\nSkipping line 36385: expected 7 fields, saw 8\\nSkipping line 36479: expected 7 fields, saw 8\\nSkipping line 40551: expected 7 fields, saw 8\\nSkipping line 58389: expected 7 fields, saw 8\\nSkipping line 60069: expected 7 fields, saw 8\\nSkipping line 61780: expected 7 fields, saw 8\\nSkipping line 64428: expected 7 fields, saw 8\\nSkipping line 65685: expected 7 fields, saw 8\\nSkipping line 67094: expected 7 fields, saw 8\\nSkipping line 73085: expected 7 fields, saw 8\\nSkipping line 79466: expected 7 fields, saw 8\\nSkipping line 81982: expected 7 fields, saw 8\\nSkipping line 85500: expected 7 fields, saw 8\\nSkipping line 85658: expected 7 fields, saw 8\\nSkipping line 89666: expected 7 fields, saw 8\\nSkipping line 94720: expected 7 fields, saw 8\\nSkipping line 100839: expected 7 fields, saw 8\\nSkipping line 102037: expected 7 fields, saw 8\\nSkipping line 114949: expected 7 fields, saw 8\\nSkipping line 119213: expected 7 fields, saw 8\\nSkipping line 121406: expected 7 fields, saw 8\\nSkipping line 127324: expected 7 fields, saw 8\\nSkipping line 128486: expected 7 fields, saw 8\\nSkipping line 128614: expected 7 fields, saw 8\\nSkipping line 128979: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 131268: expected 7 fields, saw 8\\nSkipping line 140999: expected 7 fields, saw 8\\nSkipping line 143607: expected 7 fields, saw 8\\nSkipping line 144549: expected 7 fields, saw 8\\nSkipping line 146158: expected 7 fields, saw 8\\nSkipping line 147550: expected 7 fields, saw 8\\nSkipping line 149747: expected 7 fields, saw 8\\nSkipping line 149748: expected 7 fields, saw 8\\nSkipping line 152459: expected 7 fields, saw 8\\nSkipping line 154455: expected 7 fields, saw 8\\nSkipping line 157381: expected 7 fields, saw 8\\nSkipping line 159216: expected 7 fields, saw 8\\nSkipping line 161397: expected 7 fields, saw 8\\nSkipping line 163243: expected 7 fields, saw 8\\nSkipping line 167386: expected 7 fields, saw 8\\nSkipping line 169569: expected 7 fields, saw 8\\nSkipping line 169726: expected 7 fields, saw 8\\nSkipping line 169727: expected 7 fields, saw 8\\nSkipping line 171889: expected 7 fields, saw 8\\nSkipping line 175904: expected 7 fields, saw 8\\nSkipping line 176525: expected 7 fields, saw 8\\nSkipping line 182754: expected 7 fields, saw 8\\nSkipping line 183214: expected 7 fields, saw 8\\nSkipping line 186576: expected 7 fields, saw 8\\nSkipping line 187285: expected 7 fields, saw 8\\nSkipping line 188445: expected 7 fields, saw 8\\nSkipping line 191447: expected 7 fields, saw 8\\nSkipping line 191448: expected 7 fields, saw 8\\nSkipping line 193559: expected 7 fields, saw 8\\nSkipping line 195189: expected 7 fields, saw 8\\nSkipping line 199243: expected 7 fields, saw 8\\nSkipping line 208597: expected 7 fields, saw 8\\nSkipping line 208598: expected 7 fields, saw 8\\nSkipping line 211097: expected 7 fields, saw 8\\nSkipping line 212413: expected 7 fields, saw 8\\nSkipping line 217906: expected 7 fields, saw 8\\nSkipping line 223630: expected 7 fields, saw 8\\nSkipping line 240119: expected 7 fields, saw 8\\nSkipping line 245491: expected 7 fields, saw 8\\nSkipping line 252321: expected 7 fields, saw 8\\nSkipping line 252444: expected 7 fields, saw 8\\nSkipping line 253123: expected 7 fields, saw 8\\nSkipping line 261554: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 270985: expected 7 fields, saw 8\\nSkipping line 276050: expected 7 fields, saw 8\\nSkipping line 276716: expected 7 fields, saw 8\\nSkipping line 279146: expected 7 fields, saw 8\\nSkipping line 280624: expected 7 fields, saw 8\\nSkipping line 281408: expected 7 fields, saw 8\\nSkipping line 284053: expected 7 fields, saw 8\\nSkipping line 285121: expected 7 fields, saw 8\\nSkipping line 285247: expected 7 fields, saw 8\\nSkipping line 287259: expected 7 fields, saw 8\\nSkipping line 287260: expected 7 fields, saw 8\\nSkipping line 287270: expected 7 fields, saw 8\\nSkipping line 287271: expected 7 fields, saw 8\\nSkipping line 287933: expected 7 fields, saw 8\\nSkipping line 287934: expected 7 fields, saw 8\\nSkipping line 289939: expected 7 fields, saw 8\\nSkipping line 294549: expected 7 fields, saw 8\\nSkipping line 294588: expected 7 fields, saw 8\\nSkipping line 296173: expected 7 fields, saw 8\\nSkipping line 296536: expected 7 fields, saw 8\\nSkipping line 300337: expected 7 fields, saw 8\\nSkipping line 300338: expected 7 fields, saw 8\\nSkipping line 301688: expected 7 fields, saw 8\\nSkipping line 302836: expected 7 fields, saw 8\\nSkipping line 305228: expected 7 fields, saw 8\\nSkipping line 311917: expected 7 fields, saw 8\\nSkipping line 312881: expected 7 fields, saw 8\\nSkipping line 321399: expected 7 fields, saw 8\\nSkipping line 322091: expected 7 fields, saw 8\\nSkipping line 323018: expected 7 fields, saw 8\\nSkipping line 328963: expected 7 fields, saw 8\\nSkipping line 333549: expected 7 fields, saw 8\\nSkipping line 337329: expected 7 fields, saw 8\\nSkipping line 342715: expected 7 fields, saw 8\\nSkipping line 343219: expected 7 fields, saw 8\\nSkipping line 345580: expected 7 fields, saw 8\\nSkipping line 349441: expected 7 fields, saw 8\\nSkipping line 349951: expected 7 fields, saw 8\\nSkipping line 350134: expected 7 fields, saw 8\\nSkipping line 353833: expected 7 fields, saw 8\\nSkipping line 355556: expected 7 fields, saw 8\\nSkipping line 357563: expected 7 fields, saw 8\\nSkipping line 357924: expected 7 fields, saw 8\\nSkipping line 360548: expected 7 fields, saw 8\\nSkipping line 362639: expected 7 fields, saw 8\\nSkipping line 364333: expected 7 fields, saw 8\\nSkipping line 365590: expected 7 fields, saw 8\\nSkipping line 369566: expected 7 fields, saw 8\\nSkipping line 375524: expected 7 fields, saw 8\\nSkipping line 378288: expected 7 fields, saw 8\\nSkipping line 388474: expected 7 fields, saw 8\\nSkipping line 390962: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 397343: expected 7 fields, saw 8\\nSkipping line 397428: expected 7 fields, saw 8\\nSkipping line 398522: expected 7 fields, saw 8\\nSkipping line 402198: expected 7 fields, saw 8\\nSkipping line 408128: expected 7 fields, saw 8\\nSkipping line 413127: expected 7 fields, saw 8\\nSkipping line 416541: expected 7 fields, saw 8\\nSkipping line 420058: expected 7 fields, saw 8\\nSkipping line 421588: expected 7 fields, saw 8\\nSkipping line 422767: expected 7 fields, saw 8\\nSkipping line 422969: expected 7 fields, saw 8\\nSkipping line 423502: expected 7 fields, saw 8\\nSkipping line 425716: expected 7 fields, saw 8\\nSkipping line 425728: expected 7 fields, saw 8\\nSkipping line 426584: expected 7 fields, saw 8\\nSkipping line 438032: expected 7 fields, saw 8\\nSkipping line 443341: expected 7 fields, saw 8\\nSkipping line 446621: expected 7 fields, saw 8\\nSkipping line 450826: expected 7 fields, saw 8\\nSkipping line 451963: expected 7 fields, saw 8\\nSkipping line 463230: expected 7 fields, saw 8\\nSkipping line 468980: expected 7 fields, saw 8\\nSkipping line 471566: expected 7 fields, saw 8\\nSkipping line 472947: expected 7 fields, saw 8\\nSkipping line 475050: expected 7 fields, saw 8\\nSkipping line 475796: expected 7 fields, saw 8\\nSkipping line 476753: expected 7 fields, saw 8\\nSkipping line 478442: expected 7 fields, saw 8\\nSkipping line 480947: expected 7 fields, saw 8\\nSkipping line 481620: expected 7 fields, saw 8\\nSkipping line 484686: expected 7 fields, saw 8\\nSkipping line 489003: expected 7 fields, saw 8\\nSkipping line 490827: expected 7 fields, saw 8\\nSkipping line 494073: expected 7 fields, saw 8\\nSkipping line 500814: expected 7 fields, saw 8\\nSkipping line 500958: expected 7 fields, saw 8\\nSkipping line 504091: expected 7 fields, saw 8\\nSkipping line 504769: expected 7 fields, saw 8\\nSkipping line 504886: expected 7 fields, saw 8\\nSkipping line 509446: expected 7 fields, saw 8\\nSkipping line 510791: expected 7 fields, saw 8\\nSkipping line 510980: expected 7 fields, saw 8\\nSkipping line 522430: expected 7 fields, saw 8\\nSkipping line 522619: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 532641: expected 7 fields, saw 8\\nSkipping line 533684: expected 7 fields, saw 8\\nSkipping line 534198: expected 7 fields, saw 8\\nSkipping line 534252: expected 7 fields, saw 8\\nSkipping line 535442: expected 7 fields, saw 8\\nSkipping line 536010: expected 7 fields, saw 8\\nSkipping line 536641: expected 7 fields, saw 8\\nSkipping line 542891: expected 7 fields, saw 8\\nSkipping line 543885: expected 7 fields, saw 8\\nSkipping line 547121: expected 7 fields, saw 8\\nSkipping line 549529: expected 7 fields, saw 8\\nSkipping line 550953: expected 7 fields, saw 8\\nSkipping line 551106: expected 7 fields, saw 8\\nSkipping line 552554: expected 7 fields, saw 8\\nSkipping line 552555: expected 7 fields, saw 8\\nSkipping line 555279: expected 7 fields, saw 8\\nSkipping line 555838: expected 7 fields, saw 8\\nSkipping line 568475: expected 7 fields, saw 8\\nSkipping line 569468: expected 7 fields, saw 8\\nSkipping line 571174: expected 7 fields, saw 8\\nSkipping line 571675: expected 7 fields, saw 8\\nSkipping line 579722: expected 7 fields, saw 8\\nSkipping line 581049: expected 7 fields, saw 8\\nSkipping line 585047: expected 7 fields, saw 8\\nSkipping line 587948: expected 7 fields, saw 8\\nSkipping line 593147: expected 7 fields, saw 8\\nSkipping line 594479: expected 7 fields, saw 8\\nSkipping line 594497: expected 7 fields, saw 8\\nSkipping line 595245: expected 7 fields, saw 8\\nSkipping line 599320: expected 7 fields, saw 8\\nSkipping line 603208: expected 7 fields, saw 8\\nSkipping line 605480: expected 7 fields, saw 8\\nSkipping line 606613: expected 7 fields, saw 8\\nSkipping line 613684: expected 7 fields, saw 8\\nSkipping line 618488: expected 7 fields, saw 8\\nSkipping line 618489: expected 7 fields, saw 8\\nSkipping line 618490: expected 7 fields, saw 8\\nSkipping line 622623: expected 7 fields, saw 8\\nSkipping line 622624: expected 7 fields, saw 8\\nSkipping line 622808: expected 7 fields, saw 8\\nSkipping line 623967: expected 7 fields, saw 8\\nSkipping line 626147: expected 7 fields, saw 8\\nSkipping line 643742: expected 7 fields, saw 8\\nSkipping line 650986: expected 7 fields, saw 8\\nSkipping line 652386: expected 7 fields, saw 8\\nSkipping line 653588: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 656246: expected 7 fields, saw 8\\nSkipping line 664263: expected 7 fields, saw 8\\nSkipping line 664265: expected 7 fields, saw 8\\nSkipping line 665813: expected 7 fields, saw 8\\nSkipping line 667140: expected 7 fields, saw 8\\nSkipping line 668447: expected 7 fields, saw 8\\nSkipping line 670673: expected 7 fields, saw 8\\nSkipping line 676960: expected 7 fields, saw 8\\nSkipping line 676962: expected 7 fields, saw 8\\nSkipping line 681140: expected 7 fields, saw 8\\nSkipping line 688873: expected 7 fields, saw 8\\nSkipping line 690296: expected 7 fields, saw 8\\nSkipping line 690297: expected 7 fields, saw 8\\nSkipping line 690729: expected 7 fields, saw 8\\nSkipping line 698592: expected 7 fields, saw 8\\nSkipping line 700364: expected 7 fields, saw 8\\nSkipping line 700366: expected 7 fields, saw 8\\nSkipping line 700532: expected 7 fields, saw 8\\nSkipping line 700963: expected 7 fields, saw 8\\nSkipping line 701217: expected 7 fields, saw 8\\nSkipping line 701218: expected 7 fields, saw 8\\nSkipping line 711404: expected 7 fields, saw 8\\nSkipping line 715348: expected 7 fields, saw 8\\nSkipping line 716696: expected 7 fields, saw 8\\nSkipping line 727932: expected 7 fields, saw 8\\nSkipping line 730941: expected 7 fields, saw 8\\nSkipping line 736061: expected 7 fields, saw 8\\nSkipping line 743248: expected 7 fields, saw 8\\nSkipping line 744306: expected 7 fields, saw 8\\nSkipping line 746792: expected 7 fields, saw 8\\nSkipping line 747934: expected 7 fields, saw 8\\nSkipping line 760767: expected 7 fields, saw 8\\nSkipping line 765954: expected 7 fields, saw 8\\nSkipping line 765978: expected 7 fields, saw 8\\nSkipping line 766361: expected 7 fields, saw 8\\nSkipping line 770444: expected 7 fields, saw 8\\nSkipping line 772136: expected 7 fields, saw 8\\nSkipping line 775342: expected 7 fields, saw 8\\nSkipping line 776633: expected 7 fields, saw 8\\nSkipping line 778967: expected 7 fields, saw 8\\nSkipping line 781167: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 787215: expected 7 fields, saw 8\\nSkipping line 793510: expected 7 fields, saw 8\\nSkipping line 794913: expected 7 fields, saw 8\\nSkipping line 797658: expected 7 fields, saw 8\\nSkipping line 802387: expected 7 fields, saw 8\\nSkipping line 802841: expected 7 fields, saw 8\\nSkipping line 803981: expected 7 fields, saw 8\\nSkipping line 806165: expected 7 fields, saw 8\\nSkipping line 810308: expected 7 fields, saw 8\\nSkipping line 811087: expected 7 fields, saw 8\\nSkipping line 812521: expected 7 fields, saw 8\\nSkipping line 820065: expected 7 fields, saw 8\\nSkipping line 821413: expected 7 fields, saw 8\\nSkipping line 822342: expected 7 fields, saw 8\\nSkipping line 827846: expected 7 fields, saw 8\\nSkipping line 829547: expected 7 fields, saw 8\\nSkipping line 829606: expected 7 fields, saw 8\\nSkipping line 845671: expected 7 fields, saw 8\\nSkipping line 845836: expected 7 fields, saw 8\\nSkipping line 847152: expected 7 fields, saw 8\\nSkipping line 850350: expected 7 fields, saw 8\\nSkipping line 857335: expected 7 fields, saw 8\\nSkipping line 861044: expected 7 fields, saw 8\\nSkipping line 865986: expected 7 fields, saw 8\\nSkipping line 867037: expected 7 fields, saw 8\\nSkipping line 868819: expected 7 fields, saw 8\\nSkipping line 869259: expected 7 fields, saw 8\\nSkipping line 869796: expected 7 fields, saw 8\\nSkipping line 869870: expected 7 fields, saw 8\\nSkipping line 870588: expected 7 fields, saw 8\\nSkipping line 880403: expected 7 fields, saw 8\\nSkipping line 881045: expected 7 fields, saw 8\\nSkipping line 882074: expected 7 fields, saw 8\\nSkipping line 882968: expected 7 fields, saw 8\\nSkipping line 887746: expected 7 fields, saw 8\\nSkipping line 889024: expected 7 fields, saw 8\\nSkipping line 891558: expected 7 fields, saw 8\\nSkipping line 896673: expected 7 fields, saw 8\\nSkipping line 896929: expected 7 fields, saw 8\\nSkipping line 898918: expected 7 fields, saw 8\\nSkipping line 898955: expected 7 fields, saw 8\\nSkipping line 899130: expected 7 fields, saw 8\\nSkipping line 906855: expected 7 fields, saw 8\\nSkipping line 913284: expected 7 fields, saw 8\\nSkipping line 914036: expected 7 fields, saw 8\\nSkipping line 914037: expected 7 fields, saw 8\\nSkipping line 915273: expected 7 fields, saw 8\\nSkipping line 915331: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 920119: expected 7 fields, saw 8\\nSkipping line 920911: expected 7 fields, saw 8\\nSkipping line 925448: expected 7 fields, saw 8\\nSkipping line 929208: expected 7 fields, saw 8\\nSkipping line 932791: expected 7 fields, saw 8\\nSkipping line 937768: expected 7 fields, saw 8\\nSkipping line 938039: expected 7 fields, saw 8\\nSkipping line 942458: expected 7 fields, saw 8\\nSkipping line 943573: expected 7 fields, saw 8\\nSkipping line 944440: expected 7 fields, saw 8\\nSkipping line 958305: expected 7 fields, saw 8\\nSkipping line 959275: expected 7 fields, saw 8\\nSkipping line 962152: expected 7 fields, saw 8\\nSkipping line 965484: expected 7 fields, saw 8\\nSkipping line 972926: expected 7 fields, saw 8\\nSkipping line 975063: expected 7 fields, saw 8\\nSkipping line 980810: expected 7 fields, saw 8\\nSkipping line 988680: expected 7 fields, saw 8\\nSkipping line 988868: expected 7 fields, saw 8\\nSkipping line 989277: expected 7 fields, saw 8\\nSkipping line 989278: expected 7 fields, saw 8\\nSkipping line 989444: expected 7 fields, saw 8\\nSkipping line 989620: expected 7 fields, saw 8\\nSkipping line 989621: expected 7 fields, saw 8\\nSkipping line 997754: expected 7 fields, saw 8\\nSkipping line 1004307: expected 7 fields, saw 8\\nSkipping line 1008773: expected 7 fields, saw 8\\nSkipping line 1017515: expected 7 fields, saw 8\\nSkipping line 1021279: expected 7 fields, saw 8\\nSkipping line 1021762: expected 7 fields, saw 8\\nSkipping line 1025660: expected 7 fields, saw 8\\nSkipping line 1027066: expected 7 fields, saw 8\\nSkipping line 1027904: expected 7 fields, saw 8\\nSkipping line 1028612: expected 7 fields, saw 8\\nSkipping line 1032413: expected 7 fields, saw 8\\nSkipping line 1035144: expected 7 fields, saw 8\\nSkipping line 1036248: expected 7 fields, saw 8\\nSkipping line 1039361: expected 7 fields, saw 8\\nSkipping line 1047458: expected 7 fields, saw 8\\nSkipping line 1047955: expected 7 fields, saw 8\\nSkipping line 1048080: expected 7 fields, saw 8\\nSkipping line 1048373: expected 7 fields, saw 8\\nSkipping line 1048705: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 1051932: expected 7 fields, saw 8\\nSkipping line 1054243: expected 7 fields, saw 8\\nSkipping line 1054244: expected 7 fields, saw 8\\nSkipping line 1054245: expected 7 fields, saw 8\\nSkipping line 1058526: expected 7 fields, saw 8\\nSkipping line 1074860: expected 7 fields, saw 8\\nSkipping line 1074883: expected 7 fields, saw 8\\nSkipping line 1076892: expected 7 fields, saw 8\\nSkipping line 1077560: expected 7 fields, saw 8\\nSkipping line 1086777: expected 7 fields, saw 8\\nSkipping line 1088261: expected 7 fields, saw 8\\nSkipping line 1091683: expected 7 fields, saw 8\\nSkipping line 1092215: expected 7 fields, saw 8\\nSkipping line 1096293: expected 7 fields, saw 8\\nSkipping line 1097327: expected 7 fields, saw 8\\nSkipping line 1098810: expected 7 fields, saw 8\\nSkipping line 1099708: expected 7 fields, saw 8\\nSkipping line 1099709: expected 7 fields, saw 8\\nSkipping line 1100456: expected 7 fields, saw 8\\nSkipping line 1105664: expected 7 fields, saw 8\\nSkipping line 1105857: expected 7 fields, saw 8\\nSkipping line 1108082: expected 7 fields, saw 8\\nSkipping line 1108383: expected 7 fields, saw 8\\nSkipping line 1110465: expected 7 fields, saw 8\\nSkipping line 1113862: expected 7 fields, saw 8\\nSkipping line 1120741: expected 7 fields, saw 8\\nSkipping line 1122624: expected 7 fields, saw 8\\nSkipping line 1126462: expected 7 fields, saw 8\\nSkipping line 1126491: expected 7 fields, saw 8\\nSkipping line 1126543: expected 7 fields, saw 8\\nSkipping line 1127818: expected 7 fields, saw 8\\nSkipping line 1129180: expected 7 fields, saw 8\\nSkipping line 1131440: expected 7 fields, saw 8\\nSkipping line 1131527: expected 7 fields, saw 8\\nSkipping line 1132784: expected 7 fields, saw 8\\nSkipping line 1135954: expected 7 fields, saw 8\\nSkipping line 1138376: expected 7 fields, saw 8\\nSkipping line 1139417: expected 7 fields, saw 8\\nSkipping line 1141359: expected 7 fields, saw 8\\nSkipping line 1146553: expected 7 fields, saw 8\\nSkipping line 1149009: expected 7 fields, saw 8\\nSkipping line 1155686: expected 7 fields, saw 8\\nSkipping line 1157237: expected 7 fields, saw 8\\nSkipping line 1157510: expected 7 fields, saw 8\\nSkipping line 1158737: expected 7 fields, saw 8\\nSkipping line 1160958: expected 7 fields, saw 8\\nSkipping line 1162165: expected 7 fields, saw 8\\nSkipping line 1167057: expected 7 fields, saw 8\\nSkipping line 1170367: expected 7 fields, saw 8\\nSkipping line 1170932: expected 7 fields, saw 8\\nSkipping line 1175393: expected 7 fields, saw 8\\nSkipping line 1178108: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 1181279: expected 7 fields, saw 8\\nSkipping line 1189670: expected 7 fields, saw 8\\nSkipping line 1192083: expected 7 fields, saw 8\\nSkipping line 1195023: expected 7 fields, saw 8\\nSkipping line 1195242: expected 7 fields, saw 8\\nSkipping line 1199426: expected 7 fields, saw 8\\nSkipping line 1200858: expected 7 fields, saw 8\\nSkipping line 1201505: expected 7 fields, saw 8\\nSkipping line 1201721: expected 7 fields, saw 8\\nSkipping line 1204678: expected 7 fields, saw 8\\nSkipping line 1204799: expected 7 fields, saw 8\\nSkipping line 1205179: expected 7 fields, saw 8\\nSkipping line 1217250: expected 7 fields, saw 8\\nSkipping line 1218358: expected 7 fields, saw 8\\nSkipping line 1220752: expected 7 fields, saw 8\\nSkipping line 1226416: expected 7 fields, saw 8\\nSkipping line 1227196: expected 7 fields, saw 8\\nSkipping line 1232255: expected 7 fields, saw 8\\nSkipping line 1232629: expected 7 fields, saw 8\\nSkipping line 1234500: expected 7 fields, saw 8\\nSkipping line 1237212: expected 7 fields, saw 8\\nSkipping line 1238715: expected 7 fields, saw 8\\nSkipping line 1245523: expected 7 fields, saw 8\\nSkipping line 1247145: expected 7 fields, saw 8\\nSkipping line 1250738: expected 7 fields, saw 8\\nSkipping line 1253652: expected 7 fields, saw 8\\nSkipping line 1259387: expected 7 fields, saw 8\\nSkipping line 1259576: expected 7 fields, saw 8\\nSkipping line 1260881: expected 7 fields, saw 8\\nSkipping line 1262627: expected 7 fields, saw 8\\nSkipping line 1264618: expected 7 fields, saw 8\\nSkipping line 1264906: expected 7 fields, saw 8\\nSkipping line 1265514: expected 7 fields, saw 8\\nSkipping line 1270200: expected 7 fields, saw 8\\nSkipping line 1272517: expected 7 fields, saw 8\\nSkipping line 1274878: expected 7 fields, saw 8\\nSkipping line 1277516: expected 7 fields, saw 8\\nSkipping line 1278366: expected 7 fields, saw 8\\nSkipping line 1280020: expected 7 fields, saw 8\\nSkipping line 1280831: expected 7 fields, saw 8\\nSkipping line 1280832: expected 7 fields, saw 8\\nSkipping line 1282281: expected 7 fields, saw 8\\nSkipping line 1283314: expected 7 fields, saw 8\\nSkipping line 1285126: expected 7 fields, saw 8\\nSkipping line 1293125: expected 7 fields, saw 8\\nSkipping line 1293126: expected 7 fields, saw 8\\nSkipping line 1305769: expected 7 fields, saw 8\\nSkipping line 1310293: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 1318832: expected 7 fields, saw 8\\nSkipping line 1320097: expected 7 fields, saw 8\\nSkipping line 1324386: expected 7 fields, saw 8\\nSkipping line 1326963: expected 7 fields, saw 8\\nSkipping line 1331438: expected 7 fields, saw 8\\nSkipping line 1332985: expected 7 fields, saw 8\\nSkipping line 1334006: expected 7 fields, saw 8\\nSkipping line 1334007: expected 7 fields, saw 8\\nSkipping line 1334008: expected 7 fields, saw 8\\nSkipping line 1338685: expected 7 fields, saw 8\\nSkipping line 1344942: expected 7 fields, saw 8\\nSkipping line 1346480: expected 7 fields, saw 8\\nSkipping line 1349612: expected 7 fields, saw 8\\nSkipping line 1351545: expected 7 fields, saw 8\\nSkipping line 1353742: expected 7 fields, saw 8\\nSkipping line 1356395: expected 7 fields, saw 8\\nSkipping line 1358233: expected 7 fields, saw 8\\nSkipping line 1358748: expected 7 fields, saw 8\\nSkipping line 1361263: expected 7 fields, saw 8\\nSkipping line 1365246: expected 7 fields, saw 8\\nSkipping line 1367357: expected 7 fields, saw 8\\nSkipping line 1368751: expected 7 fields, saw 8\\nSkipping line 1370592: expected 7 fields, saw 8\\nSkipping line 1371370: expected 7 fields, saw 8\\nSkipping line 1372637: expected 7 fields, saw 8\\nSkipping line 1372638: expected 7 fields, saw 8\\nSkipping line 1374467: expected 7 fields, saw 8\\nSkipping line 1375169: expected 7 fields, saw 8\\nSkipping line 1376613: expected 7 fields, saw 8\\nSkipping line 1377192: expected 7 fields, saw 8\\nSkipping line 1378571: expected 7 fields, saw 8\\nSkipping line 1378907: expected 7 fields, saw 8\\nSkipping line 1384122: expected 7 fields, saw 8\\nSkipping line 1384276: expected 7 fields, saw 8\\nSkipping line 1389980: expected 7 fields, saw 8\\nSkipping line 1390232: expected 7 fields, saw 8\\nSkipping line 1396626: expected 7 fields, saw 8\\nSkipping line 1399781: expected 7 fields, saw 8\\nSkipping line 1407500: expected 7 fields, saw 8\\nSkipping line 1414595: expected 7 fields, saw 8\\nSkipping line 1415290: expected 7 fields, saw 8\\nSkipping line 1416413: expected 7 fields, saw 8\\nSkipping line 1420305: expected 7 fields, saw 8\\nSkipping line 1425619: expected 7 fields, saw 8\\nSkipping line 1429407: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 1443250: expected 7 fields, saw 8\\nSkipping line 1445545: expected 7 fields, saw 8\\nSkipping line 1453337: expected 7 fields, saw 8\\nSkipping line 1453667: expected 7 fields, saw 8\\nSkipping line 1453668: expected 7 fields, saw 8\\nSkipping line 1459219: expected 7 fields, saw 8\\nSkipping line 1459222: expected 7 fields, saw 8\\nSkipping line 1461262: expected 7 fields, saw 8\\nSkipping line 1461981: expected 7 fields, saw 8\\nSkipping line 1463407: expected 7 fields, saw 8\\nSkipping line 1464912: expected 7 fields, saw 8\\nSkipping line 1464913: expected 7 fields, saw 8\\nSkipping line 1469784: expected 7 fields, saw 8\\nSkipping line 1470549: expected 7 fields, saw 8\\nSkipping line 1471059: expected 7 fields, saw 8\\nSkipping line 1472314: expected 7 fields, saw 8\\nSkipping line 1474037: expected 7 fields, saw 8\\nSkipping line 1485135: expected 7 fields, saw 8\\nSkipping line 1498917: expected 7 fields, saw 8\\nSkipping line 1500946: expected 7 fields, saw 8\\nSkipping line 1504589: expected 7 fields, saw 8\\nSkipping line 1509456: expected 7 fields, saw 8\\nSkipping line 1514459: expected 7 fields, saw 8\\nSkipping line 1516239: expected 7 fields, saw 8\\nSkipping line 1517687: expected 7 fields, saw 8\\nSkipping line 1518986: expected 7 fields, saw 8\\nSkipping line 1522622: expected 7 fields, saw 8\\nSkipping line 1525579: expected 7 fields, saw 8\\nSkipping line 1525935: expected 7 fields, saw 8\\nSkipping line 1525936: expected 7 fields, saw 8\\nSkipping line 1528271: expected 7 fields, saw 8\\nSkipping line 1529466: expected 7 fields, saw 8\\nSkipping line 1540570: expected 7 fields, saw 8\\nSkipping line 1543892: expected 7 fields, saw 8\\nSkipping line 1546094: expected 7 fields, saw 8\\nSkipping line 1550162: expected 7 fields, saw 8\\nSkipping line 1551220: expected 7 fields, saw 8\\nSkipping line 1552407: expected 7 fields, saw 8\\nSkipping line 1559612: expected 7 fields, saw 8\\nSkipping line 1560549: expected 7 fields, saw 8\\nSkipping line 1564978: expected 7 fields, saw 8\\nSkipping line 1564979: expected 7 fields, saw 8\\nSkipping line 1568525: expected 7 fields, saw 8\\nSkipping line 1571714: expected 7 fields, saw 8\\nSkipping line 1572583: expected 7 fields, saw 8\\n'\n",
      "b'Skipping line 1576298: expected 7 fields, saw 8\\nSkipping line 1577900: expected 7 fields, saw 8\\nSkipping line 1577901: expected 7 fields, saw 8\\nSkipping line 1588050: expected 7 fields, saw 8\\nSkipping line 1591771: expected 7 fields, saw 8\\nSkipping line 1593715: expected 7 fields, saw 8\\nSkipping line 1599342: expected 7 fields, saw 8\\nSkipping line 1601946: expected 7 fields, saw 8\\nSkipping line 1603184: expected 7 fields, saw 8\\nSkipping line 1603186: expected 7 fields, saw 8\\nSkipping line 1609797: expected 7 fields, saw 8\\nSkipping line 1612335: expected 7 fields, saw 8\\nSkipping line 1613377: expected 7 fields, saw 8\\nSkipping line 1614478: expected 7 fields, saw 8\\nSkipping line 1616283: expected 7 fields, saw 8\\nSkipping line 1616357: expected 7 fields, saw 8\\nSkipping line 1617018: expected 7 fields, saw 8\\nSkipping line 1624644: expected 7 fields, saw 8\\nSkipping line 1626841: expected 7 fields, saw 8\\nSkipping line 1631675: expected 7 fields, saw 8\\nSkipping line 1642443: expected 7 fields, saw 8\\nSkipping line 1645077: expected 7 fields, saw 8\\nSkipping line 1645303: expected 7 fields, saw 8\\nSkipping line 1645743: expected 7 fields, saw 8\\nSkipping line 1646027: expected 7 fields, saw 8\\nSkipping line 1646432: expected 7 fields, saw 8\\nSkipping line 1647875: expected 7 fields, saw 8\\nSkipping line 1649359: expected 7 fields, saw 8\\nSkipping line 1659554: expected 7 fields, saw 8\\n'\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv('IATE_en_de.csv', delimiter='|', error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1659663"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>E_ID</th>\n",
       "      <th>E_DOMAINS</th>\n",
       "      <th>L_CODE</th>\n",
       "      <th>T_TERM</th>\n",
       "      <th>T_TYPE</th>\n",
       "      <th>T_RELIABILITY</th>\n",
       "      <th>T_EVALUATION</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>234067</td>\n",
       "      <td>TRANSPORT</td>\n",
       "      <td>en</td>\n",
       "      <td>nonprecision</td>\n",
       "      <td>Term</td>\n",
       "      <td>Reliability not verified</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>254003</td>\n",
       "      <td>linguistics</td>\n",
       "      <td>en</td>\n",
       "      <td>indirect speech act</td>\n",
       "      <td>Term</td>\n",
       "      <td>Reliability not verified</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>254003</td>\n",
       "      <td>linguistics</td>\n",
       "      <td>de</td>\n",
       "      <td>indirekter Sprechakt</td>\n",
       "      <td>Term</td>\n",
       "      <td>Reliability not verified</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1602897</td>\n",
       "      <td>electronics and electrical engineering</td>\n",
       "      <td>en</td>\n",
       "      <td>antenna discrimination</td>\n",
       "      <td>Term</td>\n",
       "      <td>Reliable</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1602897</td>\n",
       "      <td>electronics and electrical engineering</td>\n",
       "      <td>en</td>\n",
       "      <td>aerial discrimination</td>\n",
       "      <td>Term</td>\n",
       "      <td>Reliable</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      E_ID                               E_DOMAINS L_CODE  \\\n",
       "0   234067                               TRANSPORT     en   \n",
       "1   254003                             linguistics     en   \n",
       "2   254003                             linguistics     de   \n",
       "3  1602897  electronics and electrical engineering     en   \n",
       "4  1602897  electronics and electrical engineering     en   \n",
       "\n",
       "                   T_TERM T_TYPE             T_RELIABILITY T_EVALUATION  \n",
       "0            nonprecision   Term  Reliability not verified          NaN  \n",
       "1     indirect speech act   Term  Reliability not verified          NaN  \n",
       "2    indirekter Sprechakt   Term  Reliability not verified          NaN  \n",
       "3  antenna discrimination   Term                  Reliable          NaN  \n",
       "4   aerial discrimination   Term                  Reliable          NaN  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29821\n"
     ]
    }
   ],
   "source": [
    "print(len(np.unique(data['E_DOMAINS'].values)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('AGRICULTURE, FORESTRY AND FISHERIES', 70685),\n",
       " ('medical science', 61678),\n",
       " ('electronics and electrical engineering', 56457),\n",
       " ('Domain code not specified', 54506),\n",
       " ('LAW', 48385)]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(data['E_DOMAINS'].values).most_common(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('en', 988440), ('de', 671223)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(data['L_CODE'].values).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Term', 1552151),\n",
       " ('Abbrev', 81767),\n",
       " ('Phrase', 17496),\n",
       " ('Short Form', 7755),\n",
       " ('Appellation', 440),\n",
       " ('Formula', 54)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(data['T_TYPE'].values).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Reliable', 1248775),\n",
       " ('Reliability not verified', 202914),\n",
       " ('Minimum reliability', 187518),\n",
       " ('Very reliable', 20456)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(data['T_RELIABILITY'].values).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1659663/1659663\r"
     ]
    }
   ],
   "source": [
    "en = {}\n",
    "de = {}\n",
    "for idx, line in enumerate(data.values):\n",
    "    if line[2]=='en':\n",
    "        if line[0] in en.keys():\n",
    "            en[line[0]].append(line[3])\n",
    "        else:\n",
    "            en[line[0]] = [line[3]]\n",
    "    if line[2]=='de':\n",
    "        if line[0] in de.keys():\n",
    "            de[line[0]].append(line[3])\n",
    "        else:\n",
    "            de[line[0]] = [line[3]]\n",
    "    print(f'{idx+1}/{len(data)}', end='\\r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "769630"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(en) # 769630"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "526001"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(de) # 526001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "814434/814434\r"
     ]
    }
   ],
   "source": [
    "E_ID = np.unique(data['E_ID'].values)\n",
    "\n",
    "en_de_dictionary = {}\n",
    "for idx, id_ in enumerate(E_ID):\n",
    "    if id_ in en.keys() and id_ in de.keys():\n",
    "        for en_word in en[id_]:\n",
    "            if len(de[id_]) > 0:\n",
    "                en_de_dictionary[en_word] = de[id_]\n",
    "    print(f'{idx+1}/{len(E_ID)}', end='\\r') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "515480"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(en_de_dictionary) # 515480"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('iate_en_de_all.json', 'w', encoding='UTF-8-sig') as fp:\n",
    "    json.dump(en_de_dictionary, fp, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}