ms_cntk_atis_dataset_reader.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ms-cntk-atis-dataset.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/github/kpe/notebooks/blob/master/ms_cntk_atis_dataset_reader.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "7w3LgfMrCdaT",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# MS CNTK ATIS DataSet Downloader\n",
        "\n",
        "Lets try to fetch the ATIS DataSet from the MS CNTK repo in github: https://github.com/Microsoft/CNTK/tree/master/Examples/LanguageUnderstanding/ATIS/Data\n",
        "\n",
        "and convert the data into a python friendly (pickle and text) format."
      ]
    },
    {
      "metadata": {
        "id": "Ik_Yi-P1CkVu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install tqdm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bAwZA1UZCdaU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from collections import defaultdict\n",
        "\n",
        "from tqdm import tqdm\n",
        "import numpy as np\n",
        "\n",
        "DATA_DIR='.model_data' # fetch location\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "R3paSftUCdaW",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## First download the raw files"
      ]
    },
    {
      "metadata": {
        "id": "boTmfHHNCdaX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "import urllib.request\n",
        "from urllib.parse import urlparse, urljoin\n",
        "\n",
        "ATIS_REPO_URL       = \"https://github.com/Microsoft/CNTK/raw/master\"\n",
        "ATIS_BASE_URL       = ATIS_REPO_URL+\"/Examples/LanguageUnderstanding/ATIS/Data/\"\n",
        "ATIS_EXTRA_BASE_URL = ATIS_REPO_URL+\"/Examples/LanguageUnderstanding/ATIS/BrainScript/\"\n",
        "\n",
        "ATIS_DS={\n",
        "    ATIS_BASE_URL: [\n",
        "        'ATIS.label',              # labels - labels_count:127\n",
        "        'ATIS.test.cntk.sparse',   # featurized\n",
        "        'ATIS.train.cntk.sparse',  # featurized\n",
        "        'ATIS.vocab',              # words - vocab_size: 944\n",
        "        'atis.test.ctf',\n",
        "        'atis.train.ctf'\n",
        "    ],\n",
        "    ATIS_EXTRA_BASE_URL: [\n",
        "        'query.wl',\n",
        "        'slots.wl',\n",
        "        'intent.wl'\n",
        "    ]\n",
        "}\n",
        "\n",
        "def fetch_ms_atis_ds():\n",
        "    for base_url,fnames in ATIS_DS.items():\n",
        "        for fname in fnames:\n",
        "            url = urljoin(base_url, fname)\n",
        "            fname = os.path.basename(urlparse(url).path)\n",
        "            loc_path = os.path.join(DATA_DIR, fname)\n",
        "            if os.path.isfile(loc_path):\n",
        "                print(\"skip downloading: {}\".format(fname))\n",
        "                continue\n",
        "            print(\"     downloading: {}\".format(fname))\n",
        "            if not os.path.isdir(os.path.dirname(loc_path)):\n",
        "                os.makedirs(os.path.dirname(loc_path))\n",
        "            urllib.request.urlretrieve(url, loc_path)\n",
        "            print(\"done downloading: {} bytes\".format(os.path.getsize(loc_path)))\n",
        "   \n",
        "fetch_ms_atis_ds()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "OCanMzAzCdaZ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## pip install the python CNTK"
      ]
    },
    {
      "metadata": {
        "id": "4sZJk_h3CdaZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install cntk"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "firaLmv6Cdac",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#\n",
        "# you might need:\n",
        "#   ln -s /usr/lib64/libmpi_cxx.so.20 /usr/local/lib64/libmpi_cxx.so.1\n",
        "#   ln -s /usr/lib64/libmpi.so.20 /usr/lib64/libmpi.so.12\n",
        "#\n",
        "import cntk\n",
        "\n",
        "def build_dicts():\n",
        "    query_wl   = [line.rstrip('\\n') for line in open(os.path.join(DATA_DIR,\"query.wl\"))]\n",
        "    slots_wl   = [line.rstrip('\\n') for line in open(os.path.join(DATA_DIR,\"slots.wl\"))]\n",
        "    intents_wl = [line.rstrip('\\n') for line in open(os.path.join(DATA_DIR,\"intent.wl\"))]\n",
        "    query_dict   = {query_wl[i]:i   for i in range(len(query_wl))}\n",
        "    slots_dict   = {slots_wl[i]:i   for i in range(len(slots_wl))}\n",
        "    intents_dict = {intents_wl[i]:i for i in range(len(intents_wl))}\n",
        "    return query_dict, slots_dict, intents_dict\n",
        "\n",
        "def create_ctf_reader(path):\n",
        "    vocab_size, num_labels, num_intents =  tuple(map(len, build_dicts()))\n",
        "    res = cntk.io.CTFDeserializer(path, cntk.io.StreamDefs(\n",
        "        query         = cntk.io.StreamDef(field='S0', shape=vocab_size,  is_sparse=True),\n",
        "        intent_labels = cntk.io.StreamDef(field='S1', shape=num_intents, is_sparse=True), \n",
        "        slot_labels   = cntk.io.StreamDef(field='S2', shape=num_labels,  is_sparse=True)\n",
        "    ))\n",
        "    return res\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2G4BA4BdCdad",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Build the lookup maps\n",
        "\n",
        "and do a minor sanity check"
      ]
    },
    {
      "metadata": {
        "id": "LJTxhjj3Cdae",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "qdict, sdict, idict = build_dicts()\n",
        "iqdict, isdict, iidict = map(lambda d: {d[k]:k for k in d.keys()}, build_dicts())\n",
        "\n",
        "# check indexes go from 0..len(ndxs)\n",
        "assert list(map(len, build_dicts())) == list(map(lambda d: 1+max(d.keys()), [iqdict,isdict,iidict]))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_GTn0rvoCdag",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Read the train and test DataSet files\n",
        "I had a hard time reading the CNTK data into anything, so be compassionate ..."
      ]
    },
    {
      "metadata": {
        "id": "VVg44CDsCdag",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def load_cntk_atis_ds(fname='atis.train.ctf'):\n",
        "    cr = create_ctf_reader(os.path.join(data_dir, fname))\n",
        "    mbs = cntk.io.MinibatchSource(cr, randomize=False, max_sweeps=1)\n",
        "\n",
        "    print('reading ATIS CTF file:',fname)\n",
        "    input_vars = {}\n",
        "    input_map = {}\n",
        "    dims={}\n",
        "    for key, val in cr['input'].items():\n",
        "        var = cntk.ops.input_variable(val['dim'], int, name=key)\n",
        "        input_map[key] = mbs.streams[key]\n",
        "        input_vars[key] = var\n",
        "        dims[key] = val['dim']\n",
        "        print('{:>15}: {}: {:3d} dims'.format(key, val['alias'], val['dim']))\n",
        "\n",
        "    ds=defaultdict(list)\n",
        "\n",
        "    count=0\n",
        "    with tqdm(desc='reading ATIS {}'.format(fname)) as pbar:\n",
        "        while True:\n",
        "            mb = mbs.next_minibatch(1,input_map=input_map)\n",
        "            if len(mb) == 0:\n",
        "                print(\"Found {} samples in DS: \".format(count, fname))\n",
        "                break\n",
        "            count += 1\n",
        "            pbar.update(count)\n",
        "\n",
        "            for name,var in input_vars.items():\n",
        "                val = mb[name].as_sequences(var)[0].toarray()\n",
        "                val = val.argmax(axis=-1)\n",
        "                ds[name].append(val)\n",
        "\n",
        "    for k,v in ds.items():\n",
        "        print('{:>13}: {:4d}: {:3d} max seq len | {:3d} dims'.format(k, len(v), max(map(len,v)), dims[k]))\n",
        "    \n",
        "    return ds"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "TDzg5JJOCdai",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dicts = {'token_ids':qdict,'slot_ids': sdict, 'intent_ids': idict}\n",
        "train_ds = load_cntk_atis_ds('atis.train.ctf')\n",
        "test_ds  = load_cntk_atis_ds('atis.test.ctf')\n",
        "\n",
        "# check dict sizes\n",
        "print('dicts',list(map(len,dicts.values())))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "q1RWsSgwCdak",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Store as pickle"
      ]
    },
    {
      "metadata": {
        "id": "ajtM4PMSCdal",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import gzip, pickle\n",
        "\n",
        "def store_ds(ds,dicts, fname='ms_cntk_atis.train.pkl.gz'):\n",
        "    with gzip.open(os.path.join(DATA_DIR, fname), 'wb') as stream:\n",
        "        pickle.dump((ds,dicts),stream, protocol=pickle.HIGHEST_PROTOCOL)\n",
        "    print('Done dumping: ', fname)\n",
        "\n",
        "def load_ds(fname='ms_cntk_atis.train.pkl.gz'):\n",
        "    with gzip.open(os.path.join(DATA_DIR, fname), 'rb') as stream:\n",
        "        ds,dicts = pickle.load(stream)\n",
        "    print('Done  loading: ', fname)\n",
        "    print('      samples: {:4d}'.format(len(ds['query'])))\n",
        "    print('   vocab_size: {:4d}'.format(len(dicts['token_ids'])))\n",
        "    print('   slot count: {:4d}'.format(len(dicts['slot_ids'])))\n",
        "    print(' intent count: {:4d}'.format(len(dicts['intent_ids'])))\n",
        "    return ds,dicts\n",
        "\n",
        "store_ds(train_ds, dicts, 'ms_cntk_atis.train.pkl.gz')\n",
        "store_ds(test_ds,  dicts, 'ms_cntk_atis.test.pkl.gz')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dCxTlVOsCdao",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Store as text"
      ]
    },
    {
      "metadata": {
        "id": "Oz5kGnYyCdaq",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def store_ds_dicts_to_csvs(dicts, fname='ms_cntk_atis.dict.%.csv.gz'):\n",
        "    t2i, s2i, in2i = map(dicts.get, ['token_ids', 'slot_ids','intent_ids'])\n",
        "    i2t, i2s, i2in = map(lambda d: {d[k]:k for k in d.keys()}, [t2i,s2i,in2i])    \n",
        "    \n",
        "    def store_dict_as_text(fname, items):\n",
        "        with gzip.open(fname, 'wt') as stream:\n",
        "            for it in items:\n",
        "                stream.write('{}\\n'.format(it))\n",
        "        print('done storing:', fname)\n",
        "        \n",
        "    store_dict_as_text(fname.replace('%', 'vocab'), map(i2t.get, range(len(i2t))))\n",
        "    store_dict_as_text(fname.replace('%', 'slots'), map(i2s.get, range(len(i2s))))\n",
        "    store_dict_as_text(fname.replace('%', 'intent'), map(i2in.get, range(len(i2in))))\n",
        "    \n",
        "def store_ds_to_csvs(ds, fname='ms_cntk_atis.%.train.csv.gz'):\n",
        "    def store_as_text(fname, store_item):\n",
        "        with gzip.open(fname, 'wt') as stream:\n",
        "            for i in range(len(ds['query'])):\n",
        "                store_item(stream, i)\n",
        "        print('done storing:', fname)\n",
        "        \n",
        "    query,intent,slots = map(ds.get, ['query','intent_labels', 'slot_labels'])\n",
        "    store_as_text(fname.replace('%','query'), \n",
        "                  lambda s, i: s.write(' '.join(map(str, query[i])) + '\\n'))\n",
        "    store_as_text(fname.replace('%','slots'), \n",
        "                  lambda s, i: s.write(' '.join(map(str, slots[i])) + '\\n'))\n",
        "    store_as_text(fname.replace('%','intent'), \n",
        "                  lambda s, i: s.write('{}\\n'.format(intent[i][0])))\n",
        "        \n",
        "def store_ds_and_dicts_to_csv(ds,dicts, \n",
        "                              fname='ms_cntk_atis.train.%.csv.gz', \n",
        "                              dname='ms_cntk_atis.dict.%.csv.gz'):\n",
        "    store_ds_to_csvs(ds,fname)\n",
        "    store_ds_dicts_to_csvs(dicts,dname)\n",
        "\n",
        "\n",
        "\n",
        "store_ds_and_dicts_to_csv(train_ds, dicts, 'ms_cntk_atis.train.%.csv.gz')\n",
        "store_ds_and_dicts_to_csv(test_ds,  dicts, 'ms_cntk_atis.test.%.csv.gz')\n",
        "   "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "gw_lNAWxCdat",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Uninstall CNTK"
      ]
    },
    {
      "metadata": {
        "id": "edSnhD9dCdau",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip uninstall -y cntk"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "L2hjosanCdax",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Read the pickle"
      ]
    },
    {
      "metadata": {
        "id": "RpLuV0bhCday",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "                \n",
        "train_ds, dicts = load_ds('ms_cntk_atis.train.pkl.gz')\n",
        "test_ds, dicts  = load_ds('ms_cntk_atis.test.pkl.gz')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bX2m4ngNCda1",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Show  head samples"
      ]
    },
    {
      "metadata": {
        "id": "Rz2_EQ5uCda3",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "t2i, s2i, in2i = map(dicts.get, ['token_ids', 'slot_ids','intent_ids'])\n",
        "i2t, i2s, i2in = map(lambda d: {d[k]:k for k in d.keys()}, [t2i,s2i,in2i])\n",
        "query, slots, intent =  map(train_ds.get, ['query', 'slot_labels', 'intent_labels'])\n",
        "\n",
        "for i in range(5):\n",
        "    print('{:4d}:{:>15}: {}'.format(i, i2in[intent[i][0]],\n",
        "                                    ' '.join(map(i2t.get, query[i]))))\n",
        "    for j in range(len(query[i])):\n",
        "        print('{:>33} {:>40}'.format(i2t[query[i][j]],\n",
        "                                     i2s[slots[i][j]]  ))\n",
        "    print('*'*74)\n",
        "                                "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3wyk6kwcCda6",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ORt3Bt5-Cda8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}