contextual.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PLAN\n",
    "\n",
    "## 1) Elmo embeddings in Keras - avec les données de l'exemple\n",
    "\n",
    "## 2) Elmo embeddings in Keras - avec les données Toxic Comment\n",
    "\n",
    "## 3) Based on another example code\n",
    "\n",
    "## 4) Actually works, adapted from 3) ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1) Elmo embeddings in Keras - avec les données de l'exemple\n",
    "\n",
    "https://towardsdatascience.com/elmo-embeddings-in-keras-with-tensorflow-hub-7eb6f0145440\n",
    "\n",
    "goes with https://github.com/strongio/keras-elmo/blob/master/Elmo%20Keras.ipynb\n",
    "\n",
    "importing https://tfhub.dev/google/elmo/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Import our dependencies\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import tensorflow_hub as hub\n",
    "import os\n",
    "import re\n",
    "from keras import backend as K\n",
    "import keras.layers as layers\n",
    "from keras.models import Model, load_model\n",
    "from keras.engine import Layer\n",
    "import numpy as np\n",
    "\n",
    "# Initialize session\n",
    "sess = tf.Session()\n",
    "K.set_session(sess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chargement des données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n",
      "84131840/84125825 [==============================] - 14s 0us/step\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Humphrey Bogart clearly did not want to be in ...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>*SPOILERS* Four men, Ed (Jon Voight), Lewis (B...</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I can honestly say I never expected this movie...</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>This film came recommended as a good action fi...</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dear Richard, I know we all loved you on Home ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence sentiment  polarity\n",
       "0  Humphrey Bogart clearly did not want to be in ...         3         0\n",
       "1  *SPOILERS* Four men, Ed (Jon Voight), Lewis (B...         9         1\n",
       "2  I can honestly say I never expected this movie...         8         1\n",
       "3  This film came recommended as a good action fi...         4         0\n",
       "4  Dear Richard, I know we all loved you on Home ...         1         0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load all files from a directory in a DataFrame.\n",
    "def load_directory_data(directory):\n",
    "  data = {}\n",
    "  data[\"sentence\"] = []\n",
    "  data[\"sentiment\"] = []\n",
    "  for file_path in os.listdir(directory):\n",
    "    with tf.gfile.GFile(os.path.join(directory, file_path), \"r\") as f:\n",
    "      data[\"sentence\"].append(f.read())\n",
    "      data[\"sentiment\"].append(re.match(\"\\d+_(\\d+)\\.txt\", file_path).group(1))\n",
    "  return pd.DataFrame.from_dict(data)\n",
    "\n",
    "# Merge positive and negative examples, add a polarity column and shuffle.\n",
    "def load_dataset(directory):\n",
    "  pos_df = load_directory_data(os.path.join(directory, \"pos\"))\n",
    "  neg_df = load_directory_data(os.path.join(directory, \"neg\"))\n",
    "  pos_df[\"polarity\"] = 1\n",
    "  neg_df[\"polarity\"] = 0\n",
    "  return pd.concat([pos_df, neg_df]).sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "# Download and process the dataset files.\n",
    "def download_and_load_datasets(force_download=False):\n",
    "  dataset = tf.keras.utils.get_file(\n",
    "      fname=\"aclImdb.tar.gz\", \n",
    "      origin=\"http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\", \n",
    "      extract=True)\n",
    "\n",
    "  train_df = load_dataset(os.path.join(os.path.dirname(dataset), \n",
    "                                       \"aclImdb\", \"train\"))\n",
    "  test_df = load_dataset(os.path.join(os.path.dirname(dataset), \n",
    "                                      \"aclImdb\", \"test\"))\n",
    "\n",
    "  return train_df, test_df\n",
    "\n",
    "# Reduce logging output.\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)\n",
    "\n",
    "train_df, test_df = download_and_load_datasets()\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Définition du modèle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ElmoEmbeddingLayer(Layer):\n",
    "    def __init__(self, **kwargs):\n",
    "        self.dimensions = 1024\n",
    "        self.trainable=True\n",
    "        super(ElmoEmbeddingLayer, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.elmo = hub.Module('https://tfhub.dev/google/elmo/2', trainable=self.trainable,\n",
    "                               name=\"{}_module\".format(self.name))\n",
    "\n",
    "        self.trainable_weights += K.tf.trainable_variables(scope=\"^{}_module/.*\".format(self.name))\n",
    "        super(ElmoEmbeddingLayer, self).build(input_shape)\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        result = self.elmo(K.squeeze(K.cast(x, tf.string), axis=1),\n",
    "                      as_dict=True,\n",
    "                      signature='default',\n",
    "                      )['default']\n",
    "        return result\n",
    "\n",
    "    def compute_mask(self, inputs, mask=None):\n",
    "        return K.not_equal(inputs, '--PAD--')\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        print(input_shape[0])\n",
    "        return (input_shape[0], self.dimensions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to build model\n",
    "\n",
    "def build_model(): \n",
    "  input_text = layers.Input(shape=(1,), dtype=\"string\")\n",
    "  embedding = ElmoEmbeddingLayer()(input_text)\n",
    "  dense = layers.Dense(256, activation='relu')(embedding)\n",
    "  pred = layers.Dense(1, activation='sigmoid')(dense)\n",
    "\n",
    "  model = Model(inputs=[input_text], outputs=pred)\n",
    "\n",
    "  model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "  model.summary()\n",
    "  \n",
    "  return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create datasets (Only take up to 150 words for memory)\n",
    "train_text = train_df['sentence'].tolist()\n",
    "train_text = [' '.join(t.split()[0:150]) for t in train_text]\n",
    "train_text = np.array(train_text, dtype=object)[:, np.newaxis]\n",
    "train_label = train_df['polarity'].tolist()\n",
    "\n",
    "test_text = test_df['sentence'].tolist()\n",
    "test_text = [' '.join(t.split()[0:150]) for t in test_text]\n",
    "test_text = np.array(test_text, dtype=object)[:, np.newaxis]\n",
    "test_label = test_df['polarity'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Humphrey Bogart clearly did not want to be in this film, and be forced to play a part-Mexican or he would have been suspended. Believe me , he made the wrong choice! Presumably, after the success of \"Dodge City\", Warners tried a follow-up with Errol Flynn and his usual list of buddies, like Alan Hale, Guinn (Big Boy) Williams, Frank Mc Hugh and the ever-present John Litel, but they made the huge mistake of trying to present Miriam Hopkins as a love interest for Flynn v. Randolph Scott, and as a singer to really make things bad, because she proved one thing, and that is she cannot sing. The story was not too bad, but with Bogie clearly miscast also, it turned out to be a poor Western that was overlong, and on a low budget, but in fairness, color would not have helped.'],\n",
       "       [\"*SPOILERS* Four men, Ed (Jon Voight), Lewis (Burt Reynolds), Drew (Ronny Cox) and Bobby (Ned Beatty), decide to go on a rafting trip on the Cahulawassee river, before it is flooded.<br /><br />They wanted to have fun, to have a nice weekend in the nature.<br /><br />But when two mountain men cross their path and rape one of them (Bobby), everything begins to go to Hell in a Handbasket, and this 'nice weekend' will even cost one of the four's life...<br /><br />'Deliverance', which in Italian is stupidly titled 'Un Tranquillo Weekend Di Paura' ('A Calm Weekend Of Fear'), is the Grandad of movies like 'Texas Chainsaw Massacre', 'The Hills Have Eyes', 'Wrong Turn', 'Last House On The Left' and all the other 'Evil Nature/Revenge' subgenre films, and one of the scariest, right next to 'The Hills Have Eyes'.<br /><br />Based on a book by James Dickey (who appears in\"],\n",
       "       [\"I can honestly say I never expected this movie to be good. I do not like family films. I am far from a fan of Shahid Kapoor. The director's last movie (MPKDH was complete stupidity. And the music was very boring and bland. But there was Amrita Rao, who has become my favorite after Main Hoon Na. There was also Seema Biswas, Alok Nath, and Anupam Kher who are very talented. So a few plus points.<br /><br />I finally saw the movie and I was very impressed. He brings us the young lovers of MPK with a pinch of the HAHK wedding, and we have a winner. The story outline is similar to HAHK, light hearted in the beginning to serious mode at the end. The director also made character that you could relate to. The thing I did not like about HAHK was that the characters were too eccentric.\"],\n",
       "       ...,\n",
       "       ['\"Le Locataire\"(\"The Tenant\")is without a doubt one of the most important horror movies ever made.Polanski stars as a Trelkovsky,a timid file clerk living in Paris,who answers an advertisement for an apartment,only to find that the previous tenant attempted suicide by leaping from the apartment window.Trelkovsky is compelled to visit her in the hospital and there he meets Stella(Isabelle Adjani).Trelkovsky immediately moves in when the previous tenant dies and,at first,is quite pleased with having found such a nice apartment.His happiness is soon replaced by waves of paranoia as he becomes increasingly suspicious of his neighbours,who seem to be trying to provoke Trelkovsky into repeating the previous tenant\\'s suicide.This film is great.Polanski manages to create a surreal atmosphere of dread and paranoia.Plenty of brilliant moments such as the classic scene where Trelkovsky discovers the previous tenant\\'s tooth in a hole in the wall,or the fever dream where he wanders into the building\\'s'],\n",
       "       ['Finally we get a TV series where we get to see the acting talent! Episode one was excellent! The script gave us a little more than usual, yeah, there was still the \"i\\'m not your father -i\\'m your father and omigod you cheated on me!\" rubbish but the script allowed the actors to actually feel and live those real moments rather than show us what it would feel like if -like so many TV soaps do. <br /><br />The camera work also gave us a little more than usual, there were no boring shots of repeated angles for hours yet there was no unnecessary \\'shots inside shots or hand-held camera crap\\' to add an \"artistic\" edge it gave us what we needed to see and also some beautiful scenery pictures as well! <br /><br />Nothing was over-dramatised or melodramatic they were real people in a real place dealing with real'],\n",
       "       [\"This movie is tremendous for uplifting the Spirits.<br /><br />Every time I watch it, I see & hear funny little things that I missed before.<br /><br />The soundtrack is unbelievable. Mick Jones (Foreigner) and Chris Difford (Squeeze) penned the songs, making Strange Fruit the best thing that ever hit today's music scene.<br /><br />Unfortunately, Strange Fruit are a strictly fictitional washed up '60's to 70's band that were never good to begin with, due to drug use and inner fighting. One wonders what might have been, while listening to their fanatstic soundtrack.<br /><br />The Fruit draw inspiration from The Rolling Stones, Deep Purple, David Bowie, and The Who.<br /><br />Each member of Fruit are quite memorable. Stephen Rea stars as down-and-dead-broke Tony Costello, who is asked by a festival promoter to reunite his band for a reunion tour, with hopes of reaping monetary benefits. Costello haply approaches ex-roadie Karen Knowles,\"]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 1)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_text.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Humphrey Bogart clearly did not want to be in this film, and be forced to play a part-Mexican or he would have been suspended. Believe me , he made the wrong choice! Presumably, after the success of \"Dodge City\", Warners tried a follow-up with Errol Flynn and his usual list of buddies, like Alan Hale, Guinn (Big Boy) Williams, Frank Mc Hugh and the ever-present John Litel, but they made the huge mistake of trying to present Miriam Hopkins as a love interest for Flynn v. Randolph Scott, and as a singer to really make things bad, because she proved one thing, and that is she cannot sing. The story was not too bad, but with Bogie clearly miscast also, it turned out to be a poor Western that was overlong, and on a low budget, but in fairness, color would not have helped.'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_text[0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25000"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_label[:40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 1)                 0         \n",
      "_________________________________________________________________\n",
      "elmo_embedding_layer_2 (Elmo (None, 1024)              4         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 256)               262400    \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 262,661\n",
      "Trainable params: 262,661\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Build and fit\n",
    "model = build_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 1)                 0         \n",
      "_________________________________________________________________\n",
      "elmo_embedding_layer_1 (Elmo (None, 1024)              4         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               262400    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 262,661\n",
      "Trainable params: 262,661\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/1\n",
      "   64/25000 [..............................] - ETA: 5:11:29 - loss: 0.7168 - acc: 0.5000"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-1393054ca340>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m           \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_text\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_label\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m           batch_size=32)\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(train_text, \n",
    "          train_label,\n",
    "          validation_data=(test_text, test_label),\n",
    "          epochs=1,\n",
    "          batch_size=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2) Elmo embeddings in Keras - avec les données Toxic Comment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Import à nous\n",
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "from keras.preprocessing import text, sequence\n",
    "\n",
    "from tools import *\n",
    "from embeddings import *\n",
    "from models import *\n",
    "\n",
    "\n",
    "# Import du tutorial sur Elmo\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import tensorflow_hub as hub\n",
    "import os\n",
    "import re\n",
    "from keras import backend as K\n",
    "import keras.layers as layers\n",
    "from keras.models import Model, load_model\n",
    "from keras.engine import Layer\n",
    "import numpy as np\n",
    "\n",
    "# Initialize session\n",
    "sess = tf.Session()\n",
    "K.set_session(sess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import des données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load raw string data\n",
    "data_train, y_train_all, data_test, id_test = load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create datasets (Only take up to 150 words for memory)\n",
    "train_text = [' '.join(t.split()[0:150]) for t in data_train]\n",
    "train_text = np.array(train_text, dtype=object)[:, np.newaxis]\n",
    "train_label = y_train_all.tolist()\n",
    "\n",
    "test_text = [' '.join(t.split()[0:150]) for t in data_test]\n",
    "test_text = np.array(test_text, dtype=object)[:, np.newaxis]\n",
    "\n",
    "# pas ici, on se passe de validation_data pour model\n",
    "# test_label = test_df['polarity'].tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Définition du modèle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ElmoEmbeddingLayer(Layer):\n",
    "    def __init__(self, **kwargs):\n",
    "        self.dimensions = 1024\n",
    "        self.trainable=True\n",
    "        super(ElmoEmbeddingLayer, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.elmo = hub.Module('https://tfhub.dev/google/elmo/2', trainable=self.trainable,\n",
    "                               name=\"{}_module\".format(self.name))\n",
    "\n",
    "        self.trainable_weights += K.tf.trainable_variables(scope=\"^{}_module/.*\".format(self.name))\n",
    "        super(ElmoEmbeddingLayer, self).build(input_shape)\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        result = self.elmo(K.squeeze(K.cast(x, tf.string), axis=1),\n",
    "                      as_dict=True,\n",
    "                      signature='default',\n",
    "                      )['default']\n",
    "        return result\n",
    "\n",
    "    def compute_mask(self, inputs, mask=None):\n",
    "        return K.not_equal(inputs, '--PAD--')\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "#         return (input_shape[0], self.dimensions)\n",
    "        return (1, self.dimensions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to build model\n",
    "\n",
    "def build_model(): \n",
    "  input_text = layers.Input(shape=(1,), dtype=\"string\")\n",
    "  embedding = ElmoEmbeddingLayer()(input_text)\n",
    "  dense = layers.Dense(256, activation='relu')(embedding)\n",
    "  pred = layers.Dense(6, activation='sigmoid')(dense)\n",
    "\n",
    "  model = Model(inputs=[input_text], outputs=pred)\n",
    "\n",
    "  model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "  model.summary()\n",
    "  \n",
    "  return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using /tmp/tfhub_modules to cache modules.\n",
      "WARNING:tensorflow:From /home/mbauw/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/control_flow_ops.py:3632: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 1)                 0         \n",
      "_________________________________________________________________\n",
      "elmo_embedding_layer_1 (Elmo (1, 1024)                 4         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (1, 256)                  262400    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (1, 6)                    1542      \n",
      "=================================================================\n",
      "Total params: 263,946\n",
      "Trainable params: 263,946\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Error when checking model target: the list of Numpy arrays that you are passing to your model is not the size the model expected. Expected to see 1 array(s), but instead got the following list of 159571 arrays: [array([[0],\n       [0],\n       [0],\n       [0],\n       [0],\n       [0]]), array([[0],\n       [0],\n       [0],\n       [0],\n       [0],\n       [0]]), array([[0],\n       [0],\n       [0],\n       [0],\n   ...",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-2e522487c0e1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m           \u001b[0mtrain_label\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m           batch_size=32)\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m    950\u001b[0m             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    951\u001b[0m             \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 952\u001b[0;31m             batch_size=batch_size)\n\u001b[0m\u001b[1;32m    953\u001b[0m         \u001b[0;31m# Prepare validation data.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    954\u001b[0m         \u001b[0mdo_validation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_user_data\u001b[0;34m(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)\u001b[0m\n\u001b[1;32m    787\u001b[0m                 \u001b[0mfeed_output_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m                 \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# Don't enforce the batch size.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 789\u001b[0;31m                 exception_prefix='target')\n\u001b[0m\u001b[1;32m    790\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m             \u001b[0;31m# Generate sample-wise weight values given the `sample_weight` and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training_utils.py\u001b[0m in \u001b[0;36mstandardize_input_data\u001b[0;34m(data, names, shapes, check_batch_axis, exception_prefix)\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0;34m'Expected to see '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' array(s), '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m                 \u001b[0;34m'but instead got the following list of '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 102\u001b[0;31m                 str(len(data)) + ' arrays: ' + str(data)[:200] + '...')\n\u001b[0m\u001b[1;32m    103\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m             raise ValueError(\n",
      "\u001b[0;31mValueError\u001b[0m: Error when checking model target: the list of Numpy arrays that you are passing to your model is not the size the model expected. Expected to see 1 array(s), but instead got the following list of 159571 arrays: [array([[0],\n       [0],\n       [0],\n       [0],\n       [0],\n       [0]]), array([[0],\n       [0],\n       [0],\n       [0],\n       [0],\n       [0]]), array([[0],\n       [0],\n       [0],\n       [0],\n   ..."
     ]
    }
   ],
   "source": [
    "# model.fit(train_text, \n",
    "#           train_label,\n",
    "#           validation_data=(test_text, test_label),\n",
    "#           epochs=1,\n",
    "#           batch_size=32)\n",
    "\n",
    "# on n'a pas le test label nous, donc pas de validation_data\n",
    "model = build_model()\n",
    "model.fit(train_text, \n",
    "          train_label,\n",
    "          epochs=1,\n",
    "          batch_size=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3) Based on another example code\n",
    "\n",
    "https://github.com/tensorflow/hub/issues/149\n",
    "\n",
    "https://github.com/PrashantRanjan09/Elmo-Tutorial\n",
    "\n",
    "https://github.com/PrashantRanjan09/WordEmbeddings-Elmo-Fasttext-Word2Vec/blob/master/word_embeddings.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Import à nous\n",
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "from keras.preprocessing import text, sequence\n",
    "\n",
    "from tools import *\n",
    "from embeddings import *\n",
    "from models import *\n",
    "\n",
    "\n",
    "# Import du tutorial sur Elmo\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import tensorflow_hub as hub\n",
    "import os\n",
    "import re\n",
    "from keras import backend as K\n",
    "import keras.layers as layers\n",
    "from keras.models import Model, load_model\n",
    "from keras.engine import Layer\n",
    "import numpy as np\n",
    "\n",
    "# Initialize session\n",
    "sess = tf.Session()\n",
    "K.set_session(sess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chargement des données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load raw string data\n",
    "data_train, y_train_all, data_test, id_test = load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create datasets (Only take up to 150 words for memory)\n",
    "train_text = [' '.join(t.split()[0:150]) for t in data_train]\n",
    "train_text = np.array(train_text, dtype=object)[:, np.newaxis]\n",
    "train_label = y_train_all.tolist()\n",
    "\n",
    "test_text = [' '.join(t.split()[0:150]) for t in data_test]\n",
    "test_text = np.array(test_text, dtype=object)[:, np.newaxis]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Elmo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ELMoEmbedding(x):\n",
    "    elmo_model = hub.Module(\"https://tfhub.dev/google/elmo/1\", trainable=True)\n",
    "    return elmo_model(tf.squeeze(tf.cast(x, tf.string)), signature=\"default\", as_dict=True)[\"default\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using /tmp/tfhub_modules to cache modules.\n",
      "WARNING:tensorflow:From /home/mbauw/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/control_flow_ops.py:3632: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
     ]
    }
   ],
   "source": [
    "input_text = layers.Input(shape=(1,), dtype=tf.string)\n",
    "embed_seq = layers.Lambda(ELMoEmbedding, output_shape=(1024,))(input_text)\n",
    "x = Dense(256,activation =\"relu\")(embed_seq)\n",
    "preds = Dense(6,activation=\"sigmoid\")(x)\n",
    "\n",
    "model = Model(input_text,preds)\n",
    "model.compile(loss=\"binary_crossentropy\",optimizer=\"adam\",metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Error when checking model target: the list of Numpy arrays that you are passing to your model is not the size the model expected. Expected to see 1 array(s), but instead got the following list of 159571 arrays: [array([[0],\n       [0],\n       [0],\n       [0],\n       [0],\n       [0]]), array([[0],\n       [0],\n       [0],\n       [0],\n       [0],\n       [0]]), array([[0],\n       [0],\n       [0],\n       [0],\n   ...",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-bc9973486367>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# model.fit(X_train_pad,y_train,epochs=10,batch_size=512,validation_data=(X_test_pad,y_test))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_text\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtrain_label\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m512\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m    950\u001b[0m             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    951\u001b[0m             \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 952\u001b[0;31m             batch_size=batch_size)\n\u001b[0m\u001b[1;32m    953\u001b[0m         \u001b[0;31m# Prepare validation data.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    954\u001b[0m         \u001b[0mdo_validation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_user_data\u001b[0;34m(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)\u001b[0m\n\u001b[1;32m    787\u001b[0m                 \u001b[0mfeed_output_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m                 \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# Don't enforce the batch size.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 789\u001b[0;31m                 exception_prefix='target')\n\u001b[0m\u001b[1;32m    790\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m             \u001b[0;31m# Generate sample-wise weight values given the `sample_weight` and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training_utils.py\u001b[0m in \u001b[0;36mstandardize_input_data\u001b[0;34m(data, names, shapes, check_batch_axis, exception_prefix)\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0;34m'Expected to see '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' array(s), '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m                 \u001b[0;34m'but instead got the following list of '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 102\u001b[0;31m                 str(len(data)) + ' arrays: ' + str(data)[:200] + '...')\n\u001b[0m\u001b[1;32m    103\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m             raise ValueError(\n",
      "\u001b[0;31mValueError\u001b[0m: Error when checking model target: the list of Numpy arrays that you are passing to your model is not the size the model expected. Expected to see 1 array(s), but instead got the following list of 159571 arrays: [array([[0],\n       [0],\n       [0],\n       [0],\n       [0],\n       [0]]), array([[0],\n       [0],\n       [0],\n       [0],\n       [0],\n       [0]]), array([[0],\n       [0],\n       [0],\n       [0],\n   ..."
     ]
    }
   ],
   "source": [
    "# model.fit(X_train_pad,y_train,epochs=10,batch_size=512,validation_data=(X_test_pad,y_test))\n",
    "\n",
    "model.fit(train_text,train_label,epochs=2,batch_size=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = model.predict(X_test_pad)\n",
    "predictions = [0 if i<0.5 else 1 for i in predictions]\n",
    "print(\"Accuracy: \",accuracy_score(y_test,predictions))\n",
    "print(\"Classification Report: \",classification_report(y_test,predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4) Actually works, adapted from 3) ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import à nous\n",
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "from keras.preprocessing import text, sequence\n",
    "\n",
    "from tools import *\n",
    "from embeddings import *\n",
    "from models import *\n",
    "\n",
    "\n",
    "# Import du tutorial sur Elmo\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import tensorflow_hub as hub\n",
    "import os\n",
    "import re\n",
    "from keras import backend as K\n",
    "import keras.layers as layers\n",
    "from keras.models import Model, load_model\n",
    "from keras.engine import Layer\n",
    "import numpy as np\n",
    "\n",
    "# Initialize session\n",
    "sess = tf.Session()\n",
    "K.set_session(sess)\n",
    "\n",
    "# load raw string data\n",
    "data_train, y_train_all, data_test, id_test = load_data()\n",
    "\n",
    "# Create datasets (Only take up to 150 words for memory)\n",
    "train_text = [' '.join(t.split()[0:150]) for t in data_train]\n",
    "train_text = np.array(train_text, dtype=object)[:, np.newaxis]\n",
    "train_label = y_train_all\n",
    "\n",
    "test_text = [' '.join(t.split()[0:150]) for t in data_test]\n",
    "test_text = np.array(test_text, dtype=object)[:, np.newaxis]\n",
    "\n",
    "def ELMoEmbedding(x):\n",
    "    elmo_model = hub.Module(\"https://tfhub.dev/google/elmo/1\", trainable=True)\n",
    "    return elmo_model(tf.squeeze(tf.cast(x, tf.string)), signature=\"default\", as_dict=True)[\"default\"]\n",
    "\n",
    "input_text = layers.Input(shape=(1,), dtype=tf.string)\n",
    "embed_seq = layers.Lambda(ELMoEmbedding, output_shape=(1024,))(input_text)\n",
    "x = Dense(256,activation =\"relu\")(embed_seq)\n",
    "preds = Dense(6,activation=\"sigmoid\")(x)\n",
    "\n",
    "model = Model(input_text,preds)\n",
    "model.compile(loss=\"binary_crossentropy\",optimizer=\"adam\",metrics=[\"accuracy\"])\n",
    "\n",
    "# model.fit(X_train_pad,y_train,epochs=10,batch_size=512,validation_data=(X_test_pad,y_test))\n",
    "\n",
    "model.fit(train_text,train_label,epochs=2,batch_size=32)\n",
    "\n",
    "# save trained nnet to disk for later use\n",
    "MODEL_NAME = \"elmo_sentence150_elmo1_dense256\"\n",
    "save_nnet(model, MODEL_NAME)\n",
    "\n",
    "# predict\n",
    "y_test_pred = model.predict(test_text, batch_size=1024)\n",
    "\n",
    "# write submission file\n",
    "submission(y_test_pred, id_test, name=MODEL_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}