Mediacloud_Clustering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.cluster import MiniBatchKMeans, KMeans\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer\n",
    "# from sklearn.decomposition import PCA\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.manifold import TSNE\n",
    "# from sklearn.metrics import silhouette_score\n",
    "import ast \n",
    "# import re\n",
    "from gensim.corpora import Dictionary\n",
    "from gensim.models.coherencemodel import CoherenceModel\n",
    "import gensim\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from scipy.cluster.hierarchy import ward, dendrogram\n",
    "# from sklearn.feature_selection import chi2\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from collections import Counter\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.cm as cm\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "import os.path\n",
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.read_csv('preprocessed_results/mediacloud_parsed_corona_df_sep.csv').shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reading dataframes with parsed text and meta-information from the csv files for <b>february, may and september</b>, getting samples of 5000 articles for each period and storing them for future work. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feb = pd.read_csv(\"preprocessed_results/mediacloud_parsed_corona_df_feb.csv\")\n",
    "# may = pd.read_csv(\"preprocessed_results/mediacloud_parsed_corona_df_may.csv\")\n",
    "# sep = pd.read_csv(\"preprocessed_results/mediacloud_parsed_corona_df_sep.csv\")\n",
    "\n",
    "def sample_and_save(path):\n",
    "    df = pd.read_csv(path)\n",
    "    df = df[~df.Text.isnull()]\n",
    "    df.reset_index(inplace=True)\n",
    "    df.drop(['Unnamed: 0','Unnamed: 0.1'],axis=1,inplace=True)\n",
    "    df.sample(n=5000).to_csv(path[:-4]+'_sample.csv')\n",
    "    print(f'Sample of 5000 is saved to {path[:-4]}_sample.csv')\n",
    "    \n",
    "sample_and_save(\"preprocessed_results/mediacloud_parsed_corona_df_feb.csv\")\n",
    "sample_and_save(\"preprocessed_results/mediacloud_parsed_corona_df_may.csv\")\n",
    "sample_and_save(\"preprocessed_results/mediacloud_parsed_corona_df_sep.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lemmatization is applied afterwards. Lemmatization considers the context and converts the word to its meaningful base form and is considered a better method for document clustering (<a href=\"https://www.researchgate.net/publication/221615320_Stemming_and_lemmatization_in_the_clustering_of_Finnish_text_documents#:~:text=In%20comparison%20with%20stemming%2C%20lemmatization,are%20clustered%20for%20information%20retrieval.&text=and%20Retrieval%20%E2%80%93%20clustering\">here</a>). Here Python NLTK WordNet Lemmatizer is used that is based on the WordNet Database."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reading the saved samples for each time period. Transforming each article to the tfidf vectors. BIGRAMS parameter is used for specifying if unigrams or bigrams are used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BIGRAMS = True #a parameter that specifies if unigrams (false) or bigrams (true) are used\n",
    "\n",
    "def lemmatize(tokens):\n",
    "    wordnet_lemmatizer = WordNetLemmatizer()\n",
    "    return [wordnet_lemmatizer.lemmatize(word, pos=\"v\") for word in tokens]\n",
    "\n",
    "def dummy_fun(doc):\n",
    "    return doc\n",
    "\n",
    "cv = CountVectorizer(analyzer='word',\n",
    "    tokenizer=dummy_fun,\n",
    "    preprocessor=dummy_fun,\n",
    "    token_pattern=None)\n",
    "\n",
    "def make_bigrams(bigram_mod, texts):\n",
    "    return [bigram_mod[doc] for doc in texts]\n",
    "\n",
    "def read_and_vectorize(path, cv, bigrams):\n",
    "    df = pd.read_csv(path)\n",
    "    df['tokens'] = df['tokens'].apply(ast.literal_eval) #transforming string of tokens to list\n",
    "    df['norm_tokens'] = df['tokens'].apply(lemmatize)\n",
    "    if bigrams == True: #specify if bigrams or unigrams are used for future clustering\n",
    "        bigram = gensim.models.Phrases(df['norm_tokens'], min_count=3, threshold=50) # higher threshold fewer phrases.\n",
    "        bigram_mod = gensim.models.phrases.Phraser(bigram)\n",
    "        df['bigrams'] = make_bigrams(bigram_mod, df['norm_tokens'])\n",
    "        print('Bigrams are created.')\n",
    "        data = cv.fit_transform(df['bigrams'])\n",
    "    else:\n",
    "        data = cv.fit_transform(df['norm_tokens'])\n",
    "    terms = cv.get_feature_names()\n",
    "    print(f'Len of terms: {len(terms)}')\n",
    "    tfidf_transformer = TfidfTransformer()\n",
    "    tfidf_matrix = tfidf_transformer.fit_transform(data)\n",
    "    print(f'Tfidf matrix is generated of shape {tfidf_matrix.shape}')\n",
    "    return df, tfidf_matrix, terms\n",
    "\n",
    "df_feb, tfidf_matrix_feb, terms_feb = read_and_vectorize('preprocessed_results/mediacloud_parsed_corona_df_feb_sample.csv', cv, BIGRAMS)\n",
    "df_may, tfidf_matrix_may, terms_may = read_and_vectorize('preprocessed_results/mediacloud_parsed_corona_df_may_sample.csv', cv, BIGRAMS)\n",
    "df_sep, tfidf_matrix_sep, terms_sep = read_and_vectorize('preprocessed_results/mediacloud_parsed_corona_df_sep_sample.csv', cv, BIGRAMS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-means\n",
    "Transforming tfidf matrix to the sparse numpy array. Running a loop of different k for Kmeans to find the best k by coherence score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform(tfidf_matrix):\n",
    "    transformed_tokens = np.empty((tfidf_matrix.shape[0], 0)).tolist()\n",
    "    for i in range(tfidf_matrix.shape[0]):\n",
    "        transformed_tokens[i] = tfidf_matrix[i].toarray()[0]\n",
    "    print(f'Matrix is tranformed into array of len {len(transformed_tokens)}')\n",
    "    return np.array(transformed_tokens)\n",
    "\n",
    "def get_coherence(topics, dct, texts):\n",
    "    cm = CoherenceModel(topics=topics, dictionary=dct, texts=texts, coherence='c_v')\n",
    "    coherence = cm.get_coherence()\n",
    "    return coherence\n",
    "\n",
    "def kmeans_loop(tfidf_matrix, k_start, k_finish, terms, df, month):\n",
    "    transformed_tokens = transform(tfidf_matrix)\n",
    "    random_state = 20\n",
    "\n",
    "    model_results = {'Num_Topics': [],\n",
    "                     'Coherence': [],\n",
    "                     'Top_terms' : [],\n",
    "                     'Count_Clusters': [],\n",
    "                     'Silhouette':[]\n",
    "                    }\n",
    "    for k in range(k_start, k_finish, 1):\n",
    "        model = KMeans(n_clusters=k, init='k-means++', max_iter=100, n_init=1, random_state = random_state)\n",
    "        clusters = model.fit_predict(transformed_tokens)\n",
    "        print(f'Num of topics: {k}')\n",
    "        centroids = model.cluster_centers_\n",
    "        order_centroids = centroids.argsort()[:, ::-1]\n",
    "        top_terms = []\n",
    "\n",
    "        for i in range(k):\n",
    "            temp = []\n",
    "            for ind in order_centroids[i-1, :10]:\n",
    "                temp.append(terms[ind])\n",
    "            top_terms.append(temp)\n",
    "            \n",
    "        texts = df['bigrams'].tolist()\n",
    "        dct = Dictionary(texts)\n",
    "        coherence = get_coherence(top_terms, dct, texts)\n",
    "        sil = metrics.silhouette_score(transformed_tokens, clusters, metric = 'euclidean')\n",
    "    \n",
    "        model_results['Num_Topics'].append(k)\n",
    "        model_results['Top_terms'].append(top_terms)\n",
    "        model_results['Coherence'].append(coherence)\n",
    "        model_results['Count_Clusters'].append(Counter(clusters))\n",
    "        model_results['Silhouette'].append(sil)\n",
    "\n",
    "    models_df = pd.DataFrame(model_results)\n",
    "    models_df.to_csv(f'preprocessed_results/models_df_lemm_{month}.csv')\n",
    "    print('Model df is saved')\n",
    "    return models_df\n",
    "    \n",
    "# models_df_feb = kmeans_loop(tfidf_matrix_feb, 3, 30, terms_feb, df_feb, 'feb')\n",
    "models_df_may = kmeans_loop(tfidf_matrix_may, 3, 30, terms_may, df_may, 'may')\n",
    "models_df_sep = kmeans_loop(tfidf_matrix_sep, 3, 30, terms_sep, df_sep, 'sep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#choosing the best model by coherence score\n",
    "\n",
    "def select_k(models_df, month):\n",
    "    display(models_df.plot.line(x='Num_Topics', y='Silhouette', title=month, figsize=(6, 4)))\n",
    "    best_model = models_df.iloc[models_df['Silhouette'].idxmax()]\n",
    "    max_k = best_model['Num_Topics']\n",
    "    print(f'Max Silhouette score is k = {max_k} for month {month}')\n",
    "    return best_model\n",
    "\n",
    "best_model_feb = select_k(models_df_feb, 'February')\n",
    "best_model_may = select_k(models_df_may, 'May')\n",
    "best_model_sep = select_k(models_df_sep, 'September')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Getting the information about the best model: how many datapoints are in each cluster, TSNE visualisation and top terms for each cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_tsne_pca(tsne, labels, month):\n",
    "    max_label = max(labels)\n",
    "\n",
    "    label_subset = [cm.hsv(i/max_label) for i in labels]\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    plt.scatter(tsne[:, 0], tsne[:, 1], c=label_subset, s=1)\n",
    "    plt.title(f'TSNE Cluster Plot for {month}')\n",
    "    \n",
    "def get_top_keywords(data, clusters, labels, n_terms):\n",
    "    df = pd.DataFrame(data).groupby(clusters).mean()\n",
    "    \n",
    "    for i,r in df.iterrows():\n",
    "        print('\\nCluster {}'.format(i))\n",
    "        print(','.join([labels[t] for t in np.argsort(r)[-n_terms:]]))\n",
    "    \n",
    "def best_model_kmeans(best_model, month, tfidf_matrix, terms):\n",
    "    random_state = 20\n",
    "    transformed_tokens = transform(tfidf_matrix)\n",
    "    \n",
    "    model = KMeans(n_clusters=best_model['Num_Topics'], init='k-means++', max_iter=100, n_init=1, random_state = random_state)\n",
    "    clusters = model.fit_predict(transformed_tokens)\n",
    "    print('Total number of points in each cluster:', best_model['Count_Clusters'])\n",
    "    if os.path.isfile(f\"preprocessed_results/tsne_lemm_{month}.csv\"):\n",
    "        tsne = np.genfromtxt(f\"preprocessed_results/tsne_lemm_{month}.csv\", delimiter=',')\n",
    "    else:\n",
    "        tsne = TSNE().fit_transform(transformed_tokens)\n",
    "        np.savetxt(f\"preprocessed_results/tsne_lemm_{month}.csv\", tsne, delimiter=\",\")\n",
    "    plot_tsne_pca(tsne, clusters, month)\n",
    "    get_top_keywords(transformed_tokens, clusters, terms, 10)\n",
    "    \n",
    "best_model_kmeans(best_model_feb, 'february', tfidf_matrix_feb, terms_feb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model_kmeans(best_model_may, 'may', tfidf_matrix_may, terms_may)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model_kmeans(best_model_sep, 'september', tfidf_matrix_sep, terms_sep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying kmeans only on the last cluster:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features = [transformed_tokens_feb[i] for i in range(len(transformed_tokens_feb)) if clusters[i]==1]\n",
    "len(selected_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_sel = KMeans(n_clusters=k, random_state=random_state)\n",
    "clusters_sel = model.fit_predict(selected_features)\n",
    "Counter(clusters_sel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_top_keywords(data, clusters, labels, n_terms):\n",
    "    df = pd.DataFrame(data).groupby(clusters).mean()\n",
    "    \n",
    "    for i,r in df.iterrows():\n",
    "        print('\\nCluster {}'.format(i))\n",
    "        print(','.join([labels[t] for t in np.argsort(r)[-n_terms:]]))\n",
    "            \n",
    "get_top_keywords(selected_features, clusters_sel, terms_feb, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Removing corona-related terms:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('corona_terms.txt', 'r') as corona_terms:\n",
    "    terms = corona_terms.read().replace('\\n', ' ').split(' ')\n",
    "    \n",
    "df['tokens_clean'] = df['tokens'].apply(lambda x: [token for token in x if token not in terms])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf.fit(df['tokens_clean']) #fit and transform to vectors\n",
    "features_clean = tfidf.transform(df['tokens_clean'])\n",
    "\n",
    "texts = df['tokens_clean'].tolist()\n",
    "dct = Dictionary(texts)\n",
    "\n",
    "\n",
    "model_nocorona_results = {'Topics': [],\n",
    "                 'Coherence': [],\n",
    "                 'Silhouette': [],\n",
    "                 'Top_terms' : []\n",
    "                }\n",
    "for k in range(3, 15, 1):\n",
    "    model = KMeans(n_clusters=k, init='k-means++', max_iter=100, n_init=1, random_state = random_state)\n",
    "    data = model.fit(features_clean)\n",
    "    sil = silhouette_score(features_clean, labels=model.predict(features_clean))\n",
    "    print(f'Num of topics: {k}')\n",
    "    centroids = model.cluster_centers_\n",
    "\n",
    "    model_tsne = TSNE(n_components=2, random_state=random_state, init=tsne_init, perplexity=tsne_perplexity,\n",
    "             early_exaggeration=tsne_early_exaggeration, learning_rate=tsne_learning_rate)\n",
    "\n",
    "    transformed_centroids = model.fit_transform(centroids)\n",
    "    plt.scatter(transformed_centroids[:, 0], transformed_centroids[:, 1], marker='x')\n",
    "    plt.show()\n",
    "    \n",
    "    terms = tfidf.get_feature_names()\n",
    "    order_centroids = centroids.argsort()[:, ::-1]\n",
    "    top_terms = []\n",
    "\n",
    "    for i in range(k):\n",
    "        temp = []\n",
    "        for ind in order_centroids[i-1, :10]:\n",
    "            temp.append(terms[ind])\n",
    "        top_terms.append(temp)\n",
    "    \n",
    "    coherence = get_coherence(top_terms, dct, texts)\n",
    "    \n",
    "    model_nocorona_results['Topics'].append(k)\n",
    "    model_nocorona_results['Silhouette'].append(sil)\n",
    "    model_nocorona_results['Top_terms'].append(top_terms)\n",
    "    model_nocorona_results['Coherence'].append(coherence)\n",
    "\n",
    "model_nocorona_results_df = pd.DataFrame(model_nocorona_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_nocorona_results_df.to_csv('mediacloud_kmeans_nocorona_results.csv')\n",
    "model_nocorona_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_nocorona_results_df.plot.line(x='Topics', y='Coherence')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_nocorona_results_df['Top_terms'][3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating bigrams and trigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the bigram and trigram models\n",
    "bigram = gensim.models.Phrases(df['tokens_clean'], min_count=5, threshold=100) # higher threshold fewer phrases.\n",
    "trigram = gensim.models.Phrases(bigram[df['tokens_clean']], threshold=100)  \n",
    "\n",
    "# Faster way to get a sentence clubbed as a trigram/bigram\n",
    "bigram_mod = gensim.models.phrases.Phraser(bigram)\n",
    "trigram_mod = gensim.models.phrases.Phraser(trigram)\n",
    "\n",
    "# See trigram example\n",
    "print(trigram_mod[bigram_mod[df['tokens_clean'][0]]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_bigrams(texts):\n",
    "    return [bigram_mod[doc] for doc in texts]\n",
    "\n",
    "def make_trigrams(texts):\n",
    "    return [trigram_mod[bigram_mod[doc]] for doc in texts]\n",
    "\n",
    "data_words_bigrams = make_bigrams(df['tokens_clean'])\n",
    "\n",
    "data_words_bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf.fit(data_words_bigrams) #fit and transform to vectors\n",
    "features_bigrams = tfidf.transform(data_words_bigrams)\n",
    "\n",
    "dct = Dictionary(data_words_bigrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_state = 0\n",
    "tsne_init = 'pca'  # could also be 'random'\n",
    "tsne_perplexity = 20.0\n",
    "tsne_early_exaggeration = 4.0\n",
    "tsne_learning_rate = 1000\n",
    "\n",
    "model_bigram_results = {'Topics': [],\n",
    "                 'Coherence': [],\n",
    "                 'Silhouette': [],\n",
    "                 'Top_terms' : []\n",
    "                }\n",
    "for k in range(3, 15, 1):\n",
    "    model = KMeans(n_clusters=k, init='k-means++', max_iter=100, n_init=1, random_state = random_state)\n",
    "    data = model.fit(features_bigrams)\n",
    "    sil = silhouette_score(features_bigrams, labels=model.predict(features_bigrams))\n",
    "    print(f'Num of topics: {k}')\n",
    "    centroids = model.cluster_centers_\n",
    "\n",
    "    model_tsne = TSNE(n_components=2, random_state=random_state, init=tsne_init, perplexity=tsne_perplexity,\n",
    "             early_exaggeration=tsne_early_exaggeration, learning_rate=tsne_learning_rate)\n",
    "\n",
    "    transformed_centroids = model.fit_transform(centroids)\n",
    "    plt.scatter(transformed_centroids[:, 0], transformed_centroids[:, 1], marker='x')\n",
    "    plt.show()\n",
    "    \n",
    "    terms = tfidf.get_feature_names()\n",
    "    order_centroids = centroids.argsort()[:, ::-1]\n",
    "    top_terms = []\n",
    "\n",
    "    for i in range(k):\n",
    "        temp = []\n",
    "        for ind in order_centroids[i-1, :10]:\n",
    "            temp.append(terms[ind])\n",
    "        top_terms.append(temp)\n",
    "    print('Terms are extracted')\n",
    "    coherence = get_coherence(top_terms, dct, data_words_bigrams)\n",
    "    print('Coherence score is calculated')\n",
    "    \n",
    "    model_bigram_results['Topics'].append(k)\n",
    "    model_bigram_results['Silhouette'].append(sil)\n",
    "    model_bigram_results['Top_terms'].append(top_terms)\n",
    "    model_bigram_results['Coherence'].append(coherence)\n",
    "    print('Result is appended')\n",
    "\n",
    "model_bigram_results_df.append(pd.DataFrame(model_bigram_results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bigram_results_df = pd.DataFrame(model_bigram_results)\n",
    "model_bigram_results_df.to_csv('mediacloud_kmeans_bigrams_nocorona_results.csv')\n",
    "model_bigram_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model_bigram_results_df = pd.DataFrame(model_bigram_results)\n",
    "model_bigram_results_df = pd.read_csv('mediacloud_kmeans_bigrams_results.csv')\n",
    "model_bigram_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bigram_results_df.plot.line(x='Topics', y='Coherence')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bigram_results_df['Top_terms'][7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_state = 0\n",
    "tsne_init = 'pca'  # could also be 'random'\n",
    "tsne_perplexity = 20.0\n",
    "tsne_early_exaggeration = 4.0\n",
    "tsne_learning_rate = 1000\n",
    "\n",
    "data_words_trigrams = make_trigrams(df['tokens_clean'])\n",
    "tfidf.fit(data_words_trigrams) #fit and transform to vectors\n",
    "features_trigrams = tfidf.transform(data_words_trigrams)\n",
    "\n",
    "dct = Dictionary(data_words_trigrams)\n",
    "\n",
    "model_trigram_results = {'Topics': [],\n",
    "                 'Coherence': [],\n",
    "                 'Silhouette': [],\n",
    "                 'Top_terms' : []\n",
    "                }\n",
    "for k in range(3, 15, 1):\n",
    "    model = KMeans(n_clusters=k, init='k-means++', max_iter=100, n_init=1, random_state = random_state)\n",
    "    data = model.fit(features_trigrams)\n",
    "    sil = silhouette_score(features_trigrams, labels=model.predict(features_trigrams))\n",
    "    print(f'Num of topics: {k}')\n",
    "    centroids = model.cluster_centers_\n",
    "\n",
    "    model_tsne = TSNE(n_components=2, random_state=random_state, init=tsne_init, perplexity=tsne_perplexity,\n",
    "             early_exaggeration=tsne_early_exaggeration, learning_rate=tsne_learning_rate)\n",
    "\n",
    "    transformed_centroids = model.fit_transform(centroids)\n",
    "    plt.scatter(transformed_centroids[:, 0], transformed_centroids[:, 1], marker='x')\n",
    "    plt.show()\n",
    "    \n",
    "    terms = tfidf.get_feature_names()\n",
    "    order_centroids = centroids.argsort()[:, ::-1]\n",
    "    top_terms = []\n",
    "\n",
    "    for i in range(k):\n",
    "        temp = []\n",
    "        for ind in order_centroids[i-1, :10]:\n",
    "            temp.append(terms[ind])\n",
    "        top_terms.append(temp)\n",
    "    \n",
    "    coherence = get_coherence(top_terms, dct, data_words_trigrams)\n",
    "    \n",
    "    model_trigram_results['Topics'].append(k)\n",
    "    model_trigram_results['Silhouette'].append(sil)\n",
    "    model_trigram_results['Top_terms'].append(top_terms)\n",
    "    model_trigram_results['Coherence'].append(coherence)\n",
    "\n",
    "model_trigram_results_df = pd.DataFrame(model_trigram_results)\n",
    "model_trigram_results_df.to_csv('mediacloud_kmeans_trigrams_results.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_trigram_results_df = pd.read_csv('mediacloud_kmeans_trigrams_results.csv')\n",
    "model_trigram_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_trigram_results_df.plot.line(x='Topics', y='Coherence')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_trigram_results_df['Top_terms'][4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hierarchical clustering\n",
    "Applying only on those clusters that seem to relate to stories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = KMeans(n_clusters=5, init='k-means++', max_iter=100, n_init=1, random_state = random_state)\n",
    "data = model.fit(features)\n",
    "labels = model.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "centroids = model.cluster_centers_\n",
    "\n",
    "terms = tfidf.get_feature_names()\n",
    "order_centroids = centroids.argsort()[:, ::-1]\n",
    "top_terms = []\n",
    "\n",
    "for i in range(5):\n",
    "    temp = []\n",
    "    for ind in order_centroids[i-1, :10]:\n",
    "        temp.append(terms[ind])\n",
    "    top_terms.append(temp)\n",
    "    \n",
    "top_terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Selecting only those about the first general topic\n",
    "selected_df = df[labels==1]\n",
    "selected_features = tfidf.fit_transform(selected_df['tokens']) #fit and transform to vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "svd = TruncatedSVD(n_components=100, random_state=random_state)\n",
    "features_red = svd.fit_transform(selected_features)\n",
    "features_red.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linkage_matrix = ward(features_red)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(15, 20)) # set size\n",
    "ax = dendrogram(linkage_matrix, orientation=\"right\", labels=selected_df['stories_id'].tolist())\n",
    "\n",
    "plt.tick_params(\\\n",
    "    axis= 'x',          # changes apply to the x-axis\n",
    "    which='both',      # both major and minor ticks are affected\n",
    "    bottom='off',      # ticks along the bottom edge are off\n",
    "    top='off',         # ticks along the top edge are off\n",
    "    labelbottom='off')\n",
    "\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import AgglomerativeClustering\n",
    "\n",
    "n_clusters = 8\n",
    "model_hierarchical = AgglomerativeClustering(n_clusters=n_clusters, affinity='euclidean', linkage='ward')  \n",
    "model_hierarchical.fit_predict(features_red)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for label in range(model_hierarchical.n_clusters_):\n",
    "    print(label)\n",
    "    display(selected_df[model_hierarchical.labels_==label]['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(model_hierarchical.labels_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from yellowbrick.text import TSNEVisualizer\n",
    "\n",
    "tsne = TSNEVisualizer()\n",
    "tsne.fit(features_red, model_hierarchical.labels_)\n",
    "tsne.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}