src/notebooks/Build-deploy-dga-xgboost_exp5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import HTML\n",
    "HTML('''<h2>IMT 575: Build DGA model using XGBOOST in Sagemaker</h2>\n",
    "<b><pre>\n",
    "    Authors: \n",
    "    Aftab Alam\n",
    "    </pre>\n",
    "</b> \n",
    "<p>Date/Time: <span id=\"datetime\"></span></p><script>var dt = new Date();\n",
    "document.getElementById(\"datetime\").innerHTML=dt.toLocaleString();</script> </p>''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In order to run aws boto3 apis locally, please setup proper role(sagement, lambda execution, cloudwatch ..)\n",
    "# below code will read credential and that can be passed in boto3 session.\n",
    "# Thanks to https://gist.github.com/wjimenez5271/defeede8eb4a63afc9d8\n",
    "def get_profile_credentials(profile_name):\n",
    "    from configparser import ConfigParser\n",
    "    from configparser import ParsingError\n",
    "    from configparser import NoOptionError\n",
    "    from configparser import NoSectionError\n",
    "    from os import path\n",
    "    config = ConfigParser()\n",
    "    config.read([path.join(path.expanduser(\"~\"),'.aws/credentials')])\n",
    "    try:\n",
    "        aws_access_key_id = config.get(profile_name, 'aws_access_key_id')\n",
    "        aws_secret_access_key = config.get(profile_name, 'aws_secret_access_key')\n",
    "    except ParsingError:\n",
    "        print('Error parsing config file')\n",
    "        raise\n",
    "    except (NoSectionError, NoOptionError):\n",
    "        try:\n",
    "            aws_access_key_id = config.get('default', 'aws_access_key_id')\n",
    "            aws_secret_access_key = config.get('default', 'aws_secret_access_key')\n",
    "        except (NoSectionError, NoOptionError):\n",
    "            print('Unable to find valid AWS credentials')\n",
    "            raise\n",
    "    return aws_access_key_id, aws_secret_access_key\n",
    "aws_access_key_id,aws_secret_access_key = get_profile_credentials('aftabuw')\n",
    "LOCAL=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "! conda install -y -c conda-forge ipywidgets\n",
    "! pip install tldextract\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import sys\n",
    "import os\n",
    "import boto3\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "# for extracting domain name\n",
    "import tldextract\n",
    "\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "# Sagemake \n",
    "import sagemaker\n",
    "from sagemaker.amazon.amazon_estimator import get_image_uri \n",
    "from sagemaker import get_execution_role\n",
    "from sagemaker.session import Session\n",
    "from sagemaker.predictor import csv_serializer\n",
    "from sagemaker.tensorflow import TensorFlow\n",
    "\n",
    "# sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# enable flag to how all output\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting up bot3 sessions and client\n",
    "print(f\"Runing locally : {LOCAL==1}\")\n",
    "if(LOCAL==1):\n",
    "    role='arn:aws:iam::099176660580:role/service-role/AmazonSageMaker-ExecutionRole-20200505T194950'\n",
    "    boto_session = boto3.Session(aws_access_key_id=aws_access_key_id,\n",
    "                      aws_secret_access_key=aws_secret_access_key)\n",
    "    region=boto_session.region_name\n",
    "    sagemaker_client = boto3.client('sagemaker',aws_access_key_id=aws_access_key_id,\n",
    "                                    aws_secret_access_key=aws_secret_access_key,\n",
    "                                    region_name=region\n",
    "                                   )\n",
    "    sagemaker_runtime_client = boto3.client('sagemaker-runtime',aws_access_key_id=aws_access_key_id,\n",
    "                                    aws_secret_access_key=aws_secret_access_key,\n",
    "                                    region_name=region\n",
    "                                   )\n",
    "    sagemaker_session = sagemaker.Session(boto_session=boto_session, \n",
    "                               sagemaker_client=sagemaker_client, \n",
    "                               sagemaker_runtime_client=sagemaker_runtime_client,)\n",
    "    bucket = sagemaker_session.default_bucket()\n",
    "    # change this ARN for role that u need to use for sage maker\n",
    "else:\n",
    "    # if running it from sagemake notebook instance\n",
    "    role = sagemaker.get_execution_role()\n",
    "    boto_session = boto3.Session()\n",
    "    region=boto_session.region_name\n",
    "    sagemaker_client = boto3.client('sagemaker',\n",
    "                                    region_name=region\n",
    "                                   )\n",
    "    sagemaker_runtime_client = boto3.client('sagemaker-runtime',\n",
    "                                    region_name=region\n",
    "                                   )\n",
    "    region = boto3.Session().region_name\n",
    "\n",
    "    # S3 bucket for saving code and model artifacts.\n",
    "    # Feel free to specify a different bucket and prefix\n",
    "    sagemaker_session = sagemaker.Session()\n",
    "    bucket = sagemaker_session.default_bucket()\n",
    "prefix = 'sagemaker/xgboost_exp5/length'\n",
    "# customize to your bucket where you have stored the data\n",
    "bucket_path = 'https://s3-{}.amazonaws.com/{}'.format(region, bucket)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Clean up and Feature engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Function for spliting data and saving it in s3 as well local file system**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# split data and save in s3\n",
    "\n",
    "import io\n",
    "import boto3\n",
    "import random\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def appendXY(X,Y):\n",
    "    df = pd.DataFrame(Y)\n",
    "    print(df.shape)\n",
    "    df.columns = ['Y']\n",
    "    dfx = pd.DataFrame(X)\n",
    "    print(dfx.shape)\n",
    "    return pd.concat([df,dfx],axis=1)\n",
    "   \n",
    "#  since we difference number of records from dga families, we need to make sure test and training data contains all type of\n",
    "# dga data set. Hence we will first spilt data from domain file using family and then and extract features. this will to avoid \n",
    "# class imbalance issue in data set\n",
    "\n",
    "def split_dataframe(data,target='family',test_size=.2):\n",
    "    \"\"\"\n",
    "    input a data frame with target on which data needs to be splited.\n",
    "    \n",
    "    \"\"\"\n",
    "    # distinct number of class in target\n",
    "    targets = data[target].unique()\n",
    "    df_test = pd.DataFrame()\n",
    "    for t in targets:\n",
    "        df_test= pd.concat([df_test,data[data[target]==t].sample(frac=test_size)])\n",
    "    df_train = data[~data.index.isin(df_test.index)]\n",
    "    \n",
    "    return df_train,df_test\n",
    "\n",
    "def save_features_file(features,output_label,file):\n",
    "    \n",
    "    y = output_label\n",
    "    X = features\n",
    "    df = appendXY(X,y)\n",
    "    df.to_csv(file,header=False,index=False)\n",
    "\n",
    "\n",
    "def write_to_s3(fobj, bucket, key):  \n",
    "    return boto_session.resource('s3').Bucket(bucket).Object(key).upload_fileobj(fobj)\n",
    "\n",
    "def upload_to_s3(bucket, channel, filename):\n",
    "    fobj=open(filename, 'rb')\n",
    "    key = prefix+'/'+channel\n",
    "    url = 's3://{}/{}/{}'.format(bucket, key, filename)\n",
    "    print('Writing to {}'.format(url))\n",
    "    write_to_s3(fobj, bucket, key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Download DGA and benign data set from S3 bucket**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download file from S3\n",
    "#FILE_DATA = 'domainsDataSet'\n",
    "#s3 = boto_session.client('s3')\n",
    "#s3.download_file(bucket, 'newDataSample.csv', FILE_DATA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_benign = pd.read_csv(\"alexa_cisco_onenpage_7mn_begin_dataset.csv.gz\").sample(frac=1,random_state=1122)\n",
    "df_dga = pd.read_csv(\"sevenmillons_dga.csv.gz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df_benign.shape\n",
    "df_benign.head()\n",
    "df_dga.shape\n",
    "df_dga.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_benign.dropna(how='any',inplace=True)\n",
    "df_dga.dropna(how='any',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Add length and number of unique char\n",
    "df_benign.loc[:,'uniquechar'] = df_benign.apply(lambda row: len(set(row.domain)) , axis=1)\n",
    "df_benign.loc[:,'length'] = df_benign.apply(lambda row: len(row.domain) , axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_benign.reset_index(inplace=True,drop=True)\n",
    "df_dga.reset_index(inplace=True,drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dga.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_benign.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine benign and dga data set\n",
    "df_domains = pd.concat([df_dga[['domain','family','label','uniquechar','length']],\n",
    "                        df_benign[['domain','family','label','uniquechar','length']]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains.drop_duplicates(subset=['domain'],keep='first',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FILE_DATA = 'domainsDataSet_15mn.csv.gz'\n",
    "df_domains.to_csv(FILE_DATA,compression=\"gzip\",index=False)\n",
    "df_domains= pd.read_csv(FILE_DATA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load and inspect data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the dataset\n",
    "#df_domains = pd.read_csv(FILE_DATA)\n",
    "\n",
    "df_domains.tail()\n",
    "df_domains.head()\n",
    "df_domains.groupby('label').agg('count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to extract domain from full domain name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "def extract_domain_subdomain(record):\n",
    "    domain = record.domainName\n",
    "    ret=''\n",
    "    try:\n",
    "        ext = tldextract.extract(domain)\n",
    "        ret = ext.domain\n",
    "    except :\n",
    "        print(record)\n",
    "    return ret\n",
    "def get_y(row):\n",
    "    if row.label.lower()=='bad':\n",
    "        return 1\n",
    "    elif row.label.lower()=='good':\n",
    "        return 0\n",
    "    else :\n",
    "        return 1\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create new columns for domain and dga binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "df_domains.loc[:,'domain_subdomain'] = df_domains.apply(lambda row : extract_domain_subdomain(row), axis=1 )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "## 1 for dga and 0 for benign\n",
    "df_domains.loc[:,'Y'] = df_domains.apply(lambda row : get_y(row), axis=1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains= df_domains[df_domains.family!='others']\n",
    "df_domains.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains.loc[:,'duplicate']=df_domains.duplicated(subset=['domain'],keep=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains[['duplicate','family','label']].groupby(['duplicate']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains[(df_domains.duplicate)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains[(df_domains.duplicate) & (df_domains.domain=='nv5')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets finds with DGA domain(extract)contains benign data. This is case we need to keep begin and drop DGA.\n",
    "# using last since begin data set is added at the end\n",
    "df_domains.drop_duplicates(subset=['domain'],keep=\"first\",inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains[['family','label']].groupby(['label']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains[df_domains.domain.isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains.drop(index=2171628,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains.head()\n",
    "df_domains.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# correctly family names\n",
    "def family(fam):\n",
    "    d = {'dnscharger':'dnschanger','conficker':'conflicker','dircypt':'dircrypt','goz':'gozi',\n",
    "        'locy':'locky','nymaim':'nymain','un_js':'unjavascript','alexa':'benign'}\n",
    "    if fam in d.keys():\n",
    "        return d[fam]\n",
    "    else:\n",
    "        return fam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains.loc[:,'family'] = df_domains.apply(lambda row: family(row.family), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains[['family','uniquechar','length']].groupby(['family']).agg(['count','mean','max'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_domains[['family','uniquechar','length']].groupby(['family']).agg(['count','mean','max'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"data_distribution.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### function to convert domain into features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "VALID_CHARS = 'abcdefghijklmnopqrstuvwxyz0123456789-_.'\n",
    "LOOKUP_TABLE = None\n",
    "def pad(l, content, width):\n",
    "        l.extend([content] * (width - len(l)))\n",
    "        return l\n",
    "    \n",
    "def check_validchar(domain):\n",
    "    for c in domain.lower():\n",
    "        if c not in VALID_CHARS:\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "    \n",
    "\n",
    "def features_extract(domain): \n",
    "    \n",
    "    global VALID_CHARS    \n",
    "    global LOOKUP_TABLE    \n",
    "    if not LOOKUP_TABLE:        \n",
    "        LOOKUP_TABLE = dict()       \n",
    "        idx = 1\n",
    "        for c in VALID_CHARS:\n",
    "            LOOKUP_TABLE[c] = int(idx)            \n",
    "            idx += int(1)    \n",
    "    #ds = tldextract.extract(fqdn)    \n",
    "    #domain = ds.domain \n",
    "    #ratio = len(set(domain))/len(domain)\n",
    "    \n",
    "    rvalue = list()  \n",
    "    if len(domain)<=63:\n",
    "        for c in domain.lower():\n",
    "            try:\n",
    "                rvalue.append(LOOKUP_TABLE[c])\n",
    "            except:\n",
    "                print(f\"Char error out in {domain}: {c}\")\n",
    "    else: \n",
    "        #print(domain)\n",
    "        pass\n",
    "            \n",
    "    rvalue=pad(rvalue,0,63)    \n",
    "    return rvalue\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "df_temp = df_domains.head(10)\n",
    "x = [features_extract(D) for D in df_temp.domain]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# check if domain is valid\n",
    "df_domains.loc[:,'valid'] = df_domains.apply(lambda row : check_validchar(row.domain), axis=1 )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domains=df_domains[df_domains.valid]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop invalid data with invalid char\n",
    "df_domains.shape\n",
    "df_domains=df_domains[df_domains.valid].copy()\n",
    "df_domains.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# split data in train,validation and train\n",
    "df_train_valid,df_test = split_dataframe(data=df_domains,test_size=.1)\n",
    "df_train,df_valid = split_dataframe(data=df_train_valid,test_size=.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.head()\n",
    "df_valid.head()\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid.family.unique()\n",
    "df_train.family.unique()\n",
    "df_test.family.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create feature Vector for train, validation and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# features X train vector\n",
    "X_train = [features_extract(D) for D in df_train_valid.domain]\n",
    "X_train = np.array(X_train)\n",
    "X_train.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# features X valid vector\n",
    "X_valid = [features_extract(D) for D in df_valid.domain]\n",
    "X_valid = np.array(X_valid)\n",
    "X_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# features X test vector\n",
    "X_test = [features_extract(D) for D in df_test.domain]\n",
    "X_test = np.array(X_test)\n",
    "X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split data feature set and create files that can be used for sagemaker model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Combine and X and Y and save into files\n",
    "FILE_TRAIN = 'domainsDataSet.train'\n",
    "FILE_VALIDATION = 'domainsDataSet.validation'\n",
    "FILE_TEST = 'domainsDataSet.test'\n",
    "\n",
    "save_features_file(features=X_train,output_label=df_train['Y'].values,file=FILE_TRAIN)\n",
    "save_features_file(features=X_valid,output_label=df_valid['Y'].values,file=FILE_VALIDATION)\n",
    "save_features_file(features=X_test,output_label=df_test['Y'].values,file=FILE_TEST)\n",
    "\n",
    "#upload the files to the S3 bucket\n",
    "upload_to_s3(bucket, 'train', FILE_TRAIN)\n",
    "upload_to_s3(bucket, 'validation', FILE_VALIDATION)\n",
    "upload_to_s3(bucket, 'test', FILE_TEST)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check fields()\n",
    "from sys import platform\n",
    "print(\"Running for platforr: \",platform)\n",
    "if 'win' in platform:\n",
    "    print(\"Validation data\")\n",
    "    !powershell -command \"& {Get-Content domainsDataSet.validation -TotalCount 2}\"\n",
    "    print(\"test data\")\n",
    "    !powershell -command \"& {Get-Content domainsDataSet.test -TotalCount 2}\"\n",
    "    print(\"train data\")\n",
    "    !powershell -command \"& {Get-Content 'domainsDataSet.train' -TotalCount 2}\"\n",
    "else:\n",
    "    print(\"Validation data\")\n",
    "    !head -3 domainsDataSet.validation\n",
    "    print(\"test data\")\n",
    "    !head -3 domainsDataSet.test\n",
    "    print(\"train data\")\n",
    "    !head -3 domainsDataSet.train\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if train and test data is already saved repeat from here."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Models\n",
    "Here we will try xgboost and lstm model and tune it for the best performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create input\n",
    "s3_input_train = sagemaker.s3_input(s3_data='s3://{}/{}/train'.format(bucket, prefix), content_type='csv')\n",
    "s3_input_validation = sagemaker.s3_input(s3_data='s3://{}/{}/validation'.format(bucket, prefix), content_type='csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### XGBOOST classification\n",
    "\n",
    "__Reading refernces__:  \n",
    "https://towardsdatascience.com/xgboost-in-amazon-sagemaker-28e5e354dbcd  \n",
    "https://github.com/awslabs/amazon-sagemaker-examples/blob/master/introduction_to_amazon_algorithms/xgboost_abalone/xgboost_abalone.ipynb  \n",
    "https://aws.amazon.com/blogs/machine-learning/call-an-amazon-sagemaker-model-endpoint-using-amazon-api-gateway-and-aws-lambda/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# train model\n",
    "from sagemaker.amazon.amazon_estimator import get_image_uri\n",
    "\n",
    "container = get_image_uri(region, 'xgboost','1.0-1')\n",
    "\n",
    "xgb = sagemaker.estimator.Estimator(container,\n",
    "                                    role, \n",
    "                                    train_instance_count=1, \n",
    "                                    train_instance_type='ml.c5.4xlarge',\n",
    "                                    output_path='s3://{}/{}/output'.format(bucket, prefix),\n",
    "                                    sagemaker_session=sagemaker_session)\n",
    "\n",
    "# fitting model with paramter from previously best tune model for this data set\n",
    "# or we can start default and tune model later \n",
    "xgb.set_hyperparameters(base_score=0.5, \n",
    "                        booster='gbtree', #['gbtree', 'gblinear', 'dart']\n",
    "                        colsample_bylevel=0.3328968814794882,\n",
    "                        colsample_bynode=1, \n",
    "                        colsample_bytree=0.7460086251908613, \n",
    "                        gamma=4.36472704596215, \n",
    "                        #reg_lambda=18.34813124562997,\n",
    "                        alpha=458.20153739471834,\n",
    "                        max_delta_step=8, max_depth=6,\n",
    "                        min_child_weight=7.4485695445680005,\n",
    "                        scale_pos_weight=1, subsample=.9, tree_method='auto',\n",
    "                        eta=0.4008765966370876,\n",
    "                        silent=1,\n",
    "                        objective='reg:squarederror', #reg:squarederror\n",
    "                        num_round=200\n",
    "                       )\n",
    "\n",
    "xgb.fit({'train': s3_input_train, 'validation': s3_input_validation})\n",
    "## Deploy trained XGBoost model endpoint to perform predictions\n",
    "xgb_predictor = xgb.deploy(initial_instance_count = 1, instance_type = 'ml.t2.medium')\n",
    "\n",
    "# make sure to set content type to csv as we have data in csv format\n",
    "xgb_predictor.content_type = 'text/csv'\n",
    "xgb_predictor.serializer = csv_serializer\n",
    "xgb_predictor.deserializer = None\n",
    "\n",
    "## Function to chunk down test set into smaller increments\n",
    "\n",
    "def predict(data, model, rows=500):\n",
    "    split_array = np.array_split(data, int(len(data) / float(rows) + 1))\n",
    "    predictions = ''\n",
    "    for array in split_array:\n",
    "        #print(array[0])\n",
    "        predictions = ','.join([predictions, model.predict(array).decode('utf-8')])\n",
    "\n",
    "    return np.fromstring(predictions[1:], sep=',')\n",
    "\n",
    "%%time\n",
    "## Generate predictions on the test set for the difference models\n",
    "with open('domainsDataSet.test', 'r') as f:\n",
    "    payload = f.read().strip()\n",
    "labels = [int(line[0]) for line in payload.split('\\n')]\n",
    "test_data = [line[2:] for line in payload.split('\\n')]\n",
    "predictions = predict(test_data, xgb_predictor)\n",
    "#xgb_predictor.predict(payload[2:]).decode('utf-8')\n",
    "from sklearn.metrics import accuracy_score ,confusion_matrix\n",
    "thresh = 0.5\n",
    "y_pred = predictions\n",
    "y_pred_binary = np.where(predictions > thresh, 1, 0)\n",
    "accuracy_score(labels,y_pred_binary)\n",
    "confusion_matrix(labels,y_pred_binary)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install pickle-mixin\n",
    "!pip install xgboost==1.0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tarfile\n",
    "import pickle as pkl\n",
    "#sagemaker/xgboost_exp4/output/sagemaker-xgboost-2020-05-17-09-23-49-951/output\n",
    "path_key = f'{prefix}/output/sagemaker-xgboost-2020-05-27-15-30-45-659/output'\n",
    "# download the model artifact from AWS S3\n",
    "s3 = boto_session.client('s3')\n",
    "s3.download_file(bucket, f'{path_key}/model.tar.gz', 'model.tar.gz')\n",
    "#opens the downloaded model artifcat and loads it as 'model' variable\n",
    "tar = tarfile.open('model.tar.gz')\n",
    "tar.extractall()\n",
    "tar.close()\n",
    "file = open('xgboost-model', 'rb')\n",
    "model = pkl.loads(file.read())\n",
    "\n",
    "# list directory and check if model file is present\n",
    "if 'win' in platform:\n",
    "    ! dir\n",
    "else:\n",
    "    ! ls -lrt\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take one records from test file\n",
    "if 'win' in platform:\n",
    "    !powershell -command \"& {Get-Content domainsDataSet.test -TotalCount 1}\" > single.test\n",
    "else:\n",
    "    !head -1 domainsDataSet.test > single.test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost\n",
    "with open('single.test', 'r') as f:\n",
    "    payload = f.read().strip()\n",
    "    print(payload)\n",
    "    dtrain = xgboost.DMatrix(payload[2:], label=payload[0])\n",
    "model.predict(dtrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost\n",
    "#map_names = dict(zip(model.feature_names, df_domains.columns))\n",
    "#model.feature_names = list(map_names.values())\n",
    "\n",
    "#plot feature importance\n",
    "fig, ax = plt.subplots(figsize=(12,12))\n",
    "xgboost.plot_importance(model, importance_type='gain', max_num_features=30, height=0.8, ax=ax, show_values = False)\n",
    "plt.title('Feature Importance')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Deploy the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Deploy trained XGBoost model endpoint to perform predictions\n",
    "xgb_predictor = xgb.deploy(initial_instance_count = 1, instance_type = 'ml.t2.medium')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure to set content type to csv as we have data in csv format\n",
    "xgb_predictor.content_type = 'text/csv'\n",
    "xgb_predictor.serializer = csv_serializer\n",
    "xgb_predictor.deserializer = None\n",
    "\n",
    "## Function to chunk down test set into smaller increments\n",
    "\n",
    "def predict(data, model, rows=500):\n",
    "    split_array = np.array_split(data, int(len(data) / float(rows) + 1))\n",
    "    predictions = ''\n",
    "    for array in split_array:\n",
    "        #print(array[0])\n",
    "        predictions = ','.join([predictions, model.predict(array).decode('utf-8')])\n",
    "\n",
    "    return np.fromstring(predictions[1:], sep=',')\n",
    "\n",
    "%%time\n",
    "## Generate predictions on the test set for the difference models\n",
    "with open('domainsDataSet.test', 'r') as f:\n",
    "    payload = f.read().strip()\n",
    "labels = [int(line[0]) for line in payload.split('\\n')]\n",
    "test_data = [line[2:] for line in payload.split('\\n')]\n",
    "predictions = predict(test_data, xgb_predictor)\n",
    "#xgb_predictor.predict(payload[2:]).decode('utf-8')\n",
    "from sklearn.metrics import accuracy_score ,confusion_matrix\n",
    "thresh = 0.5\n",
    "y_pred = predictions\n",
    "y_pred_binary = np.where(predictions > thresh, 1, 0)\n",
    "accuracy_score(labels,y_pred_binary)\n",
    "confusion_matrix(labels,y_pred_binary)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test Model using test data from local file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Function to chunk down test set into smaller increments\n",
    "\n",
    "def predict(data, model, rows=500):\n",
    "    split_array = np.array_split(data, int(len(data) / float(rows) + 1))\n",
    "    predictions = ''\n",
    "    for array in split_array:\n",
    "        #print(array[0])\n",
    "        predictions = ','.join([predictions, model.predict(array).decode('utf-8')])\n",
    "\n",
    "    return np.fromstring(predictions[1:], sep=',')\n",
    "\n",
    "%%time\n",
    "## Generate predictions on the test set for the difference models\n",
    "with open('domainsDataSet.test', 'r') as f:\n",
    "    payload = f.read().strip()\n",
    "labels = [int(line[0]) for line in payload.split('\\n')]\n",
    "test_data = [line[2:] for line in payload.split('\\n')]\n",
    "predictions = predict(test_data, xgb_predictor)\n",
    "#xgb_predictor.predict(payload[2:]).decode('utf-8')\n",
    "from sklearn.metrics import accuracy_score ,confusion_matrix\n",
    "thresh = 0.5\n",
    "y_pred = predictions\n",
    "y_pred_binary = np.where(predictions > thresh, 1, 0)\n",
    "accuracy_score(labels,y_pred_binary)\n",
    "confusion_matrix(labels,y_pred_binary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "## Generate predictions on the test set for the difference models\n",
    "with open('single.test', 'r') as f:\n",
    "    payload = f.read().strip()\n",
    "labels = [int(line[0]) for line in payload.split('\\n')]\n",
    "test_data = [line[2:] for line in payload.split('\\n')]\n",
    "predictions = predict(test_data, xgb_predictor)\n",
    "#xgb_predictor.predict(payload[2:]).decode('utf-8')\n",
    "predictions\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take one records from test file\n",
    "if 'win' in platform:\n",
    "    !powershell -command \"& {Get-Content domainsDataSet.test -TotalCount 10}\" > ten_records.test\n",
    "else:\n",
    "    !head -10 domainsDataSet.test > ten_records.test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "## Generate predictions on the test set for the difference models\n",
    "with open('ten_records.test', 'r') as f:\n",
    "    payload = f.read().strip()\n",
    "labels = [int(line[0]) for line in payload.split('\\n')]\n",
    "test_data = [line[2:] for line in payload.split('\\n')]\n",
    "predictions = predict(test_data, xgb_predictor)\n",
    "#xgb_predictor.predict(payload[2:]).decode('utf-8')\n",
    "predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test is all test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "## Generate predictions on the test set for the difference models\n",
    "with open('domainsDataSet.test', 'r') as f:\n",
    "    payload = f.read().strip()\n",
    "labels = [int(line[0]) for line in payload.split('\\n')]\n",
    "test_data = [line[2:] for line in payload.split('\\n')]\n",
    "predictions = predict(test_data, xgb_predictor)\n",
    "#xgb_predictor.predict(payload[2:]).decode('utf-8')\n",
    "from sklearn.metrics import accuracy_score ,confusion_matrix\n",
    "thresh = 0.5\n",
    "y_pred = predictions\n",
    "y_pred_binary = np.where(predictions > thresh, 1, 0)\n",
    "accuracy_score(labels,y_pred_binary)\n",
    "confusion_matrix(labels,y_pred_binary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create function to test using sagemaker runtime client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import boto3\n",
    "import json\n",
    "import tldextract\n",
    "\n",
    "# grab environment variables\n",
    "ENDPOINT_NAME = os.getenv('ENDPOINT_NAME','sagemaker-predict-endpoint')\n",
    "runtime= boto_session.client('runtime.sagemaker')\n",
    "\n",
    "def extract_domain(record):\n",
    "    domain = record\n",
    "    ret=''\n",
    "    try:\n",
    "        ext = tldextract.extract(domain)\n",
    "        ret = ext.domain\n",
    "    except :\n",
    "        print(record)\n",
    "    return ret\n",
    "\n",
    "VALID_CHARS = 'abcdefghijklmnopqrstuvwxyz0123456789-_.'\n",
    "LOOKUP_TABLE = None\n",
    "def pad(l, content, width):\n",
    "        l.extend([content] * (width - len(l)))\n",
    "        return l\n",
    "\n",
    "def features(domain): \n",
    "    \n",
    "    global VALID_CHARS    \n",
    "    global LOOKUP_TABLE    \n",
    "    if not LOOKUP_TABLE:        \n",
    "        LOOKUP_TABLE = dict()       \n",
    "        idx = 1\n",
    "        for c in VALID_CHARS:\n",
    "            LOOKUP_TABLE[c] = int(idx)            \n",
    "            idx += int(1) \n",
    "    ratio = len(set(domain))/len(domain)\n",
    "    rvalue = list()  \n",
    "    if len(domain)<=63 and ' ' not in domain:\n",
    "        for c in domain.lower():        \n",
    "            rvalue.append(str(LOOKUP_TABLE[c]))\n",
    "    else: \n",
    "        #print(domain)\n",
    "        pass\n",
    "            \n",
    "    rvalue=pad(rvalue,'0',63)\n",
    "    rvalue1=','.join(rvalue) + str(ratio) #\n",
    "    return rvalue1\n",
    "def lambda_handler(event, context):\n",
    "    print(\"Received event: \" + json.dumps(event, indent=2))\n",
    "    \n",
    "    data = json.loads(json.dumps(event))\n",
    "    payload = data['fqdn']\n",
    "    #print(payload)\n",
    "    # extract domain \n",
    "    \n",
    "    domain = extract_domain(payload)\n",
    "    feature_X = features(domain)\n",
    "    #print(feature_X)\n",
    "    response = runtime.invoke_endpoint(EndpointName=ENDPOINT_NAME,\n",
    "                                       ContentType='text/csv',\n",
    "                                       Body=feature_X)\n",
    "    #print(response)\n",
    "    pred = json.loads(response['Body'].read().decode())\n",
    "    print(pred)\n",
    "    predicted_label = 'dga' if pred > .5 else 'benign'\n",
    "    \n",
    "    return predicted_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "event= {'fqdn':'www.google.com'}\n",
    "context=None\n",
    "lambda_handler(event,context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feedback = pd.read_csv('feedback.csv')\n",
    "df_feedback.head(5)\n",
    "df_feedback.columns= ['domain','correct']\n",
    "df_feedback.groupby(['correct']).count()\n",
    "df_feedback_test = df_feedback[(df_feedback.correct == 'dga') | (df_feedback.correct =='benign')].copy()\n",
    "df_feedback_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feedback_test.loc[:,'TesT'] = df_feedback_test.apply(lambda row: lambda_handler({'fqdn':row['domain']},None), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score ,confusion_matrix\n",
    "df_feedback_test.columns\n",
    "labels = df_feedback_test.correct\n",
    "preds = df_feedback_test.TesT\n",
    "accuracy_score(labels,preds)\n",
    "confusion_matrix(labels,preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feedback[df_feedback.correct!=df_feedback.TesT]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we use the Hyperparameter Tuner\n",
    "from sagemaker.tuner import IntegerParameter\n",
    "from sagemaker.tuner import ContinuousParameter\n",
    "from sagemaker.tuner import HyperparameterTuner\n",
    "\n",
    "# Define exploration boundaries (default suggested values from Amazon SageMaker Documentation)\n",
    "hyperparameter_ranges = {\n",
    "    'alpha': ContinuousParameter(0, 1000, scaling_type=\"Auto\"),\n",
    "    'colsample_bylevel': ContinuousParameter(0.1, 1,scaling_type=\"Logarithmic\"),\n",
    "    'colsample_bytree': ContinuousParameter(0.5, 1, scaling_type='Logarithmic'),\n",
    "    'eta': ContinuousParameter(0.1, 0.5, scaling_type='Logarithmic'),\n",
    "    'gamma':ContinuousParameter(0, 5, scaling_type='Auto'),\n",
    "    'lambda': ContinuousParameter(0,100,scaling_type='Auto'),\n",
    "    'max_delta_step': IntegerParameter(0,10,scaling_type='Auto'),\n",
    "    'max_depth': IntegerParameter(0,10,scaling_type='Auto'),\n",
    "    'min_child_weight': ContinuousParameter(0,10,scaling_type='Auto'),\n",
    "    'num_round': IntegerParameter(1000,3000,scaling_type='Auto'),\n",
    "    'subsample': ContinuousParameter(0.5,1,scaling_type='Logarithmic')}\n",
    "\n",
    "objective_metric_name = 'validation:accuracy'\n",
    "\n",
    "tuner_log = HyperparameterTuner(\n",
    "    xgb,\n",
    "    objective_metric_name,\n",
    "    hyperparameter_ranges,\n",
    "    max_jobs=20,\n",
    "    max_parallel_jobs=1,\n",
    "    strategy='Bayesian'\n",
    ")\n",
    "\n",
    "## Starts the hyperparameter tuning job\n",
    "tuner_log.fit({'train': s3_input_train, 'validation': s3_input_validation}, include_cls_metadata=False)\n",
    "\n",
    "## Prints the status of the latest hyperparameter tuning job\n",
    "boto_session.client('sagemaker').describe_hyper_parameter_tuning_job(\n",
    "    HyperParameterTuningJobName=tuner_log.latest_tuning_job.job_name)['HyperParameterTuningJobStatus']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "status = sagemaker_client.describe_hyper_parameter_tuning_job(\n",
    "    HyperParameterTuningJobName=tuner_log.latest_tuning_job.job_name)['HyperParameterTuningJobStatus']\n",
    "print(status)\n",
    "while status !='Completed' and status!='Failed':\n",
    "    time.sleep(600)\n",
    "    status =  sagemaker_client.describe_hyper_parameter_tuning_job(\n",
    "    HyperParameterTuningJobName=tuner_log.latest_tuning_job.job_name)['HyperParameterTuningJobStatus']\n",
    "    print(status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the best performance model job name from console and create endpoint and then predict using test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Import model for hosting\n",
    "container = get_image_uri(region, 'xgboost','1.0-1')\n",
    "from time import gmtime, strftime\n",
    "#job_name='sagemaker-xgboost-200508-0001-004-0bb812be'\n",
    "job_name=tuner_log.latest_tuning_job.job_name\n",
    "model_name=job_name + '-model'\n",
    "print(model_name)\n",
    "\n",
    "info = sagemaker_client.describe_training_job(TrainingJobName=job_name)\n",
    "model_data = info['ModelArtifacts']['S3ModelArtifacts']\n",
    "print(model_data)\n",
    "\n",
    "primary_container = {\n",
    "    'Image': container,\n",
    "    'ModelDataUrl': model_data\n",
    "}\n",
    "\n",
    "create_model_response = sagemaker_client.create_model(\n",
    "    ModelName = model_name,\n",
    "    ExecutionRoleArn = role,\n",
    "    PrimaryContainer = primary_container)\n",
    "\n",
    "print(create_model_response['ModelArn'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create endpoint configuration\n",
    "from time import gmtime, strftime\n",
    "\n",
    "endpoint_config_name = 'XGBoostEndpointConfig-' + strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime())\n",
    "print(endpoint_config_name)\n",
    "create_endpoint_config_response = sagemaker_client.create_endpoint_config(\n",
    "    EndpointConfigName = endpoint_config_name,\n",
    "    ProductionVariants=[{\n",
    "        'InstanceType':\"ml.m5.large\",\n",
    "        'InitialVariantWeight':1,\n",
    "        'InitialInstanceCount':1,\n",
    "        'ModelName':model_name,\n",
    "        'VariantName':'AllTraffic'}])\n",
    "\n",
    "print(\"Endpoint Config Arn: \" + create_endpoint_config_response['EndpointConfigArn'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# create end point\n",
    "\n",
    "import time\n",
    "\n",
    "endpoint_name = 'XGBoostEndpoint-' + strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime())\n",
    "print(endpoint_name)\n",
    "create_endpoint_response = sagemaker_client.create_endpoint(\n",
    "    EndpointName=endpoint_name,\n",
    "    EndpointConfigName=endpoint_config_name)\n",
    "print(create_endpoint_response['EndpointArn'])\n",
    "\n",
    "resp = sagemaker_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "status = resp['EndpointStatus']\n",
    "while status=='Creating':\n",
    "    print(\"Status: \" + status)\n",
    "    time.sleep(60)\n",
    "    resp = sagemaker_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "    status = resp['EndpointStatus']\n",
    "\n",
    "print(\"Arn: \" + resp['EndpointArn'])\n",
    "print(\"Status: \" + status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feedback = pd.read_csv('model-2.csv',header=None)\n",
    "df_feedback.head(5)\n",
    "df_feedback.columns= ['domain','correct']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feedback.loc[:,'TesT'] = df_feedback.apply(lambda row: lambda_handler({'fqdn':row['domain']},None), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score ,confusion_matrix\n",
    "df_feedback.columns\n",
    "labels = df_feedback.correct\n",
    "preds = df_feedback.TesT\n",
    "accuracy_score(labels,preds)\n",
    "confusion_matrix(labels,preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feedback[(df_feedback.correct!=df_feedback.TesT) |(df_feedback.TesT!=df_feedback.correct)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Generate predictions on the test set for the difference models\n",
    "df_test.loc[:,'TesT'] = df_test.apply(lambda row: lambda_handler({'fqdn':row['domain']},None), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.metrics import accuracy_score ,confusion_matrix\n",
    "df_feedback.columns\n",
    "labels = df_test.Y\n",
    "preds = df_test.TesT\n",
    "accuracy_score(labels,preds)\n",
    "confusion_matrix(labels,preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Tensorflow model - LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "git_config = {'repo': 'https://github.com/aftabalam01/machinelearningpipeline', \n",
    "              'branch': 'staging'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = os.path.join(os.getcwd(), 'data')\n",
    "os.makedirs(data_dir, exist_ok=True)\n",
    "\n",
    "train_dir = os.path.join(os.getcwd(), 'data/train')\n",
    "os.makedirs(train_dir, exist_ok=True)\n",
    "\n",
    "test_dir = os.path.join(os.getcwd(), 'data/test')\n",
    "os.makedirs(test_dir, exist_ok=True)\n",
    "\n",
    "csv_test_dir = os.path.join(os.getcwd(), 'data/csv-test')\n",
    "os.makedirs(csv_test_dir, exist_ok=True)\n",
    "\n",
    "Y_train = np.array(df_train_valid['Y'])\n",
    "Y_test = np.array(df_test['Y'])\n",
    "np.save(os.path.join(train_dir, 'x_train.npy'), X_train)\n",
    "np.save(os.path.join(train_dir, 'y_train.npy'), Y_train)\n",
    "np.save(os.path.join(test_dir, 'x_test.npy'), X_test)\n",
    "np.save(os.path.join(test_dir, 'y_test.npy'), Y_test)\n",
    "np.savetxt(os.path.join(csv_test_dir, 'csv-test.csv'), np.array(X_valid, dtype=np.int32), fmt='%d', delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.shape\n",
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train.shape\n",
    "X_train[2:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "\n",
    "s3_prefix = 'tf-keras-dga'\n",
    "\n",
    "traindata_s3_prefix = '{}/data/train'.format(s3_prefix)\n",
    "testdata_s3_prefix = '{}/data/test'.format(s3_prefix)\n",
    "\n",
    "train_s3 = sagemaker_session.upload_data(path='./data/train/', key_prefix=traindata_s3_prefix)\n",
    "test_s3 = sagemaker_session.upload_data(path='./data/test/', key_prefix=testdata_s3_prefix)\n",
    "\n",
    "inputs = {'train':train_s3, 'test': test_s3}\n",
    "print(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# inputs ={'train': 's3://sagemaker-us-west-2-099176660580/tf-keras-dga/data/train', 'test': 's3://sagemaker-us-west-2-099176660580/tf-keras-dga/data/test'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import logging\n",
    "logging.getLogger(\"tensorflow\").setLevel(logging.ERROR)\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "from tensorflow.keras.preprocessing import sequence\n",
    "\n",
    "max_features = 64\n",
    "maxlen = 65\n",
    "\n",
    "# print(len(x_train), 'train sequences')\n",
    "# print(len(x_test), 'test sequences')\n",
    "\n",
    "# x_train = sequence.pad_sequences(x_train, maxlen=maxlen)\n",
    "# x_test = sequence.pad_sequences(x_test, maxlen=maxlen)\n",
    "# print('x_train shape:', x_train.shape)\n",
    "# print('x_test shape:', x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.tensorflow import TensorFlow\n",
    "\n",
    "model_dir = '/opt/ml/model'\n",
    "train_instance_type = 'ml.c5.4xlarge'\n",
    "hyperparameters = {'epochs': 10, 'batch_size': 256, 'learning_rate': 0.01,'maxlen':63,'max_features':64}\n",
    "\n",
    "estimator = TensorFlow(\n",
    "                       git_config=git_config,\n",
    "                       source_dir='src/notebooks',\n",
    "                       entry_point='tf-model.py',\n",
    "                       model_dir=model_dir,\n",
    "                       train_instance_type=train_instance_type,\n",
    "                       train_instance_count=1,\n",
    "                       hyperparameters=hyperparameters,\n",
    "                       role=sagemaker.get_execution_role(),\n",
    "                       base_job_name='tf-dga',\n",
    "                       framework_version='2.1',\n",
    "                       py_version='py3',\n",
    "                       script_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator.fit(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dont forgot to delete end point\n",
    "sagemaker_client.delete_endpoint(EndpointName=endpoint_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sagemaker_client.delete_endpoint_config(EndpointConfigName=endpoint_config_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}