99_aws_interface.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.7 upload to aws s3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import boto3\n",
    "from botocore.exceptions import ClientError\n",
    "\n",
    "s3c = boto3.client('s3')\n",
    "s3r = boto3.resource('s3')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ get the list of buckets from aws s3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "ClientError",
     "evalue": "An error occurred (InvalidAccessKeyId) when calling the ListBuckets operation: The AWS Access Key Id you provided does not exist in our records.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mClientError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-20-774140f617fe>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m# Print out bucket names\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mbucket\u001b[0m \u001b[1;32min\u001b[0m \u001b[0ms3r\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuckets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m     \u001b[0mbuckets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbucket\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\sheng\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\boto3\\resources\\collection.py\u001b[0m in \u001b[0;36m__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     81\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m         \u001b[0mcount\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 83\u001b[1;33m         \u001b[1;32mfor\u001b[0m \u001b[0mpage\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpages\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     84\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0mitem\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mpage\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     85\u001b[0m                 \u001b[1;32myield\u001b[0m \u001b[0mitem\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\sheng\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\boto3\\resources\\collection.py\u001b[0m in \u001b[0;36mpages\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    159\u001b[0m                          \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmeta\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mservice_name\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    160\u001b[0m                          self._py_operation_name, params)\n\u001b[1;32m--> 161\u001b[1;33m             \u001b[0mpages\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_py_operation_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    162\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    163\u001b[0m         \u001b[1;31m# Now that we have a page iterator or single page of results\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\sheng\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\botocore\\client.py\u001b[0m in \u001b[0;36m_api_call\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    355\u001b[0m                     \"%s() only accepts keyword arguments.\" % py_operation_name)\n\u001b[0;32m    356\u001b[0m             \u001b[1;31m# The \"self\" in this scope is referring to the BaseClient.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 357\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_api_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    358\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    359\u001b[0m         \u001b[0m_api_call\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpy_operation_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\sheng\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\botocore\\client.py\u001b[0m in \u001b[0;36m_make_api_call\u001b[1;34m(self, operation_name, api_params)\u001b[0m\n\u001b[0;32m    674\u001b[0m             \u001b[0merror_code\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Error\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Code\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    675\u001b[0m             \u001b[0merror_class\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_code\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merror_code\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 676\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0merror_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparsed_response\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    677\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    678\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mClientError\u001b[0m: An error occurred (InvalidAccessKeyId) when calling the ListBuckets operation: The AWS Access Key Id you provided does not exist in our records."
     ]
    }
   ],
   "source": [
    "buckets = []\n",
    "\n",
    "# Print out bucket names\n",
    "for bucket in s3r.buckets.all():\n",
    "    buckets.append(bucket)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ define the method to upload a file to an AWS bucket."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def upload_file(file_name, bucket, object_name=None):\n",
    "    \"\"\"Upload a file to an S3 bucket\n",
    "\n",
    "    :param file_name: File to upload\n",
    "    :param bucket: Bucket to upload to\n",
    "    :param object_name: S3 object name. If not specified then file_name is used\n",
    "    :return: True if file was uploaded, else False\n",
    "    \"\"\"\n",
    "\n",
    "    # If S3 object_name was not specified, use file_name\n",
    "    if object_name is None:\n",
    "        object_name = file_name\n",
    "\n",
    "    # Upload the file\n",
    "    s3_client = boto3.client('s3')\n",
    "    try:\n",
    "        response = s3_client.upload_file(file_name, bucket, object_name)\n",
    "    except ClientError as e:\n",
    "        logging.error(e)\n",
    "        return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ upload the file to the first bucket from aws s3\n",
    "+ it will directly overide the file with the same name in the directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(upload_file('meetingSticker.db',buckets[0].name))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ download the file from AWS S3 first bucket."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(s3c.download_file(buckets[0].name,'meetingSticker.db',local_file))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### initiate amazon dynamodb connection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "\n",
    "dynDBc = boto3.client('dynamodb')\n",
    "dynDBr = boto3.resource('dynamodb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# Create the DynamoDB table.\n",
    "table = dynDBr.create_table(\n",
    "    TableName='users',\n",
    "    KeySchema=[\n",
    "        {\n",
    "            'AttributeName': 'username',\n",
    "            'KeyType': 'HASH'\n",
    "        },\n",
    "        {\n",
    "            'AttributeName': 'last_name',\n",
    "            'KeyType': 'RANGE'\n",
    "        }\n",
    "    ],\n",
    "    AttributeDefinitions=[\n",
    "        {\n",
    "            'AttributeName': 'userID',\n",
    "            'AttributeType': 'S'\n",
    "        },\n",
    "        {\n",
    "            'AttributeName': 'username',\n",
    "            'AttributeType': 'S'\n",
    "        },\n",
    "        {\n",
    "            'AttributeName': 'password',\n",
    "            'AttributeType': 'S'\n",
    "        },\n",
    "        {\n",
    "            'AttributeName': 'first_name',\n",
    "            'AttributeType': 'S'\n",
    "        },\n",
    "        {\n",
    "            'AttributeName': 'last_name',\n",
    "            'AttributeType': 'S'\n",
    "        },\n",
    "        {\n",
    "            'AttributeName': 'email',\n",
    "            'AttributeType': 'S'\n",
    "        },\n",
    "        {\n",
    "            'AttributeName': 'phone',\n",
    "            'AttributeType': 'S'\n",
    "        },\n",
    "    ],\n",
    "    ProvisionedThroughput={\n",
    "        'ReadCapacityUnits': 5,\n",
    "        'WriteCapacityUnits': 5\n",
    "    }\n",
    ")\n",
    "\n",
    "# Wait until the table exists.\n",
    "table.meta.client.get_waiter('table_exists').wait(TableName='users')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### initiate amazon s3 connection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's use Amazon S3\n",
    "\n",
    "import boto3\n",
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "s3c = boto3.client('s3')\n",
    "s3r = boto3.resource('s3')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load all buckets from s3 and second bucket's objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "buckets = []\n",
    "\n",
    "# Print out bucket names\n",
    "for bucket in s3r.buckets.all():\n",
    "    buckets.append(bucket)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ an iterator to go through all the objects from bucket[1] s3 client."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in buckets[1].objects.iterator():\n",
    "    pass\n",
    "    #print(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ get all the document/object from second bucket form the bucket list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = s3c.list_objects(Bucket = buckets[1].name)['Contents']\n",
    "textract_c = boto3.client('textract')\n",
    "\n",
    "documents_ls = [d['Key'] for d in documents]\n",
    "#documents_ls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get responses and extract their contents from amazon textract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "response_ls = []\n",
    "for d in documents_ls:\n",
    "    response_ls.append(textract_c.start_document_analysis(DocumentLocation={'S3Object':{'Bucket':buckets[1].name,'Name':d}},FeatureTypes = ['TABLES']))\n",
    "    \n",
    "jobId_ls = [r['JobId'] for r in response_ls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_ls = []\n",
    "\n",
    "for j in jobId_ls:\n",
    "    result_ls.append(textract_c.get_document_analysis(JobId=j))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "jobStatus = [r['JobStatus'] for r in result_ls]\n",
    "jobStatus_df = pd.DataSeries(jobStatus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving each raw text in result_ls to .json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_head = 'city of san jose minutes_'\n",
    "file_surfix = '.json'\n",
    "i = 0\n",
    "\n",
    "for r in result_ls:\n",
    "    raw_txt,raw_ls = creatRaw(result_ls[i])\n",
    "    with open(file_head+str(i+1)+file_surfix,'w') as f:\n",
    "        json.dump(result_ls[i],f)\n",
    "        f.close()\n",
    "    i+=1\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create raw text and raw content list from textract string input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def creatRaw(result):\n",
    "\n",
    "    raw_ls=[]\n",
    "    raw_txt=''\n",
    "    for t in result['Blocks']:\n",
    "        try:\n",
    "            if t['Text'][:4]=='Page':\n",
    "                pass\n",
    "            else:\n",
    "                raw_ls.append( t['Text'])\n",
    "                raw_txt += t['Text']+' '\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    return raw_txt,raw_ls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load and create raw text file from .json file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('./raw/city of san jose minutes_19.json') as jsonFile:\n",
    "    data = json.load(jsonFile)\n",
    "    \n",
    "rawtxt, rawls = creatRaw(data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}