Indiegogo Search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get html from Indiegogo\n",
    "## Explore a category and feed that URL into function : get_pages_source(URL)\n",
    "## It will download first 10 items in that category in html file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from selenium import webdriver     \n",
    "import urllib.request\n",
    "import time\n",
    "import requests\n",
    "headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "\n",
      "<head>\n",
      "<META NAME=\"ROBOTS\" CONTENT=\"NOINDEX, NOFOLLOW\">\n",
      "<meta http-equiv=\"cache-control\" content=\"max-age=0\" />\n",
      "<meta http-equiv=\"cache-control\" content=\"no-cache\" />\n",
      "<meta http-equiv=\"expires\" content=\"0\" />\n",
      "<meta http-equiv=\"expires\" content=\"Tue, 01 Jan 1980 1:00:00 GMT\" />\n",
      "<meta http-equiv=\"pragma\" content=\"no-cache\" />\n",
      "<meta http-equiv=\"refresh\" content=\"10; url=/distil_r_captcha.html?requestId=4df34c63-7fc2-4ce7-ace2-54d8f8c25c35&httpReferrer=%2Fexplore%2Ftech-innovation%3Fquick_filter%3Dtrending%26location%3Deverywhere%26project_type%3Dall%26percent_funded%3Dall%26goal_type%3Dall%26more_options%3Dfalse%26status%3Dall\" />\n",
      "<script type=\"text/javascript\">\n",
      "\t(function(window){\n",
      "\t\ttry {\n",
      "\t\t\tif (typeof sessionStorage !== 'undefined'){\n",
      "\t\t\t\tsessionStorage.setItem('distil_referrer', document.referrer);\n",
      "\t\t\t}\n",
      "\t\t} catch (e){}\n",
      "\t})(window);\n",
      "</script>\n",
      "<script type=\"text/javascript\" src=\"/gogodstltsyxucccrxcwtzaezvatfuw.js\" defer></script><style type=\"text/css\">#d__fFH{position:absolute;top:-5000px;left:-5000px}#d__fF{font-family:serif;font-size:200px;visibility:hidden}#vaaryvrdqefezzbztccbxvszfywr{display:none!important}</style></head>\n",
      "<body>\n",
      "<div id=\"distilIdentificationBlock\">&nbsp;</div>\n",
      "</body>\n",
      "</html>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Use request to test if we can get the data.\n",
    "resp = requests.get('https://www.indiegogo.com/explore/tech-innovation?quick_filter=trending&location=everywhere&project_type=all&percent_funded=all&goal_type=all&more_options=false&status=all',headers = headers)\n",
    "print(resp.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_pages_source(url):\n",
    "    Driver = webdriver.Firefox(executable_path='/Users/Craig/modules_download/geckodriver')\n",
    "\n",
    "    # It's strange that use Selenium with Chrome will be rejected by the website...\n",
    "    # a.k.a Driver = webdriver.Chrome(executable_path='/Users/Craig/chromedriver') Dosen't work\n",
    "\n",
    "    Driver.get(url)\n",
    "\n",
    "    # To load the Massive javascript-generated data.\n",
    "    time.sleep(10)\n",
    "\n",
    "    # Collect the first ten items url.\n",
    "    urls = []\n",
    "    for i in range(1,11):\n",
    "        msg = Driver.find_element_by_xpath('/html/body/div[2]/div/div/div[4]/div/explore-detail/div/div[2]/div[4]/div['+str(i)+']/div/a[1]')\n",
    "        urls.append(msg.get_attribute('href'))\n",
    "    for url in urls:\n",
    "\n",
    "        # Wait for the javascript...\n",
    "        time.sleep(5)\n",
    "\n",
    "        # Redirect to next page\n",
    "        Driver.get(url)\n",
    "        name = Driver.find_element_by_xpath('/html/body/div[2]/div/div/campaign-page/div/div/campaign-header-basics/div/div[1]/div[1]')\n",
    "        # Save the html source to further purpose.\n",
    "        f = open('Data/Indiegogo/'+ name.text + '.html', 'w')\n",
    "        # In windows with native encode('cp950') , we have to change to\n",
    "        # f = open('Data/Indiegogo/'+ name.text + '.html','w',encoding='utf8')  \n",
    "\n",
    "        f.write(Driver.page_source)\n",
    "        f.close()\n",
    "        print('Downloaded:',name.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloaded: Ultraloq - Fingerprint, Fob & Bluetooth Smart Lock\n",
      "Downloaded: ZeTime: hybrid smartwatch with hands over screen\n",
      "Downloaded: Evapolar 2: Smart Personal Air Conditioner\n",
      "Downloaded: CineRangeFinder - Cinematography & AC accessory.\n",
      "Downloaded: The PAPR WATCH\n",
      "Downloaded: CorePro - Activate Your Core to Support Your Back\n",
      "Downloaded: Reolink Argus: Truly Wire-Free Security Camera\n",
      "Downloaded: The Freedom Charger!\n",
      "Downloaded: Titan Note: Change The Way You Take Notes\n",
      "Downloaded: Keyboard Mod: A Physical Keyboard For The Moto Z\n"
     ]
    }
   ],
   "source": [
    "get_pages_source('https://www.indiegogo.com/explore/tech-innovation?quick_filter=trending&location=everywhere&project_type=all&percent_funded=all&goal_type=all&more_options=false&status=all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}