Day025_MultiPage_Sample.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Day025_MultiPage_Sample.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMkTqKiJu7eNo0vlFp+V2yD",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kuonumber/crawler/blob/master/Day025_MultiPage_Sample.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "569_qTGVZURL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import requests\n",
        "import re\n",
        "import json\n",
        "from urllib.parse import urljoin\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "# PTT 八卦版網址\n",
        "PTT_URL = 'https://www.ptt.cc/bbs/Gossiping/index.html'"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lp6ukHSVZWDk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def crawl_article(url):\n",
        "    response = requests.get(url, cookies={'over18': '1'})\n",
        "    \n",
        "    # 假設網頁回應不是 200 OK 的話, 我們視為傳送請求失敗\n",
        "    if response.status_code != 200:\n",
        "        print('Error - {} is not available to access'.format(url))\n",
        "        return\n",
        "    \n",
        "    # 將網頁回應的 HTML 傳入 BeautifulSoup 解析器, 方便我們根據標籤 (tag) 資訊去過濾尋找\n",
        "    soup = BeautifulSoup(response.text)\n",
        "    \n",
        "    # 取得文章內容主體\n",
        "    main_content = soup.find(id='main-content')\n",
        "    \n",
        "    # 假如文章有屬性資料 (meta), 我們在從屬性的區塊中爬出作者 (author), 文章標題 (title), 發文日期 (date)\n",
        "    metas = main_content.select('div.article-metaline')\n",
        "    author = ''\n",
        "    title = ''\n",
        "    date = ''\n",
        "    if metas:\n",
        "        if metas[0].select('span.article-meta-value')[0]:\n",
        "            author = metas[0].select('span.article-meta-value')[0].string\n",
        "        if metas[1].select('span.article-meta-value')[0]:\n",
        "            title = metas[1].select('span.article-meta-value')[0].string\n",
        "        if metas[2].select('span.article-meta-value')[0]:\n",
        "            date = metas[2].select('span.article-meta-value')[0].string\n",
        "\n",
        "        # 從 main_content 中移除 meta 資訊（author, title, date 與其他看板資訊）\n",
        "        #\n",
        "        # .extract() 方法可以參考官方文件\n",
        "        #  - https://www.crummy.com/software/BeautifulSoup/bs4/doc/#extract\n",
        "        for m in metas:\n",
        "            m.extract()\n",
        "        for m in main_content.select('div.article-metaline-right'):\n",
        "            m.extract()\n",
        "    \n",
        "    # 取得留言區主體\n",
        "    pushes = main_content.find_all('div', class_='push')\n",
        "    for p in pushes:\n",
        "        p.extract()\n",
        "    \n",
        "    # 假如文章中有包含「※ 發信站: 批踢踢實業坊(ptt.cc), 來自: xxx.xxx.xxx.xxx」的樣式\n",
        "    # 透過 regular expression 取得 IP\n",
        "    # 因為字串中包含特殊符號跟中文, 這邊建議使用 unicode 的型式 u'...'\n",
        "    try:\n",
        "        ip = main_content.find(text=re.compile(u'※ 發信站:'))\n",
        "        ip = re.search('[0-9]*\\.[0-9]*\\.[0-9]*\\.[0-9]*', ip).group()\n",
        "    except Exception as e:\n",
        "        ip = ''\n",
        "    \n",
        "    # 移除文章主體中 '※ 發信站:', '◆ From:', 空行及多餘空白 (※ = u'\\u203b', ◆ = u'\\u25c6')\n",
        "    # 保留英數字, 中文及中文標點, 網址, 部分特殊符號\n",
        "    #\n",
        "    # 透過 .stripped_strings 的方式可以快速移除多餘空白並取出文字, 可參考官方文件 \n",
        "    #  - https://www.crummy.com/software/BeautifulSoup/bs4/doc/#strings-and-stripped-strings\n",
        "    filtered = []\n",
        "    for v in main_content.stripped_strings:\n",
        "        # 假如字串開頭不是特殊符號或是以 '--' 開頭的, 我們都保留其文字\n",
        "        if v[0] not in [u'※', u'◆'] and v[:2] not in [u'--']:\n",
        "            filtered.append(v)\n",
        "\n",
        "    # 定義一些特殊符號與全形符號的過濾器\n",
        "    expr = re.compile(u'[^一-龥。；，：“”（）、？《》\\s\\w:/-_.?~%()]')\n",
        "    for i in range(len(filtered)):\n",
        "        filtered[i] = re.sub(expr, '', filtered[i])\n",
        "    \n",
        "    # 移除空白字串, 組合過濾後的文字即為文章本文 (content)\n",
        "    filtered = [i for i in filtered if i]\n",
        "    content = ' '.join(filtered)\n",
        "    \n",
        "    # 處理留言區\n",
        "    # p 計算推文數量\n",
        "    # b 計算噓文數量\n",
        "    # n 計算箭頭數量\n",
        "    p, b, n = 0, 0, 0\n",
        "    messages = []\n",
        "    for push in pushes:\n",
        "        # 假如留言段落沒有 push-tag 就跳過\n",
        "        if not push.find('span', 'push-tag'):\n",
        "            continue\n",
        "        \n",
        "        # 過濾額外空白與換行符號\n",
        "        # push_tag 判斷是推文, 箭頭還是噓文\n",
        "        # push_userid 判斷留言的人是誰\n",
        "        # push_content 判斷留言內容\n",
        "        # push_ipdatetime 判斷留言日期時間\n",
        "        push_tag = push.find('span', 'push-tag').string.strip(' \\t\\n\\r')\n",
        "        push_userid = push.find('span', 'push-userid').string.strip(' \\t\\n\\r')\n",
        "        push_content = push.find('span', 'push-content').strings\n",
        "        push_content = ' '.join(push_content)[1:].strip(' \\t\\n\\r')\n",
        "        push_ipdatetime = push.find('span', 'push-ipdatetime').string.strip(' \\t\\n\\r')\n",
        "\n",
        "        # 整理打包留言的資訊, 並統計推噓文數量\n",
        "        messages.append({\n",
        "            'push_tag': push_tag,\n",
        "            'push_userid': push_userid,\n",
        "            'push_content': push_content,\n",
        "            'push_ipdatetime': push_ipdatetime})\n",
        "        if push_tag == u'推':\n",
        "            p += 1\n",
        "        elif push_tag == u'噓':\n",
        "            b += 1\n",
        "        else:\n",
        "            n += 1\n",
        "    \n",
        "    # 統計推噓文\n",
        "    # count 為推噓文相抵看這篇文章推文還是噓文比較多\n",
        "    # all 為總共留言數量 \n",
        "    message_count = {'all': p+b+n, 'count': p-b, 'push': p, 'boo': b, 'neutral': n}\n",
        "    \n",
        "    # 整理文章資訊\n",
        "    data = {\n",
        "        'url': url,\n",
        "        'article_author': author,\n",
        "        'article_title': title,\n",
        "        'article_date': date,\n",
        "        'article_content': content,\n",
        "        'ip': ip,\n",
        "        'message_count': message_count,\n",
        "        'messages': messages\n",
        "    }\n",
        "    return data"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bgR4NaVfbAYX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 390
        },
        "outputId": "d18e771f-0a70-4c0c-f9c8-80353bee6a72"
      },
      "source": [
        "# 對文章列表送出請求並取得列表主體\n",
        "resp = requests.get(PTT_URL, cookies={'over18': '1'})\n",
        "soup = BeautifulSoup(resp.text)\n",
        "main_list = soup.find('div', class_='bbs-screen')\n",
        "all_data = []\n",
        "\n",
        "# 依序檢查文章列表中的 tag, 遇到分隔線就結束, 忽略這之後的文章\n",
        "for div in main_list.findChildren('div', recursive=False):\n",
        "    class_name = div.attrs['class']\n",
        "    \n",
        "    # 遇到分隔線要處理的情況\n",
        "    if class_name and 'r-list-sep' in class_name:\n",
        "        print('Reach the last article')\n",
        "        break\n",
        "    # 遇到目標文章\n",
        "    if class_name and 'r-ent' in class_name:\n",
        "        div_title = div.find('div', class_='title')\n",
        "        a_title = div_title.find('a', href=True)\n",
        "        article_URL = urljoin(PTT_URL, a_title['href'])\n",
        "        article_title = a_title.text\n",
        "        print('Parse {} - {}'.format(article_title, article_URL))\n",
        "        \n",
        "        # 呼叫上面寫好的 function 來對文章進行爬蟲\n",
        "        parse_data = crawl_article(article_URL)\n",
        "        \n",
        "        # 將爬完的資料儲存\n",
        "        all_data.append(parse_data)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Parse Re: [新聞]廢碩引爆台大內戰 「碩班拉到3年」是解方 - https://www.ptt.cc/bbs/Gossiping/M.1599814497.A.00B.html\n",
            "Parse Re: [新聞] 無本生意煉金術？立委控：業者貸款買地埋 - https://www.ptt.cc/bbs/Gossiping/M.1599814611.A.1C0.html\n",
            "Parse [問卦] Y女算是自甘墮落嗎чОдОг - https://www.ptt.cc/bbs/Gossiping/M.1599814617.A.E42.html\n",
            "Parse [問卦] 妹妹疑似在我書桌上運動怎麼辦☹  - https://www.ptt.cc/bbs/Gossiping/M.1599814640.A.0C5.html\n",
            "Parse [新聞] 出外散步突暈眩　美11歲男童開賓士救阿嬤 - https://www.ptt.cc/bbs/Gossiping/M.1599814643.A.71F.html\n",
            "Parse [問卦] 台灣人寧願墮胎三次也不願增產報國? - https://www.ptt.cc/bbs/Gossiping/M.1599814661.A.422.html\n",
            "Parse Re: [新聞]珠海飯店爆炸255人送醫?當局急出面:數字誤 - https://www.ptt.cc/bbs/Gossiping/M.1599814685.A.93C.html\n",
            "Parse [新聞] WIRED25風雲人物 蔡總統陳建仁唐鳳上榜 - https://www.ptt.cc/bbs/Gossiping/M.1599814690.A.7E6.html\n",
            "Parse [新聞] 華為拉貨效應 台積營收創新高 - https://www.ptt.cc/bbs/Gossiping/M.1599814690.A.0E5.html\n",
            "Parse [問卦] 大家懷念當兵哪一味？帶毛豬腳正常嗎 - https://www.ptt.cc/bbs/Gossiping/M.1599814731.A.60A.html\n",
            "Parse [新聞] 酒駕男國道自撞 路過駕駛幸災樂禍拍抖音 - https://www.ptt.cc/bbs/Gossiping/M.1599814754.A.FDF.html\n",
            "Parse [問卦] 補家有女友需注意什麼嗎？ - https://www.ptt.cc/bbs/Gossiping/M.1599814790.A.687.html\n",
            "Parse [問卦] 我這摸帥沒女生聊天丁這摸醜 墮三次 - https://www.ptt.cc/bbs/Gossiping/M.1599814807.A.B02.html\n",
            "Parse Re: [問卦] 劉德華哪一首歌最好聽? - https://www.ptt.cc/bbs/Gossiping/M.1599814882.A.2F4.html\n",
            "Parse Re: [新聞]廢碩引爆台大內戰 「碩班拉到3年」是 - https://www.ptt.cc/bbs/Gossiping/M.1599814957.A.AFD.html\n",
            "Parse Re: [爆卦] 綠色女權女神周芷若：逼女性主義者表態 - https://www.ptt.cc/bbs/Gossiping/M.1599814973.A.79A.html\n",
            "Parse Re: [問卦] 日本有難吃的東西嗎？ - https://www.ptt.cc/bbs/Gossiping/M.1599815022.A.D70.html\n",
            "Parse Re: [新聞] 柯P出招!聯手591抓「違規宅」　廣告抓到 - https://www.ptt.cc/bbs/Gossiping/M.1599815120.A.605.html\n",
            "Parse Re: [問卦] 路上巧遇高中交往的女友，該復合嗎？ - https://www.ptt.cc/bbs/Gossiping/M.1599815129.A.E8E.html\n",
            "Parse Re: [問卦] 所以殺館長幕後找到了嗎？ - https://www.ptt.cc/bbs/Gossiping/M.1599815163.A.BF8.html\n",
            "Reach the last article\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OYxE1coXbD-I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('parse_data.json', 'w+') as f:\n",
        "    json.dump(all_data, f, ensure_ascii=False, indent=4)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}