升级scrapy1.0版本

alexa/alexa/pipelines.py

@@ -24,7 +24,7 @@ def process_item(self, item, spider):
         self.file.write(line)
         return item
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         self.file.close()
 
 
@@ -46,5 +46,5 @@ def process_item(self, item, spider):
             final_item = dict(item.items() + ritem.items())
         self.r.set(item['id'], final_item)
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         return

alexa/alexa/spiders/alexa_spider.py

@@ -6,12 +6,12 @@
 
 from scrapy.selector import Selector
 try:
-    from scrapy.spider import Spider
+    from scrapy.spiders import Spider
 except:
-    from scrapy.spider import BaseSpider as Spider
+    from scrapy.spiders import BaseSpider as Spider
 from scrapy.utils.response import get_base_url
-from scrapy.contrib.spiders import CrawlSpider, Rule
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor as sle
 
 
 from alexa.items import *

amazonbook/amazonbook/pipelines.py

@@ -24,7 +24,7 @@ def process_item(self, item, spider):
         self.file.write(line)
         return item
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         self.file.close()
 
 
@@ -46,5 +46,5 @@ def process_item(self, item, spider):
             final_item = dict(item.items() + ritem.items())
         self.r.set(item['id'], final_item)
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         return

amazonbook/amazonbook/spiders/spider.py

@@ -6,12 +6,12 @@
 
 from scrapy.selector import Selector
 try:
-    from scrapy.spider import Spider
+    from scrapy.spiders import Spider
 except:
-    from scrapy.spider import BaseSpider as Spider
+    from scrapy.spiders import BaseSpider as Spider
 from scrapy.utils.response import get_base_url
-from scrapy.contrib.spiders import CrawlSpider, Rule
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor as sle
 
 
 from amazonbook.items import *

dmoz/dmoz/pipelines.py

@@ -24,7 +24,7 @@ def process_item(self, item, spider):
         self.file.write(line)
         return item
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         self.file.close()
 
 
@@ -46,5 +46,5 @@ def process_item(self, item, spider):
             final_item = dict(item.items() + ritem.items())
         self.r.set(item['id'], final_item)
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         return

dmoz/dmoz/spiders/spider.py

@@ -6,12 +6,12 @@
 
 from scrapy.selector import Selector
 try:
-    from scrapy.spider import Spider
+    from scrapy.spiders import Spider
 except:
-    from scrapy.spider import BaseSpider as Spider
+    from scrapy.spiders import BaseSpider as Spider
 from scrapy.utils.response import get_base_url
-from scrapy.contrib.spiders import CrawlSpider, Rule
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor as sle
 
 
 from dmoz.items import *

doubanbook/doubanbook/pipelines.py

@@ -24,7 +24,7 @@ def process_item(self, item, spider):
         self.file.write(line)
         return item
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         self.file.close()
 
 

doubanbook/doubanbook/spiders/douban_spider.py

@@ -4,12 +4,12 @@
 
 from scrapy.selector import Selector
 try:
-    from scrapy.spider import Spider
+    from scrapy.spiders import Spider
 except:
-    from scrapy.spider import BaseSpider as Spider
+    from scrapy.spiders import BaseSpider as Spider
 from scrapy.utils.response import get_base_url
-from scrapy.contrib.spiders import CrawlSpider, Rule
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor as sle
 
 
 from doubanbook.items import *
@@ -20,12 +20,12 @@ class DoubanBookSpider(CrawlSpider):
     name = "doubanbook"
     allowed_domains = ["douban.com"]
     start_urls = [
-        "http://book.douban.com/tag/"
+        "https://book.douban.com/tag/"
     ]
     rules = [
-        Rule(sle(allow=("/subject/\d+/\?from=tag$")), callback='parse_2'),
-        Rule(sle(allow=("/tag/[^/]+/\?focus=book$", )), follow=True),
-        Rule(sle(allow=("/tag/$", )), follow=True),
+        Rule(sle(allow=("/subject/\d+$")), callback='parse_2'),
+        Rule(sle(allow=("/tag/[^/]+$", )), follow=True),
+        #Rule(sle(allow=("/tag/$", )), follow=True),
     ]
 
     def parse_2(self, response):
@@ -47,6 +47,9 @@ def parse_1(self, response):
         # url cannot encode to Chinese easily.. XXX
         info('parsed ' + str(response))
 
-    def _process_request(self, request):
+    def process_request(self, request):
         info('process ' + str(request))
         return request
+
+    def closed(self, reason):
+        info("DoubanBookSpider Closed:" + reason)

doubanmovie/doubanmovie/pipelines.py

@@ -24,9 +24,13 @@ def process_item(self, item, spider):
         self.file.write(line)
         return item
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
+        print("JsonWithEncodingPipeline closed")
         self.file.close()
 
+    def open_spider(self, spider):
+        print("JsonWithEncodingPipeline opend")
+
 
 class RedisPipeline(object):
 
@@ -46,5 +50,5 @@ def process_item(self, item, spider):
             final_item = dict(item.items() + ritem.items())
         self.r.set(item['id'], final_item)
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         return

doubanmovie/doubanmovie/settings.py

@@ -22,7 +22,7 @@
 #USER_AGENT = 'doubanmovie (+http://www.yourdomain.com)'
 
 DOWNLOADER_MIDDLEWARES = {
-   # 'misc.middleware.CustomHttpProxyMiddleware': 400,
+    #'misc.middleware.CustomHttpProxyMiddleware': 400,
     'misc.middleware.CustomUserAgentMiddleware': 401,
 }
 

doubanmovie/doubanmovie/spiders/spider.py

@@ -7,12 +7,12 @@
 
 from scrapy.selector import Selector
 try:
-    from scrapy.spider import Spider
+    from scrapy.spiders import Spider
 except:
-    from scrapy.spider import BaseSpider as Spider
+    from scrapy.spiders import BaseSpider as Spider
 from scrapy.utils.response import get_base_url
-from scrapy.contrib.spiders import CrawlSpider, Rule
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor as sle
 
 
 from doubanmovie.items import *
@@ -24,17 +24,21 @@ class doubanmovieSpider(CommonSpider):
     name = "doubanmovie"
     allowed_domains = ["douban.com"]
     start_urls = [
-        "http://movie.douban.com/chart",
+        #"https://movie.douban.com/tag/",
+        "https://movie.douban.com/chart"
     ]
     rules = [
-        Rule(sle(allow=(".*movie.douban.com/subject/[0-9]+/$")), callback='parse_1', follow=True),
+        #Rule(sle(allow=("/tag/[0-9]{4}$")), follow=True),
+        #Rule(sle(allow=("/tag/[0-9]{4}/?start=[0-9]{2,4}&type=T$")), follow=True),
+        #Rule(sle(allow=("/subject/[0-9]+$")), callback='parse_1'),
+        Rule(sle(allow=("/subject/[0-9]+/$")), callback='parse_1', follow=True),
     ]
 
     list_css_rules = { 
         '.linkto': {
             'url': 'a::attr(href)',
             'name': 'a::text',
-        }   
+        }
     }   
 
     list_css_rules_2 = { 
@@ -54,5 +58,6 @@ class doubanmovieSpider(CommonSpider):
     def parse_1(self, response):
         info('Parse '+response.url)
         x = self.parse_with_rules(response, self.content_css_rules, dict)
-        print(repr(x).decode('raw_unicode_escape'))
+        return x
+        #print(repr(x).decode('raw_unicode_escape'))
         # return self.parse_with_rules(response, self.css_rules, doubanmovieItem)

googlescholar/googlescholar/pipelines.py

@@ -24,7 +24,7 @@ def process_item(self, item, spider):
         self.file.write(line)
         return item
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         self.file.close()
 
 
@@ -46,5 +46,5 @@ def process_item(self, item, spider):
             final_item = dict(item.items() + ritem.items())
         self.r.set(item['id'], final_item)
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         return

googlescholar/googlescholar/spiders/spider.py

@@ -7,12 +7,12 @@
 
 from scrapy.selector import Selector
 try:
-    from scrapy.spider import Spider
+    from scrapy.spiders import Spider
 except:
-    from scrapy.spider import BaseSpider as Spider
+    from scrapy.spiders import BaseSpider as Spider
 from scrapy.utils.response import get_base_url
-from scrapy.contrib.spiders import CrawlSpider, Rule
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor as sle
 
 
 from googlescholar.items import *

hrtencent/hrtencent/pipelines.py

@@ -20,5 +20,5 @@ def process_item(self, item, spider):
         self.file.write(line)
         return item
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         self.file.close()

hrtencent/hrtencent/spiders/hrtencent_spider.py

@@ -4,12 +4,12 @@
 
 from scrapy.selector import Selector
 try:
-    from scrapy.spider import Spider
+    from scrapy.spiders import Spider
 except:
-    from scrapy.spider import BaseSpider as Spider
+    from scrapy.spiders import BaseSpider as Spider
 from scrapy.utils.response import get_base_url
-from scrapy.contrib.spiders import CrawlSpider, Rule
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor as sle
 
 
 from hrtencent.items import *

linkedin/linkedin/linkedin/spiders/LinkedinSpider.py

@@ -1,6 +1,6 @@
 from scrapy.selector import HtmlXPathSelector
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
-from scrapy.contrib.spiders import CrawlSpider, Rule
+from scrapy.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.spiders import CrawlSpider, Rule
 from scrapy.http import Request
 from scrapy import log
 from linkedin.items import LinkedinItem, PersonProfileItem

misc/proxy.py

@@ -5,7 +5,7 @@
 PROXIES = [
     #{"ip_port": "127.0.0.1:8087"}, #goagent
     #{"ip_port": "127.0.0.1:8118"}, #tor via privoxy
-    {"ip_port": "43.245.202.120:8080"}, #tor via privoxy
+    {"ip_port": "127.0.0.1:1080"}, #tor via privoxy
 ]
 
 FREE_PROXIES = [

misc/spider.py

@@ -12,7 +12,7 @@
     from scrapy.spiders import BaseSpider as Spider
 from scrapy.utils.response import get_base_url
 from scrapy.spiders import CrawlSpider, Rule
-from scrapy.linkextractors.sgml import SgmlLinkExtractor as sle
+from scrapy.linkextractors import LinkExtractor as sle
 
 
 from .log import *
@@ -102,8 +102,8 @@ def traversal(self, sel, rules, item_class, item, items):
                     self.traversal(i, nv, item_class, item, items)
 
     DEBUG=True
-    def debug(sth):
-        if DEBUG == True:
+    def debug(self, sth):
+        if self.DEBUG == True:
             print(sth)
 
     def deal_text(self, sel, item, force_1_item, k, v):
@@ -144,7 +144,7 @@ def dfs(self, sel, rules, item_class, force_1_item):
 
         items = []
         if item_class != dict:
-            self.traversal(sel, rules, item_class, None, items, force_1_item)
+            self.traversal(sel, rules, item_class, None, items)
         else:
             self.traversal_dict(sel, rules, item_class, None, items, force_1_item)
 

proxylist/proxylist/pipelines.py

@@ -31,7 +31,7 @@ def process_item(self, item, spider):
         self.file.write(line)
         return item
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         self.file.close()
 
 
@@ -76,5 +76,5 @@ def process_item(self, item, spider):
             final_item = dict(item.items() + ritem.items())
         self.r.set(item['id'], final_item)
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         return

proxylist/proxylist/spiders/spider.py

@@ -12,7 +12,7 @@
     from scrapy.spiders import BaseSpider as Spider
 from scrapy.utils.response import get_base_url
 from scrapy.spiders import CrawlSpider, Rule
-from scrapy.linkextractors.sgml import SgmlLinkExtractor as sle
+from scrapy.linkextractors import LinkExtractor as sle
 from scrapy.linkextractors import LinkExtractor as sle
 
 

qqnews/qqnews/pipelines.py

@@ -24,7 +24,7 @@ def process_item(self, item, spider):
         self.file.write(line)
         return item
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         self.file.close()
 
 
@@ -46,5 +46,5 @@ def process_item(self, item, spider):
             final_item = dict(item.items() + ritem.items())
         self.r.set(item['id'], final_item)
 
-    def spider_closed(self, spider):
+    def close_spider(self, spider):
         return

qqnews/qqnews/spiders/spider.py

@@ -4,12 +4,12 @@
 
 from scrapy.selector import Selector
 try:
-    from scrapy.spider import Spider
+    from scrapy.spiders import Spider
 except:
-    from scrapy.spider import BaseSpider as Spider
+    from scrapy.spiders import BaseSpider as Spider
 from scrapy.utils.response import get_base_url
-from scrapy.contrib.spiders import CrawlSpider, Rule
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor as sle
 
 
 from qqnews.items import *