1.modify MSGTYPE

2.Change class scheduler extend by class ThreadBase

Common/Msg.py

@@ -9,11 +9,12 @@
 #######################################################
 
 class MSGTYPE(object):
-  CALL = 1
-  RETURN = 2
+  STOP = 0
+  INNER = 1
+  OUTER = 2
 
 class Msg(object):
-    def __init__(self, srcModel, dstModel, data = None, msgType = MSGTYPE.CALL, action = None):
+    def __init__(self, srcModel, dstModel, data = None, msgType = None, action = None):
       self.__sequence = 0
       self.__src = srcModel
       self.__dst = dstModel

Common/Scheduler.py

@@ -3,14 +3,23 @@
 
 @author: MyGeN
 '''
+import sys
+try:
+    sys.path.index('../')
+    pass
+except Exception:
+    sys.path.append('../')
+    pass
 
 from Common.Msg import *
+from Common.ThreadBase import ThreadBase
 
-class Scheduler(object):
+class Scheduler(ThreadBase):
     '''
               消息调度器类，主要负责消息转发
     '''
     def __init__(self):
+      ThreadBase.__init__(self,'Scheduler')
       self.Models = dict()
 
     # Add a model to self
@@ -20,6 +29,10 @@ def RegisterModel(self, Name, Model):
 
     # Send message to target model
     def PostMsg(self,msg):
+      self.ReviceMsg(msg)
+      pass
+
+    def HandleMsg(self, msg):
       modelName = msg.get_dst()
       self.Models[modelName].ReviceMsg(msg)
       pass

Common/ThreadBase.py

@@ -8,62 +8,66 @@
 # Original author: MyGeN
 # 
 #######################################################
+import sys
+try:
+    sys.path.index('../')
+    pass
+except Exception:
+    sys.path.append('../')
+    pass
 
 import queue
 from threading import Thread
-
 from Common.Msg import *
-from Common.Scheduler import Scheduler
 
 # TODO: Need to Add Loger
 
 class ThreadBase(Thread):
-	def __init__(self,modelName):
-		Thread.__init__(self)
-		self.MsgQueue = queue.Queue()
-		self.HandleCallback = dict()
-		self.ModelName = modelName
-		self.Schedule = None
-		pass
+    def __init__(self,modelName):
+        Thread.__init__(self)
+        self.MsgQueue = queue.Queue()
+        self.HandleCallback = dict()
+        self.ModelName = modelName
+        self.Schedule = None
+        pass
 
-	def HandleMsg(self, msg):
-		action = 'Default' if msg.Action == None else msg.Action
-		callback = self.HandleCallback[action]
-		needReturn = callback(msg.Data)
-		if needReturn:
-			msg.WarpSrcDst()
-			self.PostMsg(msg)
-		pass
-	
-	# Send the message
-	def PostMsg(self,msg):
-		if self.Schedule != None:
-			self.Schedule.PostMsg(msg)
-		else:
-			raise('This Model is not register to any scheduler, you need to call RegisterTo(schduler) to Register a scheduler')
-		pass
-	
-	# Register self to a scheduler
-	def RegisterTo(self, scheduler):
-		self.Schedule = scheduler
-		scheduler.RegisterModel(self.ModelName, self)
-		pass
-	
-	# Revice message to message queue
-	def ReviceMsg(self, msg):
-		self.MsgQueue.put(msg, block = False)
-		pass
+    def HandleMsg(self, msg):
+        action = 'Default' if msg.Action == None else msg.Action
+        callback = self.HandleCallback[action]
+        needReturn = callback(msg.Data)
+        if needReturn:
+            msg.WarpSrcDst()
+            self.PostMsg(msg)
+        pass
+    
+    # Send the message
+    def PostMsg(self,msg):
+        if self.Schedule != None:
+            self.Schedule.PostMsg(msg)
+        else:
+            self.ReviceMsg(msg)
+        pass
+    
+    # Register self to a scheduler
+    def RegisterTo(self, scheduler):
+        self.Schedule = scheduler
+        scheduler.RegisterModel(self.ModelName, self)
+        pass
+    
+    # Revice message to message queue
+    def ReviceMsg(self, msg):
+        self.MsgQueue.put(msg, block = False)
+        pass
 
-	# overwrite the function
-	def run(self):
-		while True:
-			msg = self.MsgQueue.get(True)
-			if msg != None and msg.Type != MSGTYPE.STOP:
-				self.HandleMsg(msg)
-			elif msg.Type == MSGTYPE.STOP:
-				break
-				pass
-		# 清除消息队列
-		del self.MsgQueue
-		del self.HandleCallback
-		pass
+    # overwrite the function
+    def run(self):
+        while True:
+            msg = self.MsgQueue.get(True)
+            if msg != None and msg.Type != MSGTYPE.STOP:
+                self.HandleMsg(msg)
+            elif msg.Type == MSGTYPE.STOP:
+                break
+        # 清除消息队列
+        del self.MsgQueue
+        del self.HandleCallback
+        pass

README.md

@@ -1,2 +1,3 @@
 WebSpider
 =========
+this is a high level to config and multithread web spider

WebCrawl/Feeder.py

@@ -8,14 +8,35 @@
 # 
 #######################################################
 
-from Common.ThreadBase import ThreadBase
+import sys
+try:
+    sys.path.index('../')
+    pass
+except Exception:
+    sys.path.append('../')
+    pass
 
-class Feeder(ThreadBase):
-    def __init__(self):
+from Common.Scheduler import Scheduler
+from WebCrawl.Spider import *
+
+class Feeder(Scheduler):
+    def __init__(self,maxSpiderCount):
+        Scheduler.__init__(self,'SpiderManager')
+        self.SpiderPool = list(maxSpiderCount)
         pass
 
     def GetSpider(self,name):
+        for spider in self.SpiderPool:
+            if spider.GetSpiderStatus()==SPIDERFLAG.REDAY:
+                return spider
+                pass
+            pass
+        return None
+
+    def HandleMsg(self,msg):
+
         pass
 
     def SendSpider(self,pay):
+        #TODO: SendSpider and bulid msg 
         pass

WebCrawl/Spider.py

@@ -7,20 +7,46 @@
 # Original author: MyGeN
 # 
 #######################################################
+import sys
+try:
+    sys.path.index('../')
+    pass
+except Exception:
+    sys.path.append('../')
+    pass
+
 import urllib.request
+from Common.ThreadBase import ThreadBase
+from Common.Msg import *
 
+class SPIDERFLAG(object):
+    RUN = 0
+    REDAY = 1
+
 
-class Spider(object):
-    def __init__(self, feeder):
-        self.__feeder = feeder
+class Spider(ThreadBase):
+    def __init__(self, spiderName):
+        ThreadBase.__init__(self,spiderName)
+        self.__status = SPIDERFLAG.REDAY
         pass
 
     def Feed(self, url):
+        self.__status = SPIDERFLAG.RUN
         self.__url = url
         pass
 
     def Go(self):
+        self.start()
+        return data
+
+    def run(self):
         req = urllib.request.urlopen(self.__url)
         data = req.read().decode('utf-8')
         req.close()
-        return data
+        msg = Msg(self.ModelName,)
+        self.__status = SPIDERFLAG.REDAY
+        pass
+
+    def GetSpiderStatus(self):
+        return self.__status
+        pass