add unit test for spider

00_Reserch/Spider.py

@@ -5,20 +5,21 @@
 import sys
 import codecs
 
+
 def getHtml(url):
-  req = urllib.request.urlopen('http://www.pythontab.com/')
-  f = open('result.txt','w')
-  data = req.read()
-  req.close()
-  data = data.decode('utf-8')
-  f.write(data)
-  f.close()
+    req = urllib.request.urlopen('http://www.pythontab.com/')
+    f = open('result.txt', 'w')
+    data = req.read()
+    req.close()
+    data = data.decode('utf-8')
+    f.write(data)
+    f.close()
 
-if __name__=='__main__':
-  req = urllib.request.urlopen('http://manhua.ali213.net/comic/2121/')
-  f = open('result.txt','w',encoding='utf-8')
-  data = req.read()
-  req.close()
-  data = data.decode('utf-8')
-  f.write(data)
-  f.close()
+if __name__ == '__main__':
+    req = urllib.request.urlopen('http://manhua.ali213.net/comic/2121/')
+    f = open('result.txt', 'w', encoding='utf-8')
+    data = req.read()
+    req.close()
+    data = data.decode('utf-8')
+    f.write(data)
+    f.close()

Common/AppConfigTool.py

@@ -0,0 +1,7 @@
+'''
+Created on 2014年6月12日
+
+@author: MyGeN
+
+Description:Config Tool Define
+'''

Common/Logger.py

@@ -0,0 +1,16 @@
+'''
+Created on 2014年6月12日
+
+@author: MyGeN
+
+Description:Logger Define
+'''
+import sys
+try:
+    sys.path.index('../')
+    pass
+except Exception:
+    sys.path.append('../')
+    pass
+
+import logging

Common/Msg.py

@@ -1,94 +1,92 @@
-#######################################################
-# 
+#
+#
 # Msg.py
 # Python implementation of the Class Msg
 # Generated by Enterprise Architect
 # Created on:      22-05-2014 23:15:10
 # Original author: MyGeN
-# 
-#######################################################
+#
+#
+
 
 class MSGTYPE(object):
-  STOP = 0
-  INNER = 1
-  OUTER = 2
+    STOP = 0
+    INNER = 1
+    OUTER = 2
+
 
 class Msg(object):
-    def __init__(self, srcModel, dstModel, data = None, msgType = None, action = None):
-      self.__sequence = 0
-      self.__src = srcModel
-      self.__dst = dstModel
-      self.__data = data
-      self.__action = action
-      self.__type = msgType
-      pass
+    ID = 0
+
+    @staticmethod
+    def GenerateMsgID():
+        Msg.ID += 1
+        return Msg.ID
+
+    def __init__(self, srcModel, dstModel,
+                 data=None, msgType=None, action=None):
+        self.__sequence = 0
+        self.__src = srcModel
+        self.__dst = dstModel
+        self.__data = data
+        self.__action = action
+        self.__type = msgType
+        pass
 
     def WarpSrcDst(self):
-      self.__src, self.__dst = self.__dst, self.__src
-      pass
+        self.__src, self.__dst = self.__dst, self.__src
+        pass
 
-    #===========================================================================
+    #=========================================================================
     # 属性
-    #===========================================================================
+    #=========================================================================
     def get_data(self):
-      return self.__data
-
+        return self.__data
 
     def get_type(self):
-      return self.__type
-
+        return self.__type
 
     def get_src(self):
-      return self.__src
-
+        return self.__src
 
     def get_dst(self):
-      return self.__dst
+        return self.__dst
 
     def get_action(self):
-      return self.__action
-
+        return self.__action
 
     def set_data(self, value):
-      self.__data = value
-
+        self.__data = value
 
     def set_type(self, value):
-      self.__type = value
-
+        self.__type = value
 
     def set_src(self, value):
-      self.__src = value
-
+        self.__src = value
 
     def set_dst(self, value):
-      self.__dst = value
+        self.__dst = value
 
     def set_action(self, value):
-      self.__action = value
-
+        self.__action = value
 
     def del_data(self):
-      del self.__data
-
+        del self.__data
 
     def del_type(self):
-      del self.__type
-
+        del self.__type
 
     def del_src(self):
-      del self.__src
-
+        del self.__src
 
     def del_dst(self):
-      del self.__dst
+        del self.__dst
 
     def del_action(self):
-      del self.__action
+        del self.__action
 
     Data = property(get_data, set_data, del_data, "data's docstring")
     Type = property(get_type, set_type, del_type, "type's docstring")
     Src = property(get_src, set_src, del_src, "src's docstring")
     Dst = property(get_dst, set_dst, del_dst, "dst's docstring")
     Action = property(get_action, set_action, del_action, "action's docstring")
-

Common/Scheduler.py

@@ -14,25 +14,28 @@
 from Common.Msg import *
 from Common.ThreadBase import ThreadBase
 
+
 class Scheduler(ThreadBase):
+
     '''
               消息调度器类，主要负责消息转发
     '''
+
     def __init__(self):
-      ThreadBase.__init__(self,'Scheduler')
-      self.Models = dict()
-    
+        ThreadBase.__init__(self, 'Scheduler')
+        self.Models = dict()
+
     # Add a model to self
     def RegisterModel(self, Name, Model):
-      self.Models[Name] = Model
-      pass
-    
+        self.Models[Name] = Model
+        pass
+
     # Send message to target model
-    def PostMsg(self,msg):
-      self.ReviceMsg(msg)
-      pass
+    def PostMsg(self, msg):
+        self.ReviceMsg(msg)
+        pass
 
     def HandleMsg(self, msg):
-      modelName = msg.get_dst()
-      self.Models[modelName].ReviceMsg(msg)
-      pass
+        modelName = msg.get_dst()
+        self.Models[modelName].ReviceMsg(msg)
+        pass

Common/ThreadBase.py

@@ -1,13 +1,13 @@
 # -*- Coding:utf-8 -*-
-#######################################################
-# 
+#
+#
 # ThreadBase.py
 # Python implementation of the Class ThreadBase
 # Generated by Enterprise Architect
 # Created on:      22-����-2014 23:15:10
 # Original author: MyGeN
-# 
-#######################################################
+#
+#
 import sys
 try:
     sys.path.index('../')
@@ -22,8 +22,10 @@
 
 # TODO: Need to Add Loger
 
+
 class ThreadBase(Thread):
-    def __init__(self,modelName):
+
+    def __init__(self, modelName):
         Thread.__init__(self)
         self.MsgQueue = queue.Queue()
         self.HandleCallback = dict()
@@ -32,42 +34,44 @@ def __init__(self,modelName):
         pass
 
     def HandleMsg(self, msg):
-        action = 'Default' if msg.Action == None else msg.Action
+        action = 'Default'
+        if msg.Action is not None:
+            action = msg.Action
         callback = self.HandleCallback[action]
-        needReturn = callback(msg.Data)
-        if needReturn:
+        if callback is None:
+            return
+        returnData = callback(msg.Data)
+        if returnData:
             msg.WarpSrcDst()
+            msg.Data = returnData
             self.PostMsg(msg)
         pass
-    
+
     # Send the message
-    def PostMsg(self,msg):
-        if self.Schedule != None:
+    def PostMsg(self, msg):
+        if self.Schedule is not None:
             self.Schedule.PostMsg(msg)
         else:
             self.ReviceMsg(msg)
         pass
-    
+
     # Register self to a scheduler
     def RegisterTo(self, scheduler):
         self.Schedule = scheduler
         scheduler.RegisterModel(self.ModelName, self)
         pass
-    
+
     # Revice message to message queue
     def ReviceMsg(self, msg):
-        self.MsgQueue.put(msg, block = False)
+        self.MsgQueue.put(msg, block=False)
         pass
 
     # overwrite the function
     def run(self):
         while True:
             msg = self.MsgQueue.get(True)
-            if msg != None and msg.Type != MSGTYPE.STOP:
+            if msg is not None and msg.Type != MSGTYPE.STOP:
                 self.HandleMsg(msg)
             elif msg.Type == MSGTYPE.STOP:
                 break
-        # 清除消息队列
-        del self.MsgQueue
-        del self.HandleCallback
         pass

DataSave/DataManager.py

@@ -1,14 +1,23 @@
-#######################################################
-# 
+#
+#
 # DataManager.py
 # Python implementation of the Class DataManager
 # Generated by Enterprise Architect
 # Created on:      22-����-2014 23:10:40
 # Original author: MyGeN
-# 
-#######################################################
+#
+#
+
+import sys
+try:
+    sys.path.index('../')
+    pass
+except Exception:
+    sys.path.append('../')
+    pass
 
 from Common.ThreadBase import ThreadBase
 
+
 class DataManager(ThreadBase):
     pass

DataSave/SaveDataBase.py

@@ -1,13 +1,21 @@
-#######################################################
-# 
+#
+#
 # SaveDataBase.py
 # Python implementation of the Class SaveDataBase
 # Generated by Enterprise Architect
-# Created on:      22-ÎåÔÂ-2014 23:10:40
+# Created on:      22-????-2014 23:10:40
 # Original author: MyGeN
-# 
-#######################################################
+#
+#
+
+import sys
+try:
+    sys.path.index('../')
+    pass
+except Exception:
+    sys.path.append('../')
+    pass
 
 
 class SaveDataBase:
-    pass
+    pass