init repository

.gitignore

@@ -52,3 +52,6 @@ coverage.xml
 # Sphinx documentation
 docs/_build/
 
+.project
+.pydevproject
+*.ldb

00_Reserch/DMHtmlParser.py

@@ -0,0 +1,27 @@
+# -*- Coding:utf-8 -*-
+import codecs
+from html.parser import HTMLParser
+import sys
+
+
+class DMHtmlParser(HTMLParser):
+  def __init__(self):
+    HTMLParser.__init__(self)
+    self.__bContent=False     #
+    self.__bValueOfAttr=False #
+    self.__strAttrName=None   #
+    self.__aTaglist = []  #
+
+  def handle_starttag(self, tag, attrs):
+    if tag == "ul" :
+      for attr in attrs :
+        if attr[0] =="class" and attr [1]== "detail_body_right_sec_three" :
+          print(1)
+
+
+if __name__ == '__main__':
+  parser = DMHtmlParser()
+  parser.reset()
+  f = open("result.txt","r",encoding="utf-8")
+  d = f.read()
+  parser.feed(d)

00_Reserch/Spider.py

@@ -0,0 +1,24 @@
+# -*- coding:UTF-8 -*-
+# fiel name spider.py
+import urllib.request
+import os
+import sys
+import codecs
+
+def getHtml(url):
+  req = urllib.request.urlopen('http://www.pythontab.com/')
+  f = open('result.txt','w')
+  data = req.read()
+  req.close()
+  data = data.decode('utf-8')
+  f.write(data)
+  f.close()
+
+if __name__=='__main__':
+  req = urllib.request.urlopen('http://manhua.ali213.net/comic/2121/')
+  f = open('result.txt','w',encoding='utf-8')
+  data = req.read()
+  req.close()
+  data = data.decode('utf-8')
+  f.write(data)
+  f.close()

Common/Const.py

@@ -0,0 +1,7 @@
+'''
+Created on 2014年5月28日
+
+@author: MyGeN
+
+Description:Const Define
+'''

Common/Msg.py

@@ -0,0 +1,93 @@
+#######################################################
+# 
+# Msg.py
+# Python implementation of the Class Msg
+# Generated by Enterprise Architect
+# Created on:      22-05-2014 23:15:10
+# Original author: MyGeN
+# 
+#######################################################
+
+class MSGTYPE(object):
+  CALL = 1
+  RETURN = 2
+
+class Msg(object):
+    def __init__(self, srcModel, dstModel, data = None, msgType = MSGTYPE.CALL, action = None):
+      self.__sequence = 0
+      self.__src = srcModel
+      self.__dst = dstModel
+      self.__data = data
+      self.__action = action
+      self.__type = msgType
+      pass
+
+    def WarpSrcDst(self):
+      self.__src, self.__dst = self.__dst, self.__src
+      pass
+
+    #===========================================================================
+    # 属性
+    #===========================================================================
+    def get_data(self):
+      return self.__data
+
+
+    def get_type(self):
+      return self.__type
+
+
+    def get_src(self):
+      return self.__src
+
+
+    def get_dst(self):
+      return self.__dst
+
+    def get_action(self):
+      return self.__action
+
+
+    def set_data(self, value):
+      self.__data = value
+
+
+    def set_type(self, value):
+      self.__type = value
+
+
+    def set_src(self, value):
+      self.__src = value
+
+
+    def set_dst(self, value):
+      self.__dst = value
+
+    def set_action(self, value):
+      self.__action = value
+
+
+    def del_data(self):
+      del self.__data
+
+
+    def del_type(self):
+      del self.__type
+
+
+    def del_src(self):
+      del self.__src
+
+
+    def del_dst(self):
+      del self.__dst
+
+    def del_action(self):
+      del self.__action
+
+    Data = property(get_data, set_data, del_data, "data's docstring")
+    Type = property(get_type, set_type, del_type, "type's docstring")
+    Src = property(get_src, set_src, del_src, "src's docstring")
+    Dst = property(get_dst, set_dst, del_dst, "dst's docstring")
+    Action = property(get_action, set_action, del_action, "action's docstring")
+

Common/Scheduler.py

@@ -0,0 +1,25 @@
+'''
+Created on 2014年6月6日
+
+@author: MyGeN
+'''
+
+from Common.Msg import *
+
+class Scheduler(object):
+    '''
+              消息调度器类，主要负责消息转发
+    '''
+    def __init__(self):
+      self.Models = dict()
+
+    # Add a model to self
+    def RegisterModel(self, Name, Model):
+      self.Models[Name] = Model
+      pass
+
+    # Send message to target model
+    def PostMsg(self,msg):
+      modelName = msg.get_dst()
+      self.Models[modelName].ReviceMsg(msg)
+      pass

Common/ThreadBase.py

@@ -0,0 +1,69 @@
+# -*- Coding:utf-8 -*-
+#######################################################
+# 
+# ThreadBase.py
+# Python implementation of the Class ThreadBase
+# Generated by Enterprise Architect
+# Created on:      22-����-2014 23:15:10
+# Original author: MyGeN
+# 
+#######################################################
+
+import queue
+from threading import Thread
+
+from Common.Msg import *
+from Common.Scheduler import Scheduler
+
+# TODO: Need to Add Loger
+
+class ThreadBase(Thread):
+	def __init__(self,modelName):
+		Thread.__init__(self)
+		self.MsgQueue = queue.Queue()
+		self.HandleCallback = dict()
+		self.ModelName = modelName
+		self.Schedule = None
+		pass
+
+	def HandleMsg(self, msg):
+		action = 'Default' if msg.Action == None else msg.Action
+		callback = self.HandleCallback[action]
+		needReturn = callback(msg.Data)
+		if needReturn:
+			msg.WarpSrcDst()
+			self.PostMsg(msg)
+		pass
+
+	# Send the message
+	def PostMsg(self,msg):
+		if self.Schedule != None:
+			self.Schedule.PostMsg(msg)
+		else:
+			raise('This Model is not register to any scheduler, you need to call RegisterTo(schduler) to Register a scheduler')
+		pass
+
+	# Register self to a scheduler
+	def RegisterTo(self, scheduler):
+		self.Schedule = scheduler
+		scheduler.RegisterModel(self.ModelName, self)
+		pass
+
+	# Revice message to message queue
+	def ReviceMsg(self, msg):
+		self.MsgQueue.put(msg, block = False)
+		pass
+
+	# overwrite the function
+	def run(self):
+		while True:
+			msg = self.MsgQueue.get(True)
+			if msg != None and msg.Type != MSGTYPE.STOP:
+				self.HandleMsg(msg)
+			elif msg.Type == MSGTYPE.STOP:
+				break
+				pass
+		# 清除消息队列
+		del self.MsgQueue
+		del self.HandleCallback
+		pass

Common/__init__.py

@@ -0,0 +1 @@
+

DataSave/DataManager.py

@@ -0,0 +1,14 @@
+#######################################################
+# 
+# DataManager.py
+# Python implementation of the Class DataManager
+# Generated by Enterprise Architect
+# Created on:      22-����-2014 23:10:40
+# Original author: MyGeN
+# 
+#######################################################
+
+from Common.ThreadBase import ThreadBase
+
+class DataManager(ThreadBase):
+    pass

DataSave/SaveDataBase.py

@@ -0,0 +1,13 @@
+#######################################################
+# 
+# SaveDataBase.py
+# Python implementation of the Class SaveDataBase
+# Generated by Enterprise Architect
+# Created on:      22-ÎåÔÂ-2014 23:10:40
+# Original author: MyGeN
+# 
+#######################################################
+
+
+class SaveDataBase:
+    pass

DataSave/SaveDataToDB.py

@@ -0,0 +1,13 @@
+#######################################################
+# 
+# SaveDataToDB.py
+# Python implementation of the Class SaveDataToDB
+# Generated by Enterprise Architect
+# Created on:      22-ÎåÔÂ-2014 23:10:40
+# Original author: MyGeN
+# 
+#######################################################
+from DataSave.SaveDataBase import SaveDataBase
+
+class SaveDataToDB(SaveDataBase):
+    pass

DataSave/SaveDataToFile.py

@@ -0,0 +1,13 @@
+#######################################################
+# 
+# SaveDataToFile.py
+# Python implementation of the Class SaveDataToFile
+# Generated by Enterprise Architect
+# Created on:      22-ÎåÔÂ-2014 23:10:40
+# Original author: MyGeN
+# 
+#######################################################
+from DataSave.SaveDataBase import SaveDataBase
+
+class SaveDataToFile(SaveDataBase):
+    pass

DataSave/__init__.py

@@ -0,0 +1 @@
+

HtmlAnalyze/Resource.py

@@ -0,0 +1,13 @@
+#######################################################
+# 
+# Resource.py
+# Python implementation of the Class Resource
+# Generated by Enterprise Architect
+# Created on:      22-ÎåÔÂ-2014 23:11:22
+# Original author: MyGeN
+# 
+#######################################################
+
+
+class Resource:
+    pass

HtmlAnalyze/Worker.py

@@ -0,0 +1,26 @@
+#######################################################
+# 
+# Worker.py
+# Python implementation of the Class Worker
+# Generated by Enterprise Architect
+# Created on:      22-����-2014 23:11:22
+# Original author: MyGeN
+# 
+#######################################################
+
+from html.parser import HTMLParser
+
+class Worker(HTMLParser):
+  '''需要读取配置，获取需要爬的xpath集合'''
+  def __init__(self):
+    HTMLParser.__init__(self)
+    pass
+
+  def handle_starttag(self, tag, attrs):
+    pass
+  def handle_endtag(self, tag):
+    pass
+  def handle_data(self, data):
+    pass
+
+  pass

HtmlAnalyze/WorkerManager.py

@@ -0,0 +1,14 @@
+#######################################################
+# 
+# WorkerManager.py
+# Python implementation of the Class WorkerManager
+# Generated by Enterprise Architect
+# Created on:      22-����-2014 23:11:22
+# Original author: MyGeN
+# 
+#######################################################
+
+from Common.ThreadBase import ThreadBase
+
+class WorkerManager(ThreadBase):
+    pass

HtmlAnalyze/__init__.py

@@ -0,0 +1 @@
+