scrapy · mojtabaasadi · Jan 7, 2019 · Apr 25, 2019 · May 3, 2019 · Dec 9, 2021
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -7,6 +7,7 @@ include scrapyd/default_scrapyd.conf
 recursive-include scrapyd *.py
 recursive-include docs *
 recursive-include bin *
+recursive-include scrapyd *.html
 prune docs/_build
 recursive-include extras *
 recursive-exclude debian *

diff --git a/docs/config.rst b/docs/config.rst
@@ -151,6 +151,8 @@ A twisted web resource that represents the interface to scrapyd.
 Scrapyd includes an interface with a website to provide simple monitoring
 and access to the application's webresources.
 This setting must provide the root class of the twisted web resource.
+you can add or change views in config file, `views` section.
+views are `twisted.web.template` Elements
 
 jobstorage
 -------

diff --git a/scrapyd/default_scrapyd.conf b/scrapyd/default_scrapyd.conf
@@ -31,3 +31,8 @@ delproject.json   = scrapyd.webservice.DeleteProject
 delversion.json   = scrapyd.webservice.DeleteVersion
 listjobs.json     = scrapyd.webservice.ListJobs
 daemonstatus.json = scrapyd.webservice.DaemonStatus
+
+
+[views]
+/        = scrapyd.views.HomeElement
+jobs         = scrapyd.views.JobsElement
diff --git a/scrapyd/templates/footer.html b/scrapyd/templates/footer.html
@@ -0,0 +1 @@
+<p>For more information about the API, see the <a href="http://scrapyd.readthedocs.org/en/latest/">Scrapyd documentation</a></p>
diff --git a/scrapyd/templates/header.html b/scrapyd/templates/header.html
@@ -0,0 +1 @@
+<h1>scrapyd</h1>
diff --git a/scrapyd/templates/home.html b/scrapyd/templates/home.html
@@ -0,0 +1,27 @@
+<html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
+
+<head>
+    <title>Scrapyd</title>
+</head>
+
+<body>
+    <t:transparent t:render="header" />
+    <p>Available projects: <b t:render="projects"></b></p>
+    <ul>
+        <li><a href="/jobs">Jobs</a></li>
+        <t:transparent t:render="local_items" />
+        <li><a href="/logs/">Logs</a></li>
+        <li><a href="http://scrapyd.readthedocs.org/en/latest/">Documentation</a></li>
+    </ul>
+
+    <h2>How to schedule a spider?</h2>
+
+    <p>To schedule a spider you need to use the API (this web UI is only for monitoring)
+    </p>
+
+    <p>Example using <a href="http://curl.haxx.se/">curl</a>:</p>
+    <p><code>curl http://localhost:6800/schedule.json -d project=default -d spider=somespider</code></p>
+    <t:transparent t:render="footer" />
+</body>
+
+</html>
diff --git a/scrapyd/templates/jobs.html b/scrapyd/templates/jobs.html
@@ -0,0 +1,105 @@
+<html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
+
+<head>
+    <title>Scrapyd</title>
+    <style type="text/css">
+        #jobs>thead td {
+            text-align: center;
+            font-weight: bold
+        }
+
+        #jobs>tbody>tr:first-child {
+            background-color: #eee
+        }
+
+        #jobs>*>tr>*:nth-child(9) {
+            display: none
+        }
+
+        #jobs>*>tr>*:nth-child(10) {
+            display: none
+        }
+    </style>
+</head>
+
+<body>
+  <t:transparent t:render="header" />
+    <h2>Jobs</h2>
+    <p><a href="..">Go up</a></p>
+    <table id="jobs" border="1">
+        <thead>
+            <tr>
+                <td>Project</td>
+                <td>Spider</td>
+                <td>Job</td>
+                <td>PID</td>
+                <td>Start</td>
+                <td>Runtime</td>
+                <td>Finish</td>
+                <td>Log</td>
+                <td>Items</td>
+                <td>Cancel</td>
+            </tr>
+        </thead>
+        <tbody>
+            <tr>
+                <th colspan="10">Pending</th>
+            </tr>
+            <tr t:render="pending">
+              <td><t:slot name="project" /></td>
+              <td><t:slot name="spider" /></td>
+              <td><t:slot name="job" /></td>
+              <td><t:slot name="pid" /></td>
+              <td><t:slot name="start" /></td>
+              <td><t:slot name="runtime" /></td>
+              <td><t:slot name="finish" /></td>
+              <td><a target="_blank"><t:attr name="href"><t:slot name="log" /></t:attr>logs</a> </td>
+              <td><a target="_blank"><t:attr name="href"><t:slot name="items" /></t:attr>items</a> </td>
+              <td></td>
+            </tr>
+        </tbody>
+        <tbody>
+            <tr>
+                <th colspan="10">Running</th>
+            </tr>
+            <tr t:render="running">
+              <td><t:slot name="project" /></td>
+              <td><t:slot name="spider" /></td>
+              <td><t:slot name="job" /></td>
+              <td><t:slot name="pid" /></td>
+              <td><t:slot name="start" /></td>
+              <td><t:slot name="runtime" /></td>
+              <td><t:slot name="finish" /></td>
+              <td><a target="_blank"><t:attr name="href"><t:slot name="log" /></t:attr>logs</a> </td>
+              <td><a target="_blank"><t:attr name="href"><t:slot name="items" /></t:attr>items</a> </td>
+              <td>
+              <form method="post" action="/cancel.json" target="_blank">
+                <input type="hidden" name="project" ><t:attr name="value"><t:slot name="project" /></t:attr></input>
+                <input type="hidden" name="job" ><t:attr name="value"><t:slot name="job" /></t:attr></input>
+                <input type="submit" style="float: left;" value="Cancel"/>
+              </form>
+              </td>
+            </tr>
+        </tbody>
+        <tbody>
+            <tr>
+                <th colspan="10">Finished</th>
+            </tr>
+            <tr t:render="finished">
+              <td><t:slot name="project" /></td>
+              <td><t:slot name="spider" /></td>
+              <td><t:slot name="job" /></td>
+              <td><t:slot name="pid" /></td>
+              <td><t:slot name="start" /></td>
+              <td><t:slot name="runtime" /></td>
+              <td><t:slot name="finish" /></td>
+              <td><a target="_blank"><t:attr name="href"><t:slot name="log" /></t:attr>logs</a> </td>
+              <td><a target="_blank"><t:attr name="href"><t:slot name="items" /></t:attr>items</a> </td>
+              <td></td>
+            </tr>
+        </tbody>
+    </table>
+    <t:transparent t:render="footer" />
+</body>
+
+</html>
diff --git a/scrapyd/tests/test_website.py b/scrapyd/tests/test_website.py
@@ -25,14 +25,17 @@ def scrapyd_site():
 
 class TestWebsite:
     def test_render_jobs(self, txrequest, scrapyd_site):
-        content = scrapyd_site.children[b'jobs'].render(txrequest)
+        content = scrapyd_site.children[b'jobs'].render_GET(txrequest)
         expect_headers = {
             b'Content-Type': [b'text/html; charset=utf-8'],
-            b'Content-Length': [b'643']
+            b'Content-Length': [b'1548']
         }
         headers = txrequest.responseHeaders.getAllRawHeaders()
         assert dict(headers) == expect_headers
-        initial = '<html><head><title>Scrapyd</title><style type="text/css">#jobs>thead td {text-align: center; font-weight'
+        initial = '''<html>
+
+<head>
+    <title>Scrapyd</title>'''
         assert content.decode().startswith(initial)
 
     def test_render_home(self, txrequest, scrapyd_site):

diff --git a/scrapyd/views.py b/scrapyd/views.py
@@ -0,0 +1,109 @@
+from twisted.web.template import Element, renderer, XMLFile, tags,flattenString
+from twisted.python.filepath import FilePath
+from os import path
+from datetime import datetime,timedelta
+from zope.interface import implementer
+from scrapy.utils.misc import load_object
+from six.moves.urllib.parse import urlparse
+
+
+def microsec_trunc(timelike):
+    if hasattr(timelike, 'microsecond'):
+        ms = timelike.microsecond
+    else:
+        ms = timelike.microseconds
+    return timelike - timedelta(microseconds=ms)
+
+
+def load_template(file):
+    scrapyd_views = path.join(path.dirname(path.abspath(__file__)),"templates")
+    _path = path.join(scrapyd_views,file)
+    return XMLFile(FilePath(_path))
+
+class BaseElement(Element):
+    @property
+    def loader(self):
+        return load_template(self._template_file_)
+    @renderer
+    def footer(self,request,tag):
+        return load_template("footer.html").load()
+
+    @renderer
+    def header(self,request,tag):
+        return load_template("header.html").load()
+
+
+class HomeElement(BaseElement):
+    _template_file_  = "home.html"
+
+    @renderer
+    def projects(self,request,tag):
+        return tag(", ".join(self._root.scheduler.list_projects()))
+
+    @renderer
+    def local_items(self,request,tag):
+        itemsdir = self._root.config.get('items_dir')
+        local_items = itemsdir and (urlparse(itemsdir).scheme.lower() in ['', 'file'])
+        if local_items:
+            return tags.li(tags.a("Items",href="/items"))
+
+        return ""
+
+class JobsElement(BaseElement):
+    _template_file_  = "jobs.html"
+    def createJob(self,**job):
+        return dict(
+            project = str(job['project']) if 'project' in job else '',
+            spider = str(job['spider']) if 'spider' in job else '',
+            job = str(job['job']) if 'job' in job else '',
+            pid = str(job['pid']) if 'pid' in job else '',
+            start = str(microsec_trunc(job['start'])) if 'start' in job else '',
+            runtime = str(microsec_trunc(job['runtime'])) if 'runtime' in job else '',
+            finish = str(microsec_trunc(job['finish'])) if 'finish' in job else '',
+            log = str(job['log']) if 'log' in job else '',
+            items = str(job['items']) if 'items' in job else '',
+        )
+    @renderer
+    def pending(self,request,tag):
+        data = [self.createJob(
+                project=project or "", spider=m['name'], job=m['_job'],
+            )
+            for project, queue in self._root.poller.queues.items()
+            for m in queue.list()
+        ]
+        for job in data:
+            yield tag.clone().fillSlots(**job)
+        return ""
+
+    @renderer
+    def finished(self,request,tag):
+        data= [self.createJob(
+                project=p.project, spider=p.spider,job=p.job,
+                start=p.start_time,
+                runtime=p.end_time - p.start_time,
+                finish=p.end_time,
+                log='/logs/%s/%s/%s.log' % (p.project, p.spider, p.job),
+                items='/items/%s/%s/%s.jl' % (p.project, p.spider, p.job),
+            )
+            for p in self._root.launcher.finished
+        ]
+        for job in data:
+            yield tag.clone().fillSlots(**job)
+        return ""
+
+    @renderer
+    def running(self,request,tag):
+        data = [
+            self.createJob(
+                project=p.project, spider=p.spider,
+                job=p.job, pid=p.pid,
+                start=p.start_time,
+                runtime=datetime.now() - p.start_time,
+                log='/logs/%s/%s/%s.log' % (p.project, p.spider, p.job),
+                items='/items/%s/%s/%s.jl' % (p.project, p.spider, p.job),
+            )
+            for p in self._root.launcher.processes.values()
+        ]
+        for job in data:
+            yield tag.clone().fillSlots(**job)
+        return ""
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		<p>For more information about the API, see the <a href="http://scrapyd.readthedocs.org/en/latest/">Scrapyd documentation</a></p>