From f917e750e35fc7c8ebe707e28784edfead79b5b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Wed, 24 Apr 2019 00:30:41 +0300 Subject: [PATCH] website can now listen on unix socket --- docs/config.rst | 7 +++++++ docs/news.rst | 2 ++ scrapyd/app.py | 24 ++++++++++++++++++------ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 1f9b2a06..5cad729e 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -26,6 +26,13 @@ bind_address The IP address where the website and json webservices will listen. Defaults to ``127.0.0.1`` (localhost) +unix_socket_path +---------------- + +The filesystem path for a unix socket where the HTTP JSON API will listen. +Example: ``/var/run/scrapyd/http-api.socket`` +File permissions: 0660 + .. _max_proc: max_proc diff --git a/docs/news.rst b/docs/news.rst index 87e8950f..3f77cc87 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -13,6 +13,8 @@ Added - Jobs website shortcut to cancel a job using the cancel.json webservice. - Make project argument to listjobs.json optional, so that we can easily query for all jobs. +- Scrapyd can now listen on a unix socket. + This allows securing the application from users on the same system. Removed ~~~~~~~ diff --git a/scrapyd/app.py b/scrapyd/app.py index 24250a76..2cb0c4a2 100644 --- a/scrapyd/app.py +++ b/scrapyd/app.py @@ -1,5 +1,7 @@ +import os + from twisted.application.service import Application -from twisted.application.internet import TimerService, TCPServer +from twisted.application.internet import TimerService, TCPServer, UNIXServer from twisted.web import server from twisted.python import log @@ -16,6 +18,9 @@ def application(config): app = Application("Scrapyd") http_port = config.getint('http_port', 6800) bind_address = config.get('bind_address', '127.0.0.1') + uds_path = config.get('unix_socket_path', '') + uds_path = uds_path and os.path.abspath(uds_path) + poll_interval = config.getfloat('poll_interval', 5) poller = QueuePoller(config) @@ -33,15 +38,22 @@ def application(config): launcher = laucls(config, app) webpath = config.get('webroot', 'scrapyd.website.Root') - webcls = load_object(webpath) + website = server.Site(load_object(webpath)(config, app)) timer = TimerService(poll_interval, poller.poll) - webservice = TCPServer(http_port, server.Site(webcls(config, app)), interface=bind_address) - log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/", - bind_address=bind_address, http_port=http_port) launcher.setServiceParent(app) timer.setServiceParent(app) - webservice.setServiceParent(app) + + if http_port: + webservice = TCPServer(http_port, website, interface=bind_address) + log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/", + http_port=http_port, bind_address=bind_address) + webservice.setServiceParent(app) + if uds_path: + webservice = UNIXServer(uds_path, website, mode=0o660) + log.msg(format=u"Scrapyd web console available at http+unix://%(uds_path)s", + uds_path=uds_path) + webservice.setServiceParent(app) return app