Skip to content

Commit

Permalink
chore: Remove unused keyword arguments from native_stringify_dict and…
Browse files Browse the repository at this point in the history
… to_native_str
  • Loading branch information
jpmckinney committed Jul 20, 2024
1 parent f144395 commit 792f347
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 17 deletions.
8 changes: 4 additions & 4 deletions scrapyd/launcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ def get_crawl_args(message):
args = [to_native_str(msg["_spider"])]
del msg["_project"], msg["_spider"]
settings = msg.pop("settings", {})
for k, v in native_stringify_dict(msg, keys_only=False).items():
for k, v in native_stringify_dict(msg).items():
args += ["-a"]
args += [f"{k}={v}"]
for k, v in native_stringify_dict(settings, keys_only=False).items():
for k, v in native_stringify_dict(settings).items():
args += ["-s"]
args += [f"{k}={v}"]
return args
Expand Down Expand Up @@ -57,12 +57,12 @@ def _spawn_process(self, message, slot):
environ = self.app.getComponent(IEnvironment)
message.setdefault("settings", {})
message["settings"].update(environ.get_settings(message))
msg = native_stringify_dict(message, keys_only=False)
msg = native_stringify_dict(message)
project = msg["_project"]
args = [sys.executable, "-m", self.runner, "crawl"]
args += get_crawl_args(msg)
env = environ.get_environment(msg, slot)
env = native_stringify_dict(env, keys_only=False)
env = native_stringify_dict(env)
pp = ScrapyProcessProtocol(project, msg["_spider"], msg["_job"], env, args)
pp.deferred.addBoth(self._process_finished, slot)
reactor.spawnProcess(pp, sys.executable, args=args, env=env)
Expand Down
22 changes: 10 additions & 12 deletions scrapyd/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,27 +47,25 @@ def get_project_list(config):
return projects


def native_stringify_dict(dct_or_tuples, encoding="utf-8", *, keys_only=True):
"""Return a (new) dict with unicode keys (and values when "keys_only" is
False) of the given dict converted to strings. `dct_or_tuples` can be a
def native_stringify_dict(dct_or_tuples):
"""Return a (new) dict with unicode keys and values
of the given dict converted to strings. `dct_or_tuples` can be a
dict or a list of tuples, like any dict constructor supports.
"""
d = {}
for k, v in dct_or_tuples.items():
key = to_native_str(k, encoding)
if keys_only:
value = v
elif isinstance(v, dict):
value = native_stringify_dict(v, encoding=encoding, keys_only=keys_only)
key = to_native_str(k)
if isinstance(v, dict):
value = native_stringify_dict(v)
elif isinstance(v, list):
value = [to_native_str(e, encoding) for e in v]
value = [to_native_str(e) for e in v]
else:
value = to_native_str(v, encoding)
value = to_native_str(v)
d[key] = value
return d


def to_native_str(text, encoding="utf-8", errors="strict"):
def to_native_str(text):
if isinstance(text, str):
return text
return text.decode(encoding, errors)
return text.decode()
2 changes: 1 addition & 1 deletion scrapyd/webservice.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ def render_POST(self, txrequest, project, spider, version, jobid, priority, sett
if spider not in spiders:
raise error.Error(code=http.OK, message=b"spider '%b' not found" % spider.encode())

spider_arguments = {k: v[0] for k, v in native_stringify_dict(copy(txrequest.args), keys_only=False).items()}
spider_arguments = {k: v[0] for k, v in native_stringify_dict(copy(txrequest.args)).items()}

self.root.scheduler.schedule(
project,
Expand Down

0 comments on commit 792f347

Please sign in to comment.