From fa89944891fda3a81f45609a63a1009908ef1e6c Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 17 Nov 2021 00:34:29 +0100 Subject: [PATCH] Add initial Sphinx docs --- .flake8 | 20 +- .github/workflows/test-library.yml | 6 + .pre-commit-config.yaml | 2 +- .readthedocs.yml | 41 +++ README.md | 91 +++--- docs/_ext/spelling_stub_ext.py | 27 ++ docs/conf.py | 275 ++++++++++++++++++ docs/glossary.md | 25 ++ docs/index.md | 39 +++ docs/pkg/.gitignore | 2 + docs/requirements.in | 5 + docs/requirements.txt | 272 +++++++++++++++++ docs/spelling_wordlist.txt | 7 + proxy/__init__.py | 6 + proxy/common/__init__.py | 4 + proxy/common/_compat.py | 8 +- proxy/common/flag.py | 14 +- proxy/common/pki.py | 4 + proxy/common/utils.py | 9 +- proxy/core/__init__.py | 4 + proxy/core/acceptor/__init__.py | 7 + proxy/core/acceptor/acceptor.py | 35 ++- proxy/core/acceptor/executors.py | 12 +- proxy/core/acceptor/listener.py | 4 + proxy/core/acceptor/pool.py | 21 +- proxy/core/acceptor/threadless.py | 22 +- proxy/core/acceptor/work.py | 4 + proxy/core/base/__init__.py | 4 + proxy/core/base/tcp_server.py | 14 +- proxy/core/base/tcp_tunnel.py | 6 +- proxy/core/connection/__init__.py | 5 + proxy/core/connection/pool.py | 4 + proxy/core/connection/types.py | 4 + proxy/core/event/__init__.py | 6 + proxy/core/event/manager.py | 4 + proxy/core/event/names.py | 5 + proxy/core/event/queue.py | 28 +- proxy/core/event/subscriber.py | 2 +- proxy/core/ssh/__init__.py | 4 + proxy/core/ssh/client.py | 2 +- proxy/core/ssh/tunnel.py | 2 +- proxy/dashboard/__init__.py | 6 + proxy/dashboard/inspect_traffic.py | 5 + proxy/http/__init__.py | 6 + proxy/http/codes.py | 5 + proxy/http/exception/__init__.py | 5 + proxy/http/exception/base.py | 13 +- proxy/http/exception/http_request_rejected.py | 4 + proxy/http/exception/proxy_auth_failed.py | 7 +- proxy/http/exception/proxy_conn_failed.py | 7 +- proxy/http/handler.py | 8 +- proxy/http/inspector/__init__.py | 5 + proxy/http/inspector/devtools.py | 5 + proxy/http/inspector/transformer.py | 6 +- proxy/http/methods.py | 5 + proxy/http/parser/__init__.py | 5 + proxy/http/parser/chunk.py | 5 + proxy/http/parser/parser.py | 18 +- proxy/http/parser/protocol.py | 4 + proxy/http/parser/types.py | 5 + proxy/http/plugin.py | 11 +- proxy/http/proxy/__init__.py | 5 + proxy/http/proxy/auth.py | 5 + proxy/http/proxy/plugin.py | 16 +- proxy/http/proxy/server.py | 5 + proxy/http/server/__init__.py | 5 + proxy/http/server/pac_plugin.py | 5 + proxy/http/server/plugin.py | 9 +- proxy/http/server/protocols.py | 5 + proxy/http/server/web.py | 4 + proxy/http/url.py | 23 +- proxy/http/websocket/__init__.py | 7 + proxy/http/websocket/client.py | 5 + proxy/http/websocket/frame.py | 7 + proxy/plugin/__init__.py | 5 + proxy/plugin/cloudflare_dns.py | 22 +- proxy/plugin/custom_dns_resolver.py | 13 +- proxy/plugin/filter_by_client_ip.py | 4 + proxy/plugin/filter_by_url_regex.py | 4 + proxy/plugin/mock_rest_api.py | 6 +- proxy/plugin/reverse_proxy.py | 9 +- proxy/plugin/shortlink.py | 18 +- proxy/proxy.py | 7 +- proxy/testing/test_case.py | 2 +- tests/test_main.py | 4 + tox.ini | 134 +++++++++ 86 files changed, 1361 insertions(+), 159 deletions(-) create mode 100644 .readthedocs.yml create mode 100644 docs/_ext/spelling_stub_ext.py create mode 100644 docs/conf.py create mode 100644 docs/glossary.md create mode 100644 docs/index.md create mode 100644 docs/pkg/.gitignore create mode 100644 docs/requirements.in create mode 100644 docs/requirements.txt create mode 100644 docs/spelling_wordlist.txt diff --git a/.flake8 b/.flake8 index 575ba2cc28..1bffdb4268 100644 --- a/.flake8 +++ b/.flake8 @@ -230,9 +230,19 @@ pytest-parametrize-values-row-type = tuple pytest-mark-no-parentheses = true # flake8-rst-docstrings +rst-directives = + spelling rst-roles = - # Built-in Sphinx roles: - py:class, - py:meth, - # Sphinx's internal role: - event, + # Built-in Sphinx roles: + class, + data, + exc, + meth, + term, + py:class, + py:data, + py:exc, + py:meth, + py:term, + # Sphinx's internal role: + event, diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index f66840000d..6909a0dd88 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -305,6 +305,7 @@ jobs: --parallel auto --parallel-live --skip-missing-interpreters false + --skip-pkg-install - name: Verify that the artifacts with expected names got created run: >- ls -1 @@ -335,6 +336,10 @@ jobs: toxenv: - lint - metadata-validation + - build-docs + - doctest-docs + - linkcheck-docs + - spellcheck-docs fail-fast: false env: @@ -417,6 +422,7 @@ jobs: --parallel auto --parallel-live --skip-missing-interpreters false + --skip-pkg-install test: name: 🐍${{ matrix.python }} @ ${{ matrix.os }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc00d93e1e..db86982576 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,7 +74,7 @@ repos: $ - id: requirements-txt-fixer exclude: >- - ^requirements(|-(release|testing|tunnel))\.txt$ + ^(docs/requirements|requirements(|-(release|testing|tunnel)))\.txt$ # Non-modifying checks: - id: name-tests-test args: diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000000..6ac10f57af --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,41 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html +# for details + +--- + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + builder: dirhtml + configuration: docs/conf.py + fail_on_warning: true + +# Optionally build your docs in additional formats +# such as PDF and ePub +formats: [] + +submodules: + include: all # [] + exclude: [] + recursive: true + +build: + os: ubuntu-20.04 + tools: + python: >- # PyYAML parses it as float `3.1` it it's not an explicit string + 3.10 + +# Optionally set the version of Python and requirements required +# to build docs +python: + install: + - method: pip + path: . + - requirements: requirements-tunnel.txt + - requirements: docs/requirements.txt + system_packages: false + +... diff --git a/README.md b/README.md index a3e5f0fed8..94695dd7fc 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ ⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • 😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • 👮🏿 "Proxy Server" framework • 🌐 "Web Server" framework • ➵ ➶ ➷ ➠ "PubSub" framework • 👷 "Work" acceptor & executor framework

+[//]: # (DO-NOT-REMOVE-docs-badges-START) + [![PyPi Monthly](https://img.shields.io/pypi/dm/proxy.py?style=for-the-badge&color=darkgreen)](https://pypi.org/project/proxy.py/) [![Docker Pulls](https://img.shields.io/docker/pulls/abhinavsingh/proxy.py?style=for-the-badge&color=darkgreen)](https://hub.docker.com/r/abhinavsingh/proxy.py) [![No Dependencies](https://img.shields.io/static/v1?label=dependencies&message=0&style=for-the-badge&color=darkgreen)](https://github.com/abhinavsingh/proxy.py) @@ -82,9 +84,9 @@ - [Ephemeral Port](#ephemeral-port) - [Loading Plugins](#loading-plugins) - [Unit testing with proxy.py](#unit-testing-with-proxypy) - - [proxy.TestCase](#proxytestcase) + - [`proxy.TestCase`](#proxytestcase) - [Override Startup Flags](#override-startup-flags) - - [With unittest.TestCase](#with-unittesttestcase) + - [With `unittest.TestCase`](#with-unittesttestcase) - [Utilities](#utilities) - [TCP](#tcp-sockets) - [new_socket_connection](#new_socket_connection) @@ -105,7 +107,7 @@ - [Unable to connect with proxy.py from remote host](#unable-to-connect-with-proxypy-from-remote-host) - [Basic auth not working with a browser](#basic-auth-not-working-with-a-browser) - [Docker image not working on MacOS](#docker-image-not-working-on-macos) - - [ValueError: filedescriptor out of range in select](#valueerror-filedescriptor-out-of-range-in-select) + - [`ValueError: filedescriptor out of range in select`](#valueerror-filedescriptor-out-of-range-in-select) - [None:None in access logs](#nonenone-in-access-logs) - [OSError when wrapping client for TLS Interception](#oserror-when-wrapping-client-for-tls-interception) - [Plugin Developer and Contributor Guide](#plugin-developer-and-contributor-guide) @@ -122,6 +124,8 @@ - [v1.x](#v1x) - [v0.x](#v0x) +[//]: # (DO-NOT-REMOVE-docs-badges-END) + # Features - Fast & Scalable @@ -170,7 +174,7 @@ - Optionally, enable builtin [Web Server Plugins](#http-web-server-plugins). Example: - `--plugins proxy.plugin.ReverseProxyPlugin` - Plugin API is currently in development phase, expect breaking changes -- Realtime Dashboard +- Real-time Dashboard - Optionally, enable [proxy.py dashboard](#run-dashboard). - Available at `http://localhost:8899/dashboard`. - [Inspect, Monitor, Control and Configure](#inspect-traffic) `proxy.py` at runtime @@ -423,7 +427,7 @@ You can override flag from command line when starting the docker container. For Add support for short links in your favorite browsers / applications. -[![Shortlink Plugin](https://raw.githubusercontent.com/abhinavsingh/proxy.py/develop/shortlink.gif)](https://github.com/abhinavsingh/proxy.py#shortlinkplugin) +[![Shortlink Plugin](https://raw.githubusercontent.com/abhinavsingh/proxy.py/develop/shortlink.gif)](https://github.com/abhinavsingh/proxy.py#user-content-shortlinkplugin) Start `proxy.py` as: @@ -438,17 +442,17 @@ across all browsers. Following short links are enabled by default: -| Short Link | Destination URL | -| :--------: | :--------------: | -| a/ | amazon.com | -| i/ | instagram.com | -| l/ | linkedin.com | -| f/ | facebook.com | -| g/ | google.com | -| t/ | twitter.com | -| w/ | web.whatsapp.com | -| y/ | youtube.com | -| proxy/ | localhost:8899 | +| Short Link | Destination URL | +| :--------: | :--------------: | +| a/ | `amazon.com` | +| i/ | `instagram.com` | +| l/ | `linkedin.com` | +| f/ | `facebook.com` | +| g/ | `google.com` | +| t/ | `twitter.com` | +| w/ | `web.whatsapp.com` | +| y/ | `youtube.com` | +| proxy/ | `localhost:8899` | ### ModifyPostDataPlugin @@ -461,7 +465,7 @@ Start `proxy.py` as: --plugins proxy.plugin.ModifyPostDataPlugin ``` -By default plugin replaces POST body content with hardcoded `b'{"key": "modified"}'` +By default plugin replaces POST body content with hard-coded `b'{"key": "modified"}'` and enforced `Content-Type: application/json`. Verify the same using `curl -x localhost:8899 -d '{"key": "value"}' http://httpbin.org/post` @@ -763,7 +767,7 @@ Modify plugin to your taste e.g. Allow specific IP addresses only. ### ModifyChunkResponsePlugin -This plugin demonstrate how to modify chunked encoded responses. In able to do so, this plugin uses `proxy.py` core to parse the chunked encoded response. Then we reconstruct the response using custom hardcoded chunks, ignoring original chunks received from upstream server. +This plugin demonstrate how to modify chunked encoded responses. In able to do so, this plugin uses `proxy.py` core to parse the chunked encoded response. Then we reconstruct the response using custom hard-coded chunks, ignoring original chunks received from upstream server. Start `proxy.py` as: @@ -784,14 +788,14 @@ plugin * Closing connection 0 ``` -Modify `ModifyChunkResponsePlugin` to your taste. Example, instead of sending hardcoded chunks, parse and modify the original `JSON` chunks received from the upstream server. +Modify `ModifyChunkResponsePlugin` to your taste. Example, instead of sending hard-coded chunks, parse and modify the original `JSON` chunks received from the upstream server. ### CloudflareDnsResolverPlugin This plugin uses `Cloudflare` hosted `DNS-over-HTTPS` [API](https://developers.cloudflare.com/1.1.1.1/encrypted-dns/dns-over-https/make-api-requests/dns-json) (json). `DoH` mandates a HTTP2 compliant client. Unfortunately `proxy.py` -doesn't provide that yet, so we use a dependency. Install it: +does not provide that yet, so we use a dependency. Install it: ```console ❯ pip install "httpx[http2]" @@ -811,7 +815,7 @@ Use `--cloudflare-dns-mode family` to also enable adult content protection too. This plugin demonstrate how to use a custom DNS resolution implementation with `proxy.py`. This example plugin currently uses Python's in-built resolution mechanism. Customize code -to your taste. Example, query your custom DNS server, implement DoH or other mechanisms. +to your taste. Example, query your custom DNS server, implement `DoH` or other mechanisms. Start `proxy.py` as: @@ -969,7 +973,7 @@ response from the server. Start `proxy.py` as: --ca-signing-key-file ca-signing-key.pem ``` -[![NOTE](https://img.shields.io/static/v1?label=MacOS&message=note&color=yellow)](https://github.com/abhinavsingh/proxy.py#flags) Also provide explicit CA bundle path needed for validation of peer certificates. See `--ca-file` flag. +[![NOTE](https://img.shields.io/static/v1?label=MacOS&message=note&color=yellow)](https://github.com/abhinavsingh/proxy.py#user-content-flags) Also provide explicit CA bundle path needed for validation of peer certificates. See `--ca-file` flag. Verify TLS interception using `curl` @@ -1042,7 +1046,8 @@ Important notes about TLS Interception with Docker container: - Since `v2.2.0`, `proxy.py` docker container also ships with `openssl`. This allows `proxy.py` to generate certificates on the fly for TLS Interception. -- For security reasons, `proxy.py` docker container doesn't ship with CA certificates. +- For security reasons, `proxy.py` docker container does not ship with + CA certificates. Here is how to start a `proxy.py` docker container with TLS Interception: @@ -1329,9 +1334,9 @@ if __name__ == '__main__': # Unit testing with proxy.py -## proxy.TestCase +## `proxy.TestCase` -To setup and teardown `proxy.py` for your Python `unittest` classes, +To setup and tear down `proxy.py` for your Python `unittest` classes, simply use `proxy.TestCase` instead of `unittest.TestCase`. Example: @@ -1346,10 +1351,10 @@ class TestProxyPyEmbedded(proxy.TestCase): Note that: -1. `proxy.TestCase` overrides `unittest.TestCase.run()` method to setup and teardown `proxy.py`. +1. `proxy.TestCase` overrides `unittest.TestCase.run()` method to setup and tear down `proxy.py`. 2. `proxy.py` server will listen on a random available port on the system. This random port is available as `self.PROXY.acceptors.flags.port` within your test cases. -3. Only a single acceptor and worker is started by default (`--num-workers 1 --num-acceptors 1`) for faster setup and teardown. +3. Only a single acceptor and worker is started by default (`--num-workers 1 --num-acceptors 1`) for faster setup and tear down. 4. Most importantly, `proxy.TestCase` also ensures `proxy.py` server is up and running before proceeding with execution of tests. By default, `proxy.TestCase` will wait for `10 seconds` for `proxy.py` server to start, @@ -1373,13 +1378,15 @@ class TestProxyPyEmbedded(TestCase): self.assertTrue(True) ``` -See [test_embed.py](https://github.com/abhinavsingh/proxy.py/blob/develop/tests/test_embed.py) -for full working example. +See [test_embed.py] for full working example. + +[test_embed.py]: +https://github.com/abhinavsingh/proxy.py/blob/develop/tests/testing/test_embed.py -## With unittest.TestCase +## With `unittest.TestCase` If for some reasons you are unable to directly use `proxy.TestCase`, -then simply override `unittest.TestCase.run` yourself to setup and teardown `proxy.py`. +then simply override `unittest.TestCase.run` yourself to setup and tear down `proxy.py`. Example: ```python @@ -1400,7 +1407,7 @@ class TestProxyPyEmbedded(unittest.TestCase): super().run(result) ``` -or simply setup / teardown `proxy.py` within +or simply setup / tear down `proxy.py` within `setUpClass` and `teardownClass` class methods. # Utilities @@ -1438,7 +1445,7 @@ As a decorator: >>> ... [ use connection ] ... ``` -## Http Client +## HTTP Client ### build_http_request @@ -1482,7 +1489,7 @@ build_http_response( ### API Usage -- gen_private_key +- `gen_private_key` ```python gen_private_key( @@ -1492,7 +1499,7 @@ build_http_response( timeout: int = 10) -> bool ``` -- gen_public_key +- `gen_public_key` ```python gen_public_key( @@ -1506,7 +1513,7 @@ build_http_response( timeout: int = 10) -> bool ``` -- remove_passphrase +- `remove_passphrase` ```python remove_passphrase( @@ -1516,7 +1523,7 @@ build_http_response( timeout: int = 10) -> bool ``` -- gen_csr +- `gen_csr` ```python gen_csr( @@ -1527,7 +1534,7 @@ build_http_response( timeout: int = 10) -> bool ``` -- sign_csr +- `sign_csr` ```python sign_csr( @@ -1640,7 +1647,7 @@ Visit dashboard: Wait for embedded `Chrome Dev Console` to load. Currently, detail about all traffic flowing through `proxy.py` is pushed to the `Inspect Traffic` tab. However, received payloads are not -yet integrated with the embedded dev console. +yet integrated with the embedded developer console. Current functionality can be verified by opening the `Dev Console` of dashboard and inspecting the websocket connection that dashboard established with the `proxy.py` server. @@ -1782,7 +1789,7 @@ for some background. ## GCE log viewer integration for proxy.py -A starter [fluentd.conf](https://github.com/abhinavsingh/proxy.py/blob/develop/fluentd.conf) +A starter [fluentd.conf](https://github.com/abhinavsingh/proxy.py/blob/develop/helper/fluentd.conf) template is available. 1. Copy this configuration file as `proxy.py.conf` under @@ -1798,7 +1805,7 @@ template is available. Now `proxy.py` logs can be browsed using [GCE log viewer](https://console.cloud.google.com/logs/viewer). -## ValueError: filedescriptor out of range in select +## `ValueError: filedescriptor out of range in select` `proxy.py` is made to handle thousands of connections per second without any socket leaks. @@ -1915,7 +1922,7 @@ the incoming client connections. Each `Acceptor` process delegates the accepted client connection to a threadless process via `Work` class. Currently, `HttpProtocolHandler` -is the default work klass. +is the default work class. `HttpProtocolHandler` simply assumes that incoming clients will follow HTTP specification. Specific HTTP proxy and HTTP server implementations diff --git a/docs/_ext/spelling_stub_ext.py b/docs/_ext/spelling_stub_ext.py new file mode 100644 index 0000000000..c8989dc149 --- /dev/null +++ b/docs/_ext/spelling_stub_ext.py @@ -0,0 +1,27 @@ +"""Sphinx extension for making the spelling directive noop.""" + +from typing import List + +from sphinx.application import Sphinx +from sphinx.util.docutils import SphinxDirective +from sphinx.util.nodes import nodes + + +class SpellingNoOpDirective(SphinxDirective): + """Definition of the stub spelling directive.""" + + has_content = True + + def run(self) -> List[nodes.Node]: + """Generate nothing in place of the directive.""" + return [] + + +def setup(app: Sphinx) -> None: + """Initialize the extension.""" + app.add_directive('spelling', SpellingNoOpDirective) + + return { + 'parallel_read_safe': True, + 'version': 'builtin', + } diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000000..1be4d0cbe5 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,275 @@ +# pylint: disable=invalid-name +# Requires Python 3.6+ +# Ref: https://www.sphinx-doc.org/en/master/usage/configuration.html +"""Configuration for the Sphinx documentation generator.""" + +import sys +from functools import partial +from pathlib import Path + +from setuptools_scm import get_version + + +# -- Path setup -------------------------------------------------------------- + +PROJECT_ROOT_DIR = Path(__file__).parents[1].resolve() # pylint: disable=no-member +get_scm_version = partial(get_version, root=PROJECT_ROOT_DIR) + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + + +sys.path.insert(0, str(PROJECT_ROOT_DIR)) + +# Make in-tree extension importable in non-tox setups/envs, like RTD. +# Refs: +# https://github.com/readthedocs/readthedocs.org/issues/6311 +# https://github.com/readthedocs/readthedocs.org/issues/7182 +sys.path.insert(0, str((Path(__file__).parent / '_ext').resolve())) + +# -- Project information ----------------------------------------------------- + +github_url = 'https://github.com' +github_repo_org = 'abhinavsingh' +github_repo_name = 'proxy.py' +github_repo_slug = f'{github_repo_org}/{github_repo_name}' +github_repo_url = f'{github_url}/{github_repo_slug}' +github_sponsors_url = f'{github_url}/sponsors' + +project = github_repo_name.title() +author = f'{project} project contributors' +copyright = author # pylint: disable=redefined-builtin + +# The short X.Y version +version = '.'.join( + get_scm_version( + local_scheme='no-local-version', + ).split('.')[:3], +) + +# The full version, including alpha/beta/rc tags +release = get_scm_version() + +rst_epilog = f""" +.. |project| replace:: {project} +""" + + +# -- General configuration --------------------------------------------------- + + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +today_fmt = '%B %d, %Y' + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# Ref: python-attrs/attrs#571 +default_role = 'any' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = True + +# The name of the Pygments (syntax highlighting) style to use. +# pygments_style = 'sphinx' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + # stdlib-party extensions: + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.extlinks', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', + + # Third-party extensions: + 'myst_parser', # extended markdown; https://pypi.org/project/myst-parser/ + 'sphinxcontrib.apidoc', +] + +# Conditional third-party extensions: +try: + import sphinxcontrib.spelling as _sphinxcontrib_spelling +except ImportError: + extensions.append('spelling_stub_ext') +else: + del _sphinxcontrib_spelling + extensions.append('sphinxcontrib.spelling') + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'furo' + +html_show_sphinx = True + +html_theme_options = { +} + +html_context = { +} + + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = f'{project} Documentation' + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = 'Documentation' + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +html_last_updated_fmt = '%b %d, %Y' + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = f'https://{github_repo_name.replace(".", "")}.readthedocs.io/en/latest/' + +# The master toctree document. +root_doc = master_doc = 'index' # Sphinx 4+ / 3- # noqa: WPS429 + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), + 'python2': ('https://docs.python.org/2', None), +} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +# -- Options for sphinxcontrib.apidoc extension ------------------------------ + +apidoc_excluded_paths = [ + 'plugin/cache/*', + 'testing/*.py', +] +apidoc_extra_args = [ + '--implicit-namespaces', + '--private', # include “_private” modules +] +apidoc_module_dir = str(PROJECT_ROOT_DIR / 'proxy') +apidoc_module_first = False +apidoc_output_dir = 'pkg' +apidoc_separate_modules = True +apidoc_toc_file = None + +# -- Options for sphinxcontrib.spelling extension ---------------------------- + +spelling_ignore_acronyms = True +spelling_ignore_importable_modules = True +spelling_ignore_pypi_package_names = True +spelling_ignore_python_builtins = True +spelling_ignore_wiki_words = True +spelling_show_suggestions = True +spelling_word_list_filename = [ + 'spelling_wordlist.txt', +] + +# -- Options for extlinks extension ------------------------------------------ + +extlinks = { + 'issue': (f'{github_repo_url}/issues/%s', '#'), # noqa: WPS323 + 'pr': (f'{github_repo_url}/pull/%s', 'PR #'), # noqa: WPS323 + 'commit': (f'{github_repo_url}/commit/%s', ''), # noqa: WPS323 + 'gh': (f'{github_url}/%s', 'GitHub: '), # noqa: WPS323 + 'user': (f'{github_sponsors_url}/%s', '@'), # noqa: WPS323 +} + +# -- Options for linkcheck builder ------------------------------------------- + +linkcheck_ignore = [ + r'http://localhost:\d+/', # local URLs +] +linkcheck_workers = 25 + +# -- Options for myst_parser extension ------------------------------------------ + +myst_enable_extensions = [ + 'colon_fence', # allow to optionally use ::: instead of ``` + 'deflist', + 'html_admonition', # allow having HTML admonitions + 'html_image', # allow HTML in Markdown + # FIXME: `linkify` turns "Proxy.Py` into a link so it's disabled now + # Ref: https://github.com/executablebooks/MyST-Parser/issues/428#issuecomment-970277208 + # "linkify", # auto-detect URLs @ plain text, needs myst-parser[linkify] + 'replacements', # allows Jinja2-style replacements + 'smartquotes', # use "cursive" quotes + 'substitution', # replace common ASCII shortcuts into their symbols +] +myst_substitutions = { + 'project': project, +} + +# -- Strict mode ------------------------------------------------------------- + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# Ref: python-attrs/attrs#571 +default_role = 'any' + +nitpicky = True +_any_role = 'any' +_py_class_role = 'py:class' +nitpick_ignore = [ + (_any_role, ''), + (_any_role, '__init__'), + (_any_role, '--threadless'), + (_any_role, 'Client'), + (_any_role, 'event_queue'), + (_any_role, 'fd_queue'), + (_any_role, 'flag.flags'), + (_any_role, 'flags.work_klass'), + (_any_role, 'flush'), + (_any_role, 'httpx'), + (_any_role, 'HttpParser.state'), + (_any_role, 'HttpProtocolHandler'), + (_any_role, 'multiprocessing.Manager'), + (_any_role, 'work_klass'), + (_py_class_role, 'CacheStore'), + (_py_class_role, 'HttpParser'), + (_py_class_role, 'HttpProtocolHandlerPlugin'), + (_py_class_role, 'HttpProxyBasePlugin'), + (_py_class_role, 'HttpWebServerBasePlugin'), + (_py_class_role, 'multiprocessing.context.Process'), + (_py_class_role, 'multiprocessing.synchronize.Lock'), + (_py_class_role, 'paramiko.channel.Channel'), + (_py_class_role, 'proxy.http.parser.parser.T'), + (_py_class_role, 'proxy.plugin.cache.store.base.CacheStore'), + (_py_class_role, 'TcpClientConnection'), + (_py_class_role, 'TcpServerConnection'), + (_py_class_role, 'unittest.case.TestCase'), + (_py_class_role, 'unittest.result.TestResult'), + (_py_class_role, 'UUID'), + (_py_class_role, 'WebsocketFrame'), +] diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 0000000000..67a1035800 --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,25 @@ +(_proxy_py_glossary)= +# {{ project }} Glossary + +```{spelling} +DNS +DoH +Py +``` + +```{glossary} +DoH +DNS over HTTP +DNS-over-HTTP + + [DNS over HTTP] +``` + +```{glossary} +DNS + + Domain Name System +``` + +[DNS over HTTP]: +https://datatracker.ietf.org/doc/html/draft-pauly-dprive-oblivious-doh-04.html diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000000..d99edc7e48 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,39 @@ +(_proxy_py_index)= +# {{ project }} Documentation + +```{spelling} +acceptor +acceptors +Changelog +decrypt +decrypted +discoverable +DNS +Facebook +http +macOS +Pre +Py +Scalable +``` + +```{include} ../README.md +:end-before: (DO-NOT-REMOVE-docs-badges-START) +``` + +```{include} ../README.md +:start-after: (DO-NOT-REMOVE-docs-badges-END) +``` + +```{toctree} +:hidden: + +Glossary +``` + +```{toctree} +:caption: Contributing +:hidden: + +Private unsupported (dev) API autodoc +``` diff --git a/docs/pkg/.gitignore b/docs/pkg/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/docs/pkg/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 0000000000..86acf774af --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,5 @@ +myst-parser[linkify] >= 0.15.2 +setuptools-scm >= 6.3.2 +Sphinx >= 4.3.0 +furo >= 2021.11.15 +sphinxcontrib-apidoc >= 0.3.0 diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000000..977c0889a6 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,272 @@ +# +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: +# +# pip-compile --allow-unsafe --generate-hashes --output-file=docs/requirements.txt --strip-extras docs/requirements.in +# +alabaster==0.7.12 \ + --hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \ + --hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 + # via sphinx +attrs==21.2.0 \ + --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \ + --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb + # via markdown-it-py +babel==2.9.1 \ + --hash=sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9 \ + --hash=sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0 + # via sphinx +beautifulsoup4==4.10.0 \ + --hash=sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf \ + --hash=sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891 + # via furo +certifi==2021.10.8 \ + --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \ + --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569 + # via requests +charset-normalizer==2.0.7 \ + --hash=sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0 \ + --hash=sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b + # via requests +docutils==0.17.1 \ + --hash=sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125 \ + --hash=sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61 + # via + # myst-parser + # sphinx +furo==2021.11.15 \ + --hash=sha256:17b9fcf4de20f661d13db1ea83f11f7bf30be13738cffc88637889bf79c0469f \ + --hash=sha256:bdca82c3f211a24f850dcb12be3cb0e3f152cd3f2adfc0449bf9db6a07856bd3 + # via -r docs/requirements.in +idna==3.3 \ + --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ + --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d + # via requests +imagesize==1.3.0 \ + --hash=sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c \ + --hash=sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d + # via sphinx +jinja2==3.0.3 \ + --hash=sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 \ + --hash=sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7 + # via + # myst-parser + # sphinx +linkify-it-py==1.0.2 \ + --hash=sha256:4f416e72a41d9a00ecf1270ffb28b033318e458ac1144eb7c326563968a5dd24 \ + --hash=sha256:6c37ef4fc3001b38bc2359ccb5dc7e54388ec5d54fe46d2dbcd9a081f90fdbe3 + # via myst-parser +markdown-it-py==1.1.0 \ + --hash=sha256:36be6bb3ad987bfdb839f5ba78ddf094552ca38ccbd784ae4f74a4e1419fc6e3 \ + --hash=sha256:98080fc0bc34c4f2bcf0846a096a9429acbd9d5d8e67ed34026c03c61c464389 + # via + # mdit-py-plugins + # myst-parser +markupsafe==2.0.1 \ + --hash=sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298 \ + --hash=sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64 \ + --hash=sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b \ + --hash=sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194 \ + --hash=sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567 \ + --hash=sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff \ + --hash=sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724 \ + --hash=sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74 \ + --hash=sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646 \ + --hash=sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35 \ + --hash=sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6 \ + --hash=sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a \ + --hash=sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6 \ + --hash=sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad \ + --hash=sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26 \ + --hash=sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38 \ + --hash=sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac \ + --hash=sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7 \ + --hash=sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6 \ + --hash=sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047 \ + --hash=sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75 \ + --hash=sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f \ + --hash=sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b \ + --hash=sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135 \ + --hash=sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8 \ + --hash=sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a \ + --hash=sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a \ + --hash=sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1 \ + --hash=sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9 \ + --hash=sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864 \ + --hash=sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914 \ + --hash=sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee \ + --hash=sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f \ + --hash=sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18 \ + --hash=sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8 \ + --hash=sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2 \ + --hash=sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d \ + --hash=sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b \ + --hash=sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b \ + --hash=sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86 \ + --hash=sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6 \ + --hash=sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f \ + --hash=sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb \ + --hash=sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833 \ + --hash=sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28 \ + --hash=sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e \ + --hash=sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415 \ + --hash=sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902 \ + --hash=sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f \ + --hash=sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d \ + --hash=sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9 \ + --hash=sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d \ + --hash=sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145 \ + --hash=sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066 \ + --hash=sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c \ + --hash=sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1 \ + --hash=sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a \ + --hash=sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207 \ + --hash=sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f \ + --hash=sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53 \ + --hash=sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd \ + --hash=sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134 \ + --hash=sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85 \ + --hash=sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9 \ + --hash=sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5 \ + --hash=sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94 \ + --hash=sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509 \ + --hash=sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51 \ + --hash=sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872 + # via jinja2 +mdit-py-plugins==0.2.8 \ + --hash=sha256:1833bf738e038e35d89cb3a07eb0d227ed647ce7dd357579b65343740c6d249c \ + --hash=sha256:5991cef645502e80a5388ec4fc20885d2313d4871e8b8e320ca2de14ac0c015f + # via myst-parser +myst-parser==0.15.2 \ + --hash=sha256:40124b6f27a4c42ac7f06b385e23a9dcd03d84801e9c7130b59b3729a554b1f9 \ + --hash=sha256:f7f3b2d62db7655cde658eb5d62b2ec2a4631308137bd8d10f296a40d57bbbeb + # via -r docs/requirements.in +packaging==21.2 \ + --hash=sha256:096d689d78ca690e4cd8a89568ba06d07ca097e3306a4381635073ca91479966 \ + --hash=sha256:14317396d1e8cdb122989b916fa2c7e9ca8e2be9e8060a6eff75b6b7b4d8a7e0 + # via + # setuptools-scm + # sphinx +pbr==5.7.0 \ + --hash=sha256:4651ca1445e80f2781827305de3d76b3ce53195f2227762684eb08f17bc473b7 \ + --hash=sha256:60002958e459b195e8dbe61bf22bcf344eedf1b4e03a321a5414feb15566100c + # via sphinxcontrib-apidoc +pygments==2.10.0 \ + --hash=sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380 \ + --hash=sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6 + # via sphinx +pyparsing==2.4.7 \ + --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ + --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b + # via packaging +pytz==2021.3 \ + --hash=sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c \ + --hash=sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326 + # via babel +pyyaml==6.0 \ + --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ + --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ + --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ + --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ + --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ + --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ + --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ + --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ + --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ + --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ + --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ + --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ + --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ + --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ + --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ + --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ + --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ + --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ + --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ + --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ + --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ + --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ + --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ + --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ + --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ + --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ + --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ + --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ + --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ + --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ + --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ + --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ + --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 + # via myst-parser +requests==2.26.0 \ + --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \ + --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7 + # via sphinx +setuptools-scm==6.3.2 \ + --hash=sha256:4c64444b1d49c4063ae60bfe1680f611c8b13833d556fd1d6050c0023162a119 \ + --hash=sha256:a49aa8081eeb3514eb9728fa5040f2eaa962d6c6f4ec9c32f6c1fba88f88a0f2 + # via -r docs/requirements.in +snowballstemmer==2.1.0 \ + --hash=sha256:b51b447bea85f9968c13b650126a888aabd4cb4463fca868ec596826325dedc2 \ + --hash=sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914 + # via sphinx +soupsieve==2.3.1 \ + --hash=sha256:1a3cca2617c6b38c0343ed661b1fa5de5637f257d4fe22bd9f1338010a1efefb \ + --hash=sha256:b8d49b1cd4f037c7082a9683dfa1801aa2597fb11c3a1155b7a5b94829b4f1f9 + # via beautifulsoup4 +sphinx==4.3.0 \ + --hash=sha256:6d051ab6e0d06cba786c4656b0fe67ba259fe058410f49e95bee6e49c4052cbf \ + --hash=sha256:7e2b30da5f39170efcd95c6270f07669d623c276521fee27ad6c380f49d2bf5b + # via + # -r docs/requirements.in + # furo + # myst-parser + # sphinxcontrib-apidoc +sphinxcontrib-apidoc==0.3.0 \ + --hash=sha256:6671a46b2c6c5b0dca3d8a147849d159065e50443df79614f921b42fbd15cb09 \ + --hash=sha256:729bf592cf7b7dd57c4c05794f732dc026127275d785c2a5494521fdde773fb9 + # via -r docs/requirements.in +sphinxcontrib-applehelp==1.0.2 \ + --hash=sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a \ + --hash=sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58 + # via sphinx +sphinxcontrib-devhelp==1.0.2 \ + --hash=sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e \ + --hash=sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4 + # via sphinx +sphinxcontrib-htmlhelp==2.0.0 \ + --hash=sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07 \ + --hash=sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2 + # via sphinx +sphinxcontrib-jsmath==1.0.1 \ + --hash=sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 \ + --hash=sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8 + # via sphinx +sphinxcontrib-qthelp==1.0.3 \ + --hash=sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72 \ + --hash=sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 \ + --hash=sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd \ + --hash=sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952 + # via sphinx +tomli==1.2.2 \ + --hash=sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee \ + --hash=sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade + # via setuptools-scm +uc-micro-py==1.0.1 \ + --hash=sha256:316cfb8b6862a0f1d03540f0ae6e7b033ff1fa0ddbe60c12cbe0d4cec846a69f \ + --hash=sha256:b7cdf4ea79433043ddfe2c82210208f26f7962c0cfbe3bacb05ee879a7fdb596 + # via linkify-it-py +urllib3==1.26.7 \ + --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece \ + --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844 + # via requests + +# The following packages are considered to be unsafe in a requirements file: +setuptools==59.0.1 \ + --hash=sha256:899d27ec8104a68d4ba813b1afd66708a1a10e9391e79be92c8c60f9c77d05e5 \ + --hash=sha256:dedb38ba61844d9df36072dad313cb79426fd50497aaac9c0da4cd50dbeeb110 + # via + # setuptools-scm + # sphinx diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt new file mode 100644 index 0000000000..79ecc548cf --- /dev/null +++ b/docs/spelling_wordlist.txt @@ -0,0 +1,7 @@ +IPv +Nginx +Pluggable +scm +Threadless +threadless +youtube diff --git a/proxy/__init__.py b/proxy/__init__.py index a2e0fa77ad..94b6d7203d 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -7,6 +7,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing + Submodules + Subpackages """ from .proxy import entry_point, main, Proxy from .testing import TestCase diff --git a/proxy/common/__init__.py b/proxy/common/__init__.py index 232621f0b5..02bacd6ce5 100644 --- a/proxy/common/__init__.py +++ b/proxy/common/__init__.py @@ -7,4 +7,8 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Submodules """ diff --git a/proxy/common/_compat.py b/proxy/common/_compat.py index 389c81a99d..eaaddfb9d5 100644 --- a/proxy/common/_compat.py +++ b/proxy/common/_compat.py @@ -1,4 +1,10 @@ -"""Compatibility code for using Proxy.py across various versions of Python.""" +"""Compatibility code for using Proxy.py across various versions of Python. + +.. spelling:: + + compat + py +""" import platform diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 96d65d5d0a..1c52438149 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -54,12 +54,14 @@ class FlagParser: Import `flag.flags` and use `add_argument` API to define custom flags within respective Python files. - Best Practice: - 1. Define flags at the top of your class files. - 2. DO NOT add flags within your class `__init__` method OR - within class methods. It MAY result into runtime exception, - especially if your class is initialized multiple times or if - class method registering the flag gets invoked multiple times. + Best Practice:: + + 1. Define flags at the top of your class files. + 2. DO NOT add flags within your class `__init__` method OR + within class methods. It MAY result into runtime exception, + especially if your class is initialized multiple times or if + class method registering the flag gets invoked multiple times. + """ def __init__(self) -> None: diff --git a/proxy/common/pki.py b/proxy/common/pki.py index 30eea2aa78..2a4395e904 100644 --- a/proxy/common/pki.py +++ b/proxy/common/pki.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + pki """ import os import sys diff --git a/proxy/common/utils.py b/proxy/common/utils.py index 52b7ac231a..14a162364e 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -7,6 +7,13 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + utils + websocket + Websocket + WebSocket """ import sys import ssl @@ -38,7 +45,7 @@ def is_threadless(threadless: bool, threaded: bool) -> bool: def is_py2() -> bool: - """Exists only to avoid mocking sys.version_info in tests.""" + """Exists only to avoid mocking :data:`sys.version_info` in tests.""" return sys.version_info.major == 2 diff --git a/proxy/core/__init__.py b/proxy/core/__init__.py index 232621f0b5..ae3ea4267f 100644 --- a/proxy/core/__init__.py +++ b/proxy/core/__init__.py @@ -7,4 +7,8 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Subpackages """ diff --git a/proxy/core/acceptor/__init__.py b/proxy/core/acceptor/__init__.py index c3fd68776b..73da204d72 100644 --- a/proxy/core/acceptor/__init__.py +++ b/proxy/core/acceptor/__init__.py @@ -7,6 +7,13 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor + acceptors + pre + Submodules """ from .acceptor import Acceptor from .pool import AcceptorPool diff --git a/proxy/core/acceptor/acceptor.py b/proxy/core/acceptor/acceptor.py index b264c1363b..ee380ccf4d 100644 --- a/proxy/core/acceptor/acceptor.py +++ b/proxy/core/acceptor/acceptor.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor + pre """ import socket import logging @@ -40,16 +45,28 @@ class Acceptor(multiprocessing.Process): `Acceptor` goes on to listen for new work over the received server socket. By default, `Acceptor` will spawn a new thread to handle each work. - However, when `--threadless` option is enabled, `Acceptor` process will also pre-spawns a - `Threadless` process during start-up. Accepted work is passed to these `Threadless` processes. - `Acceptor` process shares accepted work with a `Threadless` process over it's dedicated pipe. + However, when `--threadless` option is enabled, `Acceptor` process + will also pre-spawns a + :class:`~proxy.core.acceptor.threadless.Threadless` process during + start-up. Accepted work is passed to these + :class:`~proxy.core.acceptor.threadless.Threadless` processes. + `Acceptor` process shares accepted work with a + :class:`~proxy.core.acceptor.threadless.Threadless` process over + it's dedicated pipe. + + TODO(abhinavsingh): Open questions:: + + 1. Instead of starting + :class:`~proxy.core.acceptor.threadless.Threadless` process, + can we work with a + :class:`~proxy.core.acceptor.threadless.Threadless` thread? + 2. What are the performance implications of sharing fds between + threads vs processes? + 3. How much performance degradation happens when acceptor and + threadless processes are running on separate CPU cores? + 4. Can we ensure both acceptor and threadless process are pinned to + the same CPU core? - TODO(abhinavsingh): Open questions: - 1) Instead of starting `Threadless` process, can we work with a `Threadless` thread? - 2) What are the performance implications of sharing fds between threads vs processes? - 3) How much performance degradation happens when acceptor and threadless processes are - running on separate CPU cores? - 4) Can we ensure both acceptor and threadless process are pinned to the same CPU core? """ def __init__( diff --git a/proxy/core/acceptor/executors.py b/proxy/core/acceptor/executors.py index 0070fade7e..3bafe48091 100644 --- a/proxy/core/acceptor/executors.py +++ b/proxy/core/acceptor/executors.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor """ import socket import logging @@ -64,7 +68,7 @@ class ThreadlessPool: """Manages lifecycle of threadless pool and delegates work to them using a round-robin strategy. - Example usage: + Example usage:: with ThreadlessPool(flags=...) as pool: while True: @@ -74,10 +78,10 @@ class ThreadlessPool: work classes. TODO: We could optimize multiple-work-type scenario - by making Threadless class constructor independent of work_klass. - We could then relay the work_klass during work delegation. + by making Threadless class constructor independent of ``work_klass``. + We could then relay the ``work_klass`` during work delegation. This will also make ThreadlessPool constructor agnostic - of work_klass. + of ``work_klass``. """ def __init__( diff --git a/proxy/core/acceptor/listener.py b/proxy/core/acceptor/listener.py index 745903816e..53559f1fbe 100644 --- a/proxy/core/acceptor/listener.py +++ b/proxy/core/acceptor/listener.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor """ import os import socket diff --git a/proxy/core/acceptor/pool.py b/proxy/core/acceptor/pool.py index ff680f6bb8..66d0421b9e 100644 --- a/proxy/core/acceptor/pool.py +++ b/proxy/core/acceptor/pool.py @@ -7,6 +7,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor + acceptors + pre """ import logging import argparse @@ -41,12 +47,15 @@ class AcceptorPool: - """AcceptorPool is a helper class which pre-spawns `Acceptor` processes - to utilize all available CPU cores for accepting new work. - - A file descriptor to consume work from is shared with `Acceptor` processes - over a pipe. Each `Acceptor` process then concurrently accepts new work over - the shared file descriptor. + """AcceptorPool is a helper class which pre-spawns + :py:class:`~proxy.core.acceptor.acceptor.Acceptor` processes to + utilize all available CPU cores for accepting new work. + + A file descriptor to consume work from is shared with + :py:class:`~proxy.core.acceptor.acceptor.Acceptor` processes over a + pipe. Each :py:class:`~proxy.core.acceptor.acceptor.Acceptor` + process then concurrently accepts new work over the shared file + descriptor. Example usage: diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py index b64589bc87..006e1c4ba2 100644 --- a/proxy/core/acceptor/threadless.py +++ b/proxy/core/acceptor/threadless.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor """ import argparse import os @@ -37,19 +41,23 @@ class Threadless(multiprocessing.Process): """Work executor process. Threadless process provides an event loop, which is shared across - multiple `Work` instances to handle work. + multiple :class:`~proxy.core.acceptor.work.Work` instances to handle + work. Threadless takes input a `work_klass` and an `event_queue`. `work_klass` - must conform to the `Work` protocol. Work is received over the - `event_queue`. + must conform to the :class:`~proxy.core.acceptor.work.Work` + protocol. Work is received over the `event_queue`. When a work is accepted, threadless creates a new instance of `work_klass`. - Threadless will then invoke necessary lifecycle of the `Work` protocol, + Threadless will then invoke necessary lifecycle of the + :class:`~proxy.core.acceptor.work.Work` protocol, allowing `work_klass` implementation to handle the assigned work. - Example, `BaseTcpServerHandler` implements `Work` protocol. It expects - a client connection as work payload and hooks into the threadless - event loop to handle the client connection. + Example, :class:`~proxy.core.base.tcp_server.BaseTcpServerHandler` + implements :class:`~proxy.core.acceptor.work.Work` protocol. It + expects a client connection as work payload and hooks into the + threadless event loop to handle the client connection. + """ def __init__( diff --git a/proxy/core/acceptor/work.py b/proxy/core/acceptor/work.py index 52d5103d11..ea05a8056b 100644 --- a/proxy/core/acceptor/work.py +++ b/proxy/core/acceptor/work.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + acceptor """ import argparse import socket diff --git a/proxy/core/base/__init__.py b/proxy/core/base/__init__.py index c60f14778e..721d83e2e1 100644 --- a/proxy/core/base/__init__.py +++ b/proxy/core/base/__init__.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Submodules """ from .tcp_server import BaseTcpServerHandler from .tcp_tunnel import BaseTcpTunnelHandler diff --git a/proxy/core/base/tcp_server.py b/proxy/core/base/tcp_server.py index fa1ca9a14b..fcab812341 100644 --- a/proxy/core/base/tcp_server.py +++ b/proxy/core/base/tcp_server.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + tcp """ import socket import logging @@ -36,10 +40,12 @@ class BaseTcpServerHandler(Work): Most importantly, BaseTcpServerHandler ensures that pending buffers to the client are flushed before connection is closed. - Implementations must provide: - a) handle_data(data: memoryview) implementation - b) Optionally, also implement other Work method - e.g. initialize, is_inactive, shutdown + Implementations must provide:: + + a. handle_data(data: memoryview) implementation + b. Optionally, also implement other Work method + e.g. initialize, is_inactive, shutdown + """ def __init__(self, *args: Any, **kwargs: Any) -> None: diff --git a/proxy/core/base/tcp_tunnel.py b/proxy/core/base/tcp_tunnel.py index af46557c3a..9e4b7c156f 100644 --- a/proxy/core/base/tcp_tunnel.py +++ b/proxy/core/base/tcp_tunnel.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + tcp """ import socket import logging @@ -26,7 +30,7 @@ class BaseTcpTunnelHandler(BaseTcpServerHandler): - """BaseTcpTunnelHandler build on-top of BaseTcpServerHandler work klass. + """BaseTcpTunnelHandler build on-top of BaseTcpServerHandler work class. On-top of BaseTcpServerHandler implementation, BaseTcpTunnelHandler introduces an upstream TcpServerConnection diff --git a/proxy/core/connection/__init__.py b/proxy/core/connection/__init__.py index 88f5902900..952ee08f9e 100644 --- a/proxy/core/connection/__init__.py +++ b/proxy/core/connection/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + reusability + Submodules """ from .connection import TcpConnection, TcpConnectionUninitializedException from .client import TcpClientConnection diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py index 13a8f9b2a5..16cd5096b1 100644 --- a/proxy/core/connection/pool.py +++ b/proxy/core/connection/pool.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + reusability """ import logging diff --git a/proxy/core/connection/types.py b/proxy/core/connection/types.py index 44522c81b8..663c0ae4d4 100644 --- a/proxy/core/connection/types.py +++ b/proxy/core/connection/types.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + iterable """ from typing import NamedTuple diff --git a/proxy/core/event/__init__.py b/proxy/core/event/__init__.py index 17e1074e6e..08b6f5be49 100644 --- a/proxy/core/event/__init__.py +++ b/proxy/core/event/__init__.py @@ -7,6 +7,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing + iterable + Submodules """ from .queue import EventQueue from .names import EventNames, eventNames diff --git a/proxy/core/event/manager.py b/proxy/core/event/manager.py index 164cf90095..3923c6c12e 100644 --- a/proxy/core/event/manager.py +++ b/proxy/core/event/manager.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing """ import logging import threading diff --git a/proxy/core/event/names.py b/proxy/core/event/names.py index 5040c880a8..9a58926c6c 100644 --- a/proxy/core/event/names.py +++ b/proxy/core/event/names.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing + iterable """ from typing import NamedTuple diff --git a/proxy/core/event/queue.py b/proxy/core/event/queue.py index 21c0cc3c57..f0110e0401 100644 --- a/proxy/core/event/queue.py +++ b/proxy/core/event/queue.py @@ -26,19 +26,21 @@ class EventQueue: subscribers need to dispatch their subscription queue over this global queue. - Each published event contains following schema: - { - 'request_id': 'Globally unique request ID', - 'process_id': 'Process ID of event publisher. ' - 'This will be the process ID of acceptor workers.', - 'thread_id': 'Thread ID of event publisher. ' - 'When --threadless is enabled, this value will be ' - 'same for all the requests.' - 'event_timestamp': 'Time when this event occured', - 'event_name': 'one of the pre-defined or custom event name', - 'event_payload': 'Optional data associated with the event', - 'publisher_id': 'Optional publisher entity unique name', - } + Each published event contains following schema:: + + { + 'request_id': 'Globally unique request ID', + 'process_id': 'Process ID of event publisher. This ' + 'will be the process ID of acceptor workers.', + 'thread_id': 'Thread ID of event publisher. ' + 'When --threadless is enabled, this value ' + 'will be same for all the requests.' + 'event_timestamp': 'Time when this event occured', + 'event_name': 'one of the pre-defined or custom event name', + 'event_payload': 'Optional data associated with the event', + 'publisher_id': 'Optional publisher entity unique name', + } + """ def __init__(self, queue: DictQueueType) -> None: diff --git a/proxy/core/event/subscriber.py b/proxy/core/event/subscriber.py index 3e9b705168..c92803074f 100644 --- a/proxy/core/event/subscriber.py +++ b/proxy/core/event/subscriber.py @@ -79,7 +79,7 @@ def setup(self, do_subscribe: bool = True) -> None: self.subscribe() def shutdown(self, do_unsubscribe: bool = True) -> None: - """Teardown subscription thread. + """Tear down subscription thread. Call unsubscribe() to actually stop subscription. """ diff --git a/proxy/core/ssh/__init__.py b/proxy/core/ssh/__init__.py index f793bc839a..e37310801c 100644 --- a/proxy/core/ssh/__init__.py +++ b/proxy/core/ssh/__init__.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Submodules """ from .client import SshClient from .tunnel import Tunnel diff --git a/proxy/core/ssh/client.py b/proxy/core/ssh/client.py index 650d894809..4657b1a3c1 100644 --- a/proxy/core/ssh/client.py +++ b/proxy/core/ssh/client.py @@ -18,7 +18,7 @@ class SshClient(TcpClientConnection): """Overrides TcpClientConnection. - This is necessary because paramiko fileno() can be used for polling + This is necessary because paramiko ``fileno()`` can be used for polling but not for send / recv. """ diff --git a/proxy/core/ssh/tunnel.py b/proxy/core/ssh/tunnel.py index 5de8ae27ab..4a899543ae 100644 --- a/proxy/core/ssh/tunnel.py +++ b/proxy/core/ssh/tunnel.py @@ -19,7 +19,7 @@ class Tunnel: """Establishes a tunnel between local (machine where Tunnel is running) and remote host. Once a tunnel has been established, remote host can route HTTP(s) traffic to - localhost over tunnel. + ``localhost`` over tunnel. """ def __init__( diff --git a/proxy/dashboard/__init__.py b/proxy/dashboard/__init__.py index 0f5d329522..61f5ec0232 100644 --- a/proxy/dashboard/__init__.py +++ b/proxy/dashboard/__init__.py @@ -7,6 +7,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Submodules + websocket + Websocket """ from .dashboard import ProxyDashboard from .inspect_traffic import InspectTrafficPlugin diff --git a/proxy/dashboard/inspect_traffic.py b/proxy/dashboard/inspect_traffic.py index 33fa212091..15edc0d687 100644 --- a/proxy/dashboard/inspect_traffic.py +++ b/proxy/dashboard/inspect_traffic.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + websocket + Websocket """ import json from typing import List, Dict, Any diff --git a/proxy/http/__init__.py b/proxy/http/__init__.py index f8d2e4fa4f..b1d6877d15 100644 --- a/proxy/http/__init__.py +++ b/proxy/http/__init__.py @@ -7,6 +7,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Subpackages + Submodules """ from .handler import HttpProtocolHandler from .plugin import HttpProtocolHandlerPlugin diff --git a/proxy/http/codes.py b/proxy/http/codes.py index c4f9a91cf7..ad6716090d 100644 --- a/proxy/http/codes.py +++ b/proxy/http/codes.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable """ from typing import NamedTuple diff --git a/proxy/http/exception/__init__.py b/proxy/http/exception/__init__.py index 513d2bd510..bb0bf7b1e8 100644 --- a/proxy/http/exception/__init__.py +++ b/proxy/http/exception/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules """ from .base import HttpProtocolException from .http_request_rejected import HttpRequestRejected diff --git a/proxy/http/exception/base.py b/proxy/http/exception/base.py index 65138e87b7..37817c9265 100644 --- a/proxy/http/exception/base.py +++ b/proxy/http/exception/base.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ from typing import Optional @@ -14,11 +18,12 @@ class HttpProtocolException(Exception): - """Top level HttpProtocolException exception class. + """Top level :exc:`HttpProtocolException` exception class. - All exceptions raised during execution of Http request lifecycle MUST - inherit HttpProtocolException base class. Implement response() method - to optionally return custom response to client.""" + All exceptions raised during execution of HTTP request lifecycle MUST + inherit :exc:`HttpProtocolException` base class. Implement + ``response()`` method to optionally return custom response to client. + """ def response(self, request: HttpParser) -> Optional[memoryview]: return None # pragma: no cover diff --git a/proxy/http/exception/http_request_rejected.py b/proxy/http/exception/http_request_rejected.py index a0fa810fc1..d17571bd26 100644 --- a/proxy/http/exception/http_request_rejected.py +++ b/proxy/http/exception/http_request_rejected.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ from typing import Optional, Dict diff --git a/proxy/http/exception/proxy_auth_failed.py b/proxy/http/exception/proxy_auth_failed.py index 60c6e72219..d82211d092 100644 --- a/proxy/http/exception/proxy_auth_failed.py +++ b/proxy/http/exception/proxy_auth_failed.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + auth + http """ from .base import HttpProtocolException @@ -18,7 +23,7 @@ class ProxyAuthenticationFailed(HttpProtocolException): - """Exception raised when Http Proxy auth is enabled and + """Exception raised when HTTP Proxy auth is enabled and incoming request doesn't present necessary credentials.""" RESPONSE_PKT = memoryview( diff --git a/proxy/http/exception/proxy_conn_failed.py b/proxy/http/exception/proxy_conn_failed.py index daef357647..86d13cc880 100644 --- a/proxy/http/exception/proxy_conn_failed.py +++ b/proxy/http/exception/proxy_conn_failed.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + conn + http """ from .base import HttpProtocolException @@ -18,7 +23,7 @@ class ProxyConnectionFailed(HttpProtocolException): - """Exception raised when HttpProxyPlugin is unable to establish connection to upstream server.""" + """Exception raised when ``HttpProxyPlugin`` is unable to establish connection to upstream server.""" RESPONSE_PKT = memoryview( build_http_response( diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 6d6d15116a..2cc10b8ea1 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ import ssl import time @@ -90,7 +94,7 @@ def __init__(self, *args: Any, **kwargs: Any): ## def initialize(self) -> None: - """Optionally upgrades connection to HTTPS, set conn in non-blocking mode and initializes plugins.""" + """Optionally upgrades connection to HTTPS, set ``conn`` in non-blocking mode and initializes plugins.""" conn = self._optionally_wrap_socket(self.work.connection) conn.setblocking(False) # Update client connection reference if connection was wrapped @@ -170,7 +174,7 @@ def handle_events( readables: Readables, writables: Writables, ) -> bool: - """Returns True if proxy must teardown.""" + """Returns True if proxy must tear down.""" # Flush buffer for ready to write sockets teardown = self.handle_writables(writables) if teardown: diff --git a/proxy/http/inspector/__init__.py b/proxy/http/inspector/__init__.py index 8e5d4aa017..6e968ddebe 100644 --- a/proxy/http/inspector/__init__.py +++ b/proxy/http/inspector/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules """ from .devtools import DevtoolsProtocolPlugin diff --git a/proxy/http/inspector/devtools.py b/proxy/http/inspector/devtools.py index 2199b404a6..ee2a4d8e67 100644 --- a/proxy/http/inspector/devtools.py +++ b/proxy/http/inspector/devtools.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + devtools + http """ import json import logging diff --git a/proxy/http/inspector/transformer.py b/proxy/http/inspector/transformer.py index 2512a48c35..97fec409b3 100644 --- a/proxy/http/inspector/transformer.py +++ b/proxy/http/inspector/transformer.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ import json import time @@ -23,7 +27,7 @@ class CoreEventsToDevtoolsProtocol: """Open in Chrome - devtools://devtools/bundled/inspector.html?ws=localhost:8899/devtools + ``devtools://devtools/bundled/inspector.html?ws=localhost:8899/devtools`` """ RESPONSES: Dict[str, bytes] = {} diff --git a/proxy/http/methods.py b/proxy/http/methods.py index 791b41c5c5..dbf1f2f7be 100644 --- a/proxy/http/methods.py +++ b/proxy/http/methods.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable """ from typing import NamedTuple diff --git a/proxy/http/parser/__init__.py b/proxy/http/parser/__init__.py index 27326c6233..be633d4873 100644 --- a/proxy/http/parser/__init__.py +++ b/proxy/http/parser/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules """ from .parser import HttpParser from .chunk import ChunkParser, chunkParserStates diff --git a/proxy/http/parser/chunk.py b/proxy/http/parser/chunk.py index 2d976c4449..fa60d437df 100644 --- a/proxy/http/parser/chunk.py +++ b/proxy/http/parser/chunk.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable """ from typing import NamedTuple, Tuple, List, Optional diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py index d43e0c4fe1..f81a100b22 100644 --- a/proxy/http/parser/parser.py +++ b/proxy/http/parser/parser.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ from typing import TypeVar, Optional, Dict, Type, Tuple, List @@ -40,13 +44,13 @@ class HttpParser: """HTTP request/response parser. - TODO: Make me zero-copy by using memoryview. + TODO: Make me zero-copy by using :class:`memoryview`. Currently due to chunk/buffer handling we - are not able to utilize memoryview + are not able to utilize :class:`memoryview` efficiently. - For this to happen we must store `buffer` - as List[memoryview] instead of raw bytes and + For this to happen we must store ``buffer`` + as ``List[memoryview]`` instead of raw bytes and update parser to work accordingly. """ @@ -101,7 +105,7 @@ def response(cls: Type[T], raw: bytes) -> T: return parser def header(self, key: bytes) -> bytes: - """Convenient method to return original header value from internal datastructure.""" + """Convenient method to return original header value from internal data structure.""" if key.lower() not in self.headers: raise KeyError('%s not found in headers', text_(key)) return self.headers[key.lower()][1] @@ -125,7 +129,7 @@ def del_header(self, header: bytes) -> None: del self.headers[header.lower()] def del_headers(self, headers: List[bytes]) -> None: - """Delete headers from internal datastructure.""" + """Delete headers from internal data structure.""" for key in headers: self.del_header(key.lower()) @@ -172,7 +176,7 @@ def body_expected(self) -> bool: return self.content_expected() or self.is_chunked_encoded() def parse(self, raw: bytes) -> None: - """Parses Http request out of raw bytes. + """Parses HTTP request out of raw bytes. Check for `HttpParser.state` after `parse` has successfully returned.""" self.total_size += len(raw) diff --git a/proxy/http/parser/protocol.py b/proxy/http/parser/protocol.py index d749fdc08a..84982d1022 100644 --- a/proxy/http/parser/protocol.py +++ b/proxy/http/parser/protocol.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ from typing import Optional, Tuple diff --git a/proxy/http/parser/types.py b/proxy/http/parser/types.py index 582b22921b..c5e019c6e3 100644 --- a/proxy/http/parser/types.py +++ b/proxy/http/parser/types.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable """ from typing import NamedTuple diff --git a/proxy/http/plugin.py b/proxy/http/plugin.py index b9e15b7d56..f5d49ef192 100644 --- a/proxy/http/plugin.py +++ b/proxy/http/plugin.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ import socket import argparse @@ -41,7 +45,7 @@ class HttpProtocolHandlerPlugin(ABC): 4. Server Response Chunk Received on_response_chunk is called for every chunk received from the server. 5. Client Connection Closed - Add your logic within `on_client_connection_close` for any per connection teardown. + Add your logic within `on_client_connection_close` for any per connection tear-down. """ def __init__( @@ -79,8 +83,9 @@ def write_to_descriptors(self, w: Writables) -> bool: """Implementations must now write/flush data over the socket. Note that buffer management is in-build into the connection classes. - Hence implementations MUST call `flush` here, to send any buffered data - over the socket. + Hence implementations MUST call + :meth:`~proxy.core.connection.TcpConnection.flush` here, to send + any buffered data over the socket. """ return False # pragma: no cover diff --git a/proxy/http/proxy/__init__.py b/proxy/http/proxy/__init__.py index 4e18002c0d..69568bd96f 100644 --- a/proxy/http/proxy/__init__.py +++ b/proxy/http/proxy/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules """ from .plugin import HttpProxyBasePlugin from .server import HttpProxyPlugin diff --git a/proxy/http/proxy/auth.py b/proxy/http/proxy/auth.py index 103e082061..5653a9528c 100644 --- a/proxy/http/proxy/auth.py +++ b/proxy/http/proxy/auth.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + auth + http """ from typing import Optional diff --git a/proxy/http/proxy/plugin.py b/proxy/http/proxy/plugin.py index 7ca25d049e..94eed56934 100644 --- a/proxy/http/proxy/plugin.py +++ b/proxy/http/proxy/plugin.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ import socket import argparse @@ -67,8 +71,9 @@ def write_to_descriptors(self, w: Writables) -> bool: """Implementations must now write/flush data over the socket. Note that buffer management is in-build into the connection classes. - Hence implementations MUST call `flush` here, to send any buffered data - over the socket. + Hence implementations MUST call + :meth:`~proxy.core.connection.connection.TcpConnection.flush` + here, to send any buffered data over the socket. """ return False # pragma: no cover @@ -85,7 +90,7 @@ def resolve_dns(self, host: str, port: int) -> Tuple[Optional[str], Optional[Tup For upstream IP: Return None to use default resolver available to the system. - Return ip address as string to use your custom resolver. + Return IP address as string to use your custom resolver. For source address: Return None to use default source address @@ -119,7 +124,7 @@ def handle_client_data( Essentially, if you return None from within before_upstream_connection, be prepared to handle_client_data and not handle_client_request. - Raise HttpRequestRejected to teardown the connection + Raise HttpRequestRejected to tear down the connection Return None to drop the connection """ return raw # pragma: no cover @@ -142,7 +147,8 @@ def handle_client_request( Return optionally modified request object to dispatch to upstream. Return None to drop the request data, e.g. in case a response has already been queued. Raise HttpRequestRejected or HttpProtocolException directly to - teardown the connection with client. + tear down the connection with client. + """ return request # pragma: no cover diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 6f76e94eaa..e7c1e9d34f 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + reusability """ import os import ssl diff --git a/proxy/http/server/__init__.py b/proxy/http/server/__init__.py index 059c2cc128..f100ddaf31 100644 --- a/proxy/http/server/__init__.py +++ b/proxy/http/server/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules """ from .web import HttpWebServerPlugin from .pac_plugin import HttpWebServerPacFilePlugin diff --git a/proxy/http/server/pac_plugin.py b/proxy/http/server/pac_plugin.py index a2e0237a4a..581f185a4b 100644 --- a/proxy/http/server/pac_plugin.py +++ b/proxy/http/server/pac_plugin.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + pac """ import gzip diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index 658ce1a5d2..55e66f39c2 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ import socket import argparse @@ -66,8 +70,9 @@ def write_to_descriptors(self, w: Writables) -> bool: """Implementations must now write/flush data over the socket. Note that buffer management is in-build into the connection classes. - Hence implementations MUST call `flush` here, to send any buffered data - over the socket. + Hence implementations MUST call + :meth:`~proxy.core.connection.connection.TcpConnection.flush` + here, to send any buffered data over the socket. """ return False # pragma: no cover diff --git a/proxy/http/server/protocols.py b/proxy/http/server/protocols.py index a3d8434808..a561cb936f 100644 --- a/proxy/http/server/protocols.py +++ b/proxy/http/server/protocols.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable """ from typing import NamedTuple diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index b3773bc2ac..9c44f836c3 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http """ import re import gzip diff --git a/proxy/http/url.py b/proxy/http/url.py index 05d5fec385..5c7eb7c7b9 100644 --- a/proxy/http/url.py +++ b/proxy/http/url.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + url """ from typing import Optional, Tuple @@ -14,9 +19,9 @@ class Url: - """urllib.urlparse doesn't work for proxy.py, so we wrote a simple Url. + """``urllib.urlparse`` doesn't work for proxy.py, so we wrote a simple URL. - Currently, Url only implements what is necessary for HttpParser to work. + Currently, URL only implements what is necessary for HttpParser to work. """ def __init__( @@ -33,19 +38,19 @@ def __init__( @classmethod def from_bytes(cls, raw: bytes) -> 'Url': - """A Url within proxy.py core can have several styles, + """A URL within proxy.py core can have several styles, because proxy.py supports both proxy and web server use cases. Example: - For a Web server, url is like "/" or "/get" or "/get?key=value" - For a HTTPS connect tunnel, url is like "httpbin.org:443" - For a HTTP proxy request, url is like "http://httpbin.org/get" + For a Web server, url is like ``/`` or ``/get`` or ``/get?key=value`` + For a HTTPS connect tunnel, url is like ``httpbin.org:443`` + For a HTTP proxy request, url is like ``http://httpbin.org/get`` Further: - 1) Url may contain unicode characters - 2) Url may contain IPv4 and IPv6 format addresses instead of domain names + 1) URL may contain unicode characters + 2) URL may contain IPv4 and IPv6 format addresses instead of domain names - We use heuristics based approach for our Url parser. + We use heuristics based approach for our URL parser. """ sraw = raw.decode('utf-8') if sraw[0] == SLASH.decode('utf-8'): diff --git a/proxy/http/websocket/__init__.py b/proxy/http/websocket/__init__.py index 2870e3b26f..350ab7b8ac 100644 --- a/proxy/http/websocket/__init__.py +++ b/proxy/http/websocket/__init__.py @@ -7,6 +7,13 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + Submodules + websocket + Websocket """ from .frame import WebsocketFrame, websocketOpcodes from .client import WebsocketClient diff --git a/proxy/http/websocket/client.py b/proxy/http/websocket/client.py index 648dee0802..8613b5fa8d 100644 --- a/proxy/http/websocket/client.py +++ b/proxy/http/websocket/client.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + websocket """ import ssl import base64 diff --git a/proxy/http/websocket/frame.py b/proxy/http/websocket/frame.py index 8207ca9b60..97387e59ef 100644 --- a/proxy/http/websocket/frame.py +++ b/proxy/http/websocket/frame.py @@ -7,6 +7,13 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + http + iterable + websocket + Websocket """ import io import hashlib diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index 6e2ce5e6a6..1581c64933 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -7,6 +7,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Cloudflare + Submodules """ from .cache import CacheResponsesPlugin, BaseCacheResponsesPlugin from .filter_by_upstream import FilterByUpstreamHostPlugin diff --git a/proxy/plugin/cloudflare_dns.py b/proxy/plugin/cloudflare_dns.py index 95ead3eda5..781cb8d489 100644 --- a/proxy/plugin/cloudflare_dns.py +++ b/proxy/plugin/cloudflare_dns.py @@ -7,6 +7,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + Cloudflare + cloudflare + dns """ import logging @@ -34,17 +40,23 @@ class CloudflareDnsResolverPlugin(HttpProxyBasePlugin): """This plugin uses Cloudflare DNS resolver to provide protection - against malwares and adult content. Implementation uses DoH specification. + against malware and adult content. Implementation uses :term:`DoH` + specification. See https://developers.cloudflare.com/1.1.1.1/1.1.1.1-for-families See https://developers.cloudflare.com/1.1.1.1/encrypted-dns/dns-over-https/make-api-requests/dns-json - NOTE: For this plugin to work, make sure to bypass proxy for 1.1.1.1 + .. note:: + + For this plugin to work, make sure to bypass proxy for 1.1.1.1 + + .. note:: - NOTE: This plugin requires additional dependency because DoH mandates - a HTTP2 complaint client. Install `httpx` dependency as: + This plugin requires additional dependency because :term:`DoH` + mandates a HTTP2 complaint client. Install `httpx` dependency + as:: - pip install "httpx[http2]" + pip install "httpx[http2]" """ def resolve_dns(self, host: str, port: int) -> Tuple[Optional[str], Optional[Tuple[str, int]]]: diff --git a/proxy/plugin/custom_dns_resolver.py b/proxy/plugin/custom_dns_resolver.py index a61480aa2f..f40a93e8ed 100644 --- a/proxy/plugin/custom_dns_resolver.py +++ b/proxy/plugin/custom_dns_resolver.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + dns """ import socket @@ -21,11 +25,12 @@ class CustomDnsResolverPlugin(HttpProxyBasePlugin): def resolve_dns(self, host: str, port: int) -> Tuple[Optional[str], Optional[Tuple[str, int]]]: """Here we are using in-built python resolver for demonstration. - Ideally you would like to query your custom DNS server or even use DoH to make - real sense out of this plugin. + Ideally you would like to query your custom DNS server or even + use :term:`DoH` to make real sense out of this plugin. - 2nd parameter returned is None. Return a 2-tuple to configure underlying interface - to use for connection to the upstream server. + The second parameter returned is None. Return a 2-tuple to + configure underlying interface to use for connection to the + upstream server. """ try: return socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)[0][4][0], None diff --git a/proxy/plugin/filter_by_client_ip.py b/proxy/plugin/filter_by_client_ip.py index e946186c5c..4b3724e96d 100644 --- a/proxy/plugin/filter_by_client_ip.py +++ b/proxy/plugin/filter_by_client_ip.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + ip """ from typing import Optional diff --git a/proxy/plugin/filter_by_url_regex.py b/proxy/plugin/filter_by_url_regex.py index 15d2531452..ef5dd80530 100644 --- a/proxy/plugin/filter_by_url_regex.py +++ b/proxy/plugin/filter_by_url_regex.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + url """ import json import logging diff --git a/proxy/plugin/mock_rest_api.py b/proxy/plugin/mock_rest_api.py index 2643d1e038..12e3543deb 100644 --- a/proxy/plugin/mock_rest_api.py +++ b/proxy/plugin/mock_rest_api.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + api """ import json from typing import Optional @@ -28,7 +32,7 @@ class ProposedRestApiPlugin(HttpProxyBasePlugin): without establishing upstream connection. Note: This plugin won't work if your client is making - HTTPS connection to api.example.com. + HTTPS connection to ``api.example.com``. """ API_SERVER = b'api.example.com' diff --git a/proxy/plugin/reverse_proxy.py b/proxy/plugin/reverse_proxy.py index 13f5b80c84..23afd986a1 100644 --- a/proxy/plugin/reverse_proxy.py +++ b/proxy/plugin/reverse_proxy.py @@ -34,14 +34,17 @@ class ReverseProxyPlugin(HttpWebServerBasePlugin): """Extend in-built Web Server to add Reverse Proxy capabilities. - This example plugin is equivalent to following Nginx configuration: + This example plugin is equivalent to following Nginx configuration:: + ```text location /get { proxy_pass http://httpbin.org/get } + ``` - Example: + Example:: + ```console $ curl http://localhost:9000/get { "args": {}, @@ -53,6 +56,8 @@ class ReverseProxyPlugin(HttpWebServerBasePlugin): "origin": "1.2.3.4, 5.6.7.8", "url": "http://localhost/get" } + ``` + """ # TODO: We must use nginx python parser and diff --git a/proxy/plugin/shortlink.py b/proxy/plugin/shortlink.py index 73db81e710..5b1ff48ffa 100644 --- a/proxy/plugin/shortlink.py +++ b/proxy/plugin/shortlink.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + shortlink """ from typing import Optional @@ -23,15 +27,15 @@ class ShortLinkPlugin(HttpProxyBasePlugin): Enable ShortLinkPlugin and speed up your daily browsing experience. - Example: - * f/ for facebook.com - * g/ for google.com - * t/ for twitter.com - * y/ for youtube.com - * proxy/ for py internal web servers. + Example:: + * ``f/`` for ``facebook.com`` + * ``g/`` for ``google.com` + * ``t/`` for ``twitter.com`` + * ``y/`` for ``youtube.com`` + * ``proxy/`` for ``py`` internal web servers. Customize map below for your taste and need. - Paths are also preserved. E.g. t/imoracle will + Paths are also preserved. E.g. ``t/imoracle`` will resolve to http://twitter.com/imoracle. """ diff --git a/proxy/proxy.py b/proxy/proxy.py index 706f864441..899b723dbc 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing """ import os import sys @@ -112,7 +116,8 @@ class Proxy: """Context manager to control AcceptorPool, ExecutorPool & EventingCore lifecycle. - By default, AcceptorPool is started with `HttpProtocolHandler` work class. + By default, AcceptorPool is started with + :class:`~proxy.http.handler.HttpProtocolHandler` work class. By definition, it expects HTTP traffic to flow between clients and server. Optionally, it also initializes the eventing core, a multi-process safe diff --git a/proxy/testing/test_case.py b/proxy/testing/test_case.py index 2bec2778d1..d9f47c143f 100644 --- a/proxy/testing/test_case.py +++ b/proxy/testing/test_case.py @@ -20,7 +20,7 @@ class TestCase(unittest.TestCase): - """Base TestCase class that automatically setup and teardown proxy.py.""" + """Base TestCase class that automatically setup and tear down proxy.py.""" DEFAULT_PROXY_PY_STARTUP_FLAGS = [ '--num-workers', '1', diff --git a/tests/test_main.py b/tests/test_main.py index f7959127b1..d2bc641920 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,6 +7,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. + + .. spelling:: + + eventing """ import os import tempfile diff --git a/tox.ini b/tox.ini index 236ea354ab..5e315c8241 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,139 @@ setenv = PEP517_OUT_DIR = {env:PEP517_OUT_DIR:{toxinidir}{/}dist} +[testenv:build-docs] +allowlist_externals = + git +basepython = python3 +commands_pre = + # Paramiko: + {envpython} -m pip install -r{toxinidir}/requirements-tunnel.txt +commands = + # Retrieve possibly missing commits: + -git fetch --unshallow + -git fetch --tags + + # Build the html docs with Sphinx: + {envpython} -m sphinx \ + -j auto \ + -b html \ + {tty:--color} \ + -a \ + -n \ + -W --keep-going \ + -d "{temp_dir}/.doctrees" \ + . \ + "{envdir}/docs_out" + + # Print out the output docs dir and a way to serve html: + -{envpython} -c\ + 'import pathlib;\ + docs_dir = pathlib.Path(r"{envdir}") / "docs_out";\ + index_file = docs_dir / "index.html";\ + print("\n" + "=" * 120 +\ + f"\n\nDocumentation available under:\n\n\ + \tfile://\{index_file\}\n\nTo serve docs, use\n\n\ + \t$ python3 -m http.server --directory \ + \N\{QUOTATION MARK\}\{docs_dir\}\N\{QUOTATION MARK\} 0\n\n" +\ + "=" * 120)' +changedir = {toxinidir}/docs +deps = + -r{toxinidir}/docs/requirements.txt + # FIXME: re-enable the "-r" + "-c" paradigm once the pip bug is fixed. + # Ref: https://github.com/pypa/pip/issues/9243 + # -r{toxinidir}/docs/requirements.in + # -c{toxinidir}/docs/requirements.txt +description = Build The Docs +isolated_build = true +passenv = + SSH_AUTH_SOCK +skip_install = false +usedevelop = false + + +[testenv:doctest-docs] +allowlist_externals = + {[testenv:build-docs]allowlist_externals} +basepython = {[testenv:build-docs]basepython} +commands_pre = {[testenv:build-docs]commands_pre} +commands = + # Retrieve possibly missing commits: + -git fetch --unshallow + -git fetch --tags + + # Spellcheck docs site: + python -m sphinx \ + -j auto \ + -a -n -W \ + --keep-going \ + -b doctest --color \ + -d "{toxworkdir}/docs_doctree" \ + . "{toxworkdir}/docs_out" +changedir = {[testenv:build-docs]changedir} +deps = {[testenv:build-docs]deps} +description = Doctest The Docs +isolated_build = {[testenv:build-docs]isolated_build} +passenv = {[testenv:build-docs]passenv} +skip_install = {[testenv:build-docs]skip_install} +usedevelop = {[testenv:build-docs]usedevelop} + + +[testenv:linkcheck-docs] +allowlist_externals = + {[testenv:build-docs]allowlist_externals} +basepython = {[testenv:build-docs]basepython} +commands_pre = {[testenv:build-docs]commands_pre} +commands = + # Retrieve possibly missing commits: + -git fetch --unshallow + -git fetch --tags + + # Spellcheck docs site: + python -m sphinx \ + -j auto \ + -a -n -W \ + --keep-going \ + -b linkcheck --color \ + -d "{toxworkdir}/docs_doctree" \ + . "{toxworkdir}/docs_out" +changedir = {[testenv:build-docs]changedir} +deps = {[testenv:build-docs]deps} +description = Linkcheck The Docs +isolated_build = {[testenv:build-docs]isolated_build} +passenv = {[testenv:build-docs]passenv} +skip_install = {[testenv:build-docs]skip_install} +usedevelop = {[testenv:build-docs]usedevelop} + + +[testenv:spellcheck-docs] +allowlist_externals = + {[testenv:build-docs]allowlist_externals} +basepython = {[testenv:build-docs]basepython} +commands_pre = {[testenv:build-docs]commands_pre} +commands = + # Retrieve possibly missing commits: + -git fetch --unshallow + -git fetch --tags + + # Spellcheck docs site: + python -m sphinx \ + -j auto \ + -a -n -W \ + --keep-going \ + -b spelling --color \ + -d "{toxworkdir}/docs_doctree" \ + . "{toxworkdir}/docs_out" +changedir = {[testenv:build-docs]changedir} +deps = + sphinxcontrib-spelling >= 7.2.0 + -r{toxinidir}/docs/requirements.in +description = Spellcheck The Docs +isolated_build = {[testenv:build-docs]isolated_build} +passenv = {[testenv:build-docs]passenv} +skip_install = {[testenv:build-docs]skip_install} +usedevelop = {[testenv:build-docs]usedevelop} + + [testenv:cleanup-dists] description = Wipe the the `{env:PEP517_OUT_DIR}{/}` folder @@ -95,6 +228,7 @@ deps = pre-commit pylint >= 2.5.3 pylint-pytest < 1.1.0 + -r docs/requirements.in -r requirements-tunnel.txt isolated_build = true skip_install = true