Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update dependencies and pin elasticsearch version to <8 #106

Merged
merged 7 commits into from
Jan 5, 2024
Merged

Update dependencies and pin elasticsearch version to <8 #106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7885b91
Switch to allowlist_externals in tox configuration
felixedel Jan 5, 2024
98fa2ea
Pin elasticsearch dependencies to version <8
felixedel Jan 5, 2024
2f088b3
Update dependencies
felixedel Jan 5, 2024
45524d1
Fix Sphinx tests
felixedel Jan 5, 2024
7608380
Improve custom Zubbi directives
felixedel Jan 5, 2024
010a0a1
Fix Elasticsearch tests
felixedel Jan 5, 2024
4b649b3
Reformat files with newer black version
felixedel Jan 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,440 changes: 820 additions & 620 deletions Pipfile.lock
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@
requires = [
"arrow",
"click",
"elasticsearch-dsl>=7.0.0",
"elasticsearch<8.0.0",
"elasticsearch-dsl>=7.0.0,<8.0.0",
"pyyaml",
"pyjwt",
"pyzmq",
Expand Down
145 changes: 70 additions & 75 deletions test-requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,80 +1,75 @@
-i https://pypi.python.org/simple
attrs==22.1.0 ; python_version >= '3.5'
certifi==2022.9.24 ; python_version >= '3.6'
cfgv==3.3.1 ; python_full_version >= '3.6.1'
charset-normalizer==2.1.1 ; python_full_version >= '3.6.0'
coverage[toml]==6.5.0 ; python_version >= '3.7'
distlib==0.3.6
exceptiongroup==1.0.4 ; python_version < '3.11'
filelock==3.8.0 ; python_version >= '3.7'
flake8==5.0.4
flake8-docstrings==1.6.0
flake8-import-order==0.18.1
freezegun==1.2.2
identify==2.5.9 ; python_version >= '3.7'
idna==3.4 ; python_version >= '3.5'
iniconfig==1.1.1
mccabe==0.7.0 ; python_version >= '3.6'
nodeenv==1.7.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
packaging==21.3 ; python_version >= '3.6'
platformdirs==2.5.4 ; python_version >= '3.7'
pluggy==1.0.0 ; python_version >= '3.6'
pre-commit==2.20.0
pycodestyle==2.9.1 ; python_version >= '3.6'
pydocstyle==6.1.1 ; python_version >= '3.6'
pyflakes==2.5.0 ; python_version >= '3.6'
pyparsing==3.0.9 ; python_full_version >= '3.6.8'
pytest==7.2.0
pytest-cov==4.0.0
python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
pyyaml==6.0 ; python_version >= '3.6'
requests==2.28.1 ; python_version >= '3.7' and python_version < '4'
requests-mock==1.10.0
setuptools==65.6.0 ; python_version >= '3.7'
six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
certifi==2023.11.17; python_version >= '3.6'
cfgv==3.4.0; python_version >= '3.8'
charset-normalizer==3.3.2; python_full_version >= '3.7.0'
coverage[toml]==7.4.0; python_version >= '3.8'
distlib==0.3.8
filelock==3.13.1; python_version >= '3.8'
flake8==7.0.0; python_full_version >= '3.8.1'
flake8-docstrings==1.7.0; python_version >= '3.7'
flake8-import-order==0.18.2
freezegun==1.4.0; python_version >= '3.7'
identify==2.5.33; python_version >= '3.8'
idna==3.6; python_version >= '3.5'
iniconfig==2.0.0; python_version >= '3.7'
mccabe==0.7.0; python_version >= '3.6'
nodeenv==1.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
packaging==23.2; python_version >= '3.7'
platformdirs==4.1.0; python_version >= '3.8'
pluggy==1.3.0; python_version >= '3.8'
pre-commit==3.6.0; python_version >= '3.9'
pycodestyle==2.11.1; python_version >= '3.8'
pydocstyle==6.3.0; python_version >= '3.6'
pyflakes==3.2.0; python_version >= '3.8'
pytest==7.4.4; python_version >= '3.7'
pytest-cov==4.1.0; python_version >= '3.7'
python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
pyyaml==6.0.1; python_version >= '3.6'
requests==2.31.0; python_version >= '3.7'
requests-mock==1.11.0
setuptools==69.0.3; python_version >= '3.8'
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
snowballstemmer==2.2.0
toml==0.10.2 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
tomli==2.0.1 ; python_version < '3.11'
urllib3==1.26.12 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'
virtualenv==20.16.7 ; python_version >= '3.6'
alabaster==0.7.12
arrow==1.2.3 ; python_version >= '3.6'
babel==2.11.0 ; python_version >= '3.6'
bleach==5.0.1 ; python_version >= '3.7'
cachelib==0.9.0 ; python_version >= '3.7'
cffi==1.15.1
click==8.1.3 ; python_version >= '3.7'
urllib3==1.26.18; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
virtualenv==20.25.0; python_version >= '3.7'
alabaster==0.7.13; python_version >= '3.6'
arrow==1.3.0; python_version >= '3.8'
babel==2.14.0; python_version >= '3.7'
blinker==1.7.0; python_version >= '3.8'
cachelib==0.10.2; python_version >= '3.7'
cffi==1.16.0; python_version >= '3.8'
click==8.1.7; python_version >= '3.7'
cmarkgfm==2022.10.27
cryptography==38.0.3 ; python_version >= '3.6'
docutils==0.19 ; python_version >= '3.7'
elasticsearch==7.17.7 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'
elasticsearch-dsl==7.4.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
flask==2.2.2 ; python_version >= '3.7'
gitdb==4.0.9 ; python_version >= '3.6'
github3.py==3.2.0 ; python_version >= '3.6'
gitpython==3.1.29 ; python_version >= '3.7'
imagesize==1.4.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
itsdangerous==2.1.2 ; python_version >= '3.7'
jinja2==3.1.2 ; python_version >= '3.7'
markupsafe==2.1.1 ; python_version >= '3.7'
pbr==5.11.0 ; python_version >= '2.6'
cryptography==41.0.7; python_version >= '3.7'
docutils==0.20.1; python_version >= '3.7'
elasticsearch==7.17.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'
elasticsearch-dsl==7.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
flask==3.0.0; python_version >= '3.8'
gitdb==4.0.11; python_version >= '3.7'
github3.py==4.0.1; python_version >= '3.7'
gitpython==3.1.40; python_version >= '3.7'
imagesize==1.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
itsdangerous==2.1.2; python_version >= '3.7'
jinja2==3.1.2; python_version >= '3.7'
markupsafe==2.1.3; python_version >= '3.7'
nh3==0.2.15
pbr==6.0.0; python_version >= '2.6'
pycparser==2.21
pygments==2.13.0 ; python_version >= '3.6'
pyjwt==2.6.0 ; python_version >= '3.7'
pytz==2022.6
pyzmq==24.0.1 ; python_version >= '3.6'
readme-renderer[md]==37.3 ; python_version >= '3.7'
smmap==5.0.0 ; python_version >= '3.6'
sphinx==5.3.0 ; python_version >= '3.6'
sphinxcontrib-applehelp==1.0.2 ; python_version >= '3.5'
sphinxcontrib-devhelp==1.0.2 ; python_version >= '3.5'
sphinxcontrib-htmlhelp==2.0.0 ; python_version >= '3.6'
sphinxcontrib-jsmath==1.0.1 ; python_version >= '3.5'
sphinxcontrib-qthelp==1.0.3 ; python_version >= '3.5'
sphinxcontrib-serializinghtml==1.1.5 ; python_version >= '3.5'
tabulate==0.9.0 ; python_version >= '3.7'
uritemplate==4.1.1 ; python_version >= '3.6'
webencodings==0.5.1
werkzeug==2.2.2 ; python_version >= '3.7'
pygments==2.17.2; python_version >= '3.7'
pyjwt[crypto]==2.8.0; python_version >= '3.7'
pyzmq==25.1.2; python_version >= '3.6'
readme-renderer[md]==42.0; python_version >= '3.8'
smmap==5.0.1; python_version >= '3.7'
sphinx==7.2.6; python_version >= '3.9'
sphinxcontrib-applehelp==1.0.7; python_version >= '3.9'
sphinxcontrib-devhelp==1.0.5; python_version >= '3.9'
sphinxcontrib-htmlhelp==2.0.4; python_version >= '3.9'
sphinxcontrib-jsmath==1.0.1; python_version >= '3.5'
sphinxcontrib-qthelp==1.0.6; python_version >= '3.9'
sphinxcontrib-serializinghtml==1.1.9; python_version >= '3.9'
tabulate==0.9.0; python_version >= '3.7'
types-python-dateutil==2.8.19.14
uritemplate==4.1.1; python_version >= '3.6'
werkzeug==3.0.1; python_version >= '3.8'
-e .
zuul-sphinx==0.6.0
zuul-sphinx==0.7.0
1 change: 0 additions & 1 deletion tests/scraper/connections/test_github.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ def test_get_app_auth_headers():


def test_get_installation_key(mock_github_api_endpoints):

mock_github_api_endpoints(GITHUB_URL)
# Initialize GitHubConnection
gh_con = GitHubConnection(**GITHUB_CON_CONFIG)
Expand Down
1 change: 0 additions & 1 deletion tests/scraper/connections/test_init.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ def test_init_gerrit_con_invalid_webtype(patch_es):


def test_init_github_con(patch_es, mock_github_api_endpoints):

mock_github_api_endpoints("https://localhost/github")
config = {
"ES_HOST": "localhost",
Expand Down
1 change: 0 additions & 1 deletion tests/scraper/test_events.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ def test_event_installation_created(
def test_event_installation_deleted(
scrape_mock, patched_connections, payload_webhook_installation_deleted
):

# Ensure that some repositories can be looked up for this installation as they
# are not part of the payload for a delete event.
with mock.patch(
Expand Down
2 changes: 0 additions & 2 deletions tests/scraper/test_integration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@


class MockContents:

FILE = "file"
DIR = "dir"

Expand All @@ -78,7 +77,6 @@ def __init__(self, path, type):


class MockGitHubRepository(GitHubRepository):

# TODO (felix): Improve the mocked repository class and allow mocking
# single repositories for certain test cases. Currently, all test repos
# are defined in this class definition.
Expand Down
6 changes: 3 additions & 3 deletions tests/scraper/test_repo_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ def test_parse(repo_data):
'<span class="pre">mandatory_variable</span>'
"</span>"
'<a class="headerlink" href="#rolevar-mandatory_variable" '
'title="Permalink to this definition">'
'title="Link to this definition">'
"¶</a><br /></dt>\n"
"<dd><p>This variable is mandatory.</p>\n"
"</dd></dl>\n"
Expand All @@ -185,7 +185,7 @@ def test_parse(repo_data):
'<span class="pre">optional_variable</span>'
"</span>"
'<a class="headerlink" href="#rolevar-optional_variable" '
'title="Permalink to this definition">'
'title="Link to this definition">'
'¶</a><br /><span class="pre">Default:</span> '
'<code class="docutils literal notranslate"><span class="pre">'
"some_value</span></code><br /></dt>\n"
Expand All @@ -198,7 +198,7 @@ def test_parse(repo_data):
'<span class="pre">list_variable</span>'
"</span>"
'<a class="headerlink" href="#rolevar-list_variable" '
'title="Permalink to this definition">¶</a><br />'
'title="Link to this definition">¶</a><br />'
'<span class="pre">Default:</span> '
'<code class="docutils literal notranslate"><span class="pre">[]</span>'
'</code><br /><span class="pre">Type:</span> <em><span class="pre">'
Expand Down
8 changes: 4 additions & 4 deletions tests/test_doc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,16 +59,16 @@ def test_render_sphinx_yaml_code(readme_yaml_code):
'<div class="highlight-yaml notranslate"><div class="highlight">'
'<pre><span></span><span class="p p-Indicator">-</span>'
'<span class="w"> </span><span class="nt">job</span>'
'<span class="p">:</span><span class="w"></span>\n'
'<span class="p">:</span>\n'
'<span class="w"> </span><span class="nt">name</span>'
'<span class="p">:</span><span class="w"> </span>'
'<span class="l l-Scalar l-Scalar-Plain">foo</span><span class="w"></span>\n'
'<span class="l l-Scalar l-Scalar-Plain">foo</span>\n'
'<span class="w"> </span><span class="nt">parent</span>'
'<span class="p">:</span><span class="w"> </span>'
'<span class="l l-Scalar l-Scalar-Plain">bar</span><span class="w"></span>\n'
'<span class="l l-Scalar l-Scalar-Plain">bar</span>\n'
'<span class="w"> </span><span class="nt">playbook</span>'
'<span class="p">:</span><span class="w"> </span>'
'<span class="l l-Scalar l-Scalar-Plain">foo-bar.yaml</span><span class="w"></span>\n'
'<span class="l l-Scalar l-Scalar-Plain">foo-bar.yaml</span>\n'
"</pre></div>\n"
"</div>\n"
)
Expand Down
4 changes: 2 additions & 2 deletions tests/test_elasticsearch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ def test_elasticsearch_write(elmock):
zt.save()

assert elmock.return_value.index.call_args == mock.call(
index="zuul-tenants", body=zt.to_dict()
index="zuul-tenants", document=zt.to_dict()
)


Expand All @@ -207,5 +207,5 @@ def test_elasticsearch_write_with_prefix(elmock):
zt.save()

assert elmock.return_value.index.call_args == mock.call(
index="zubbi-zuul-tenants", body=zt.to_dict()
index="zubbi-zuul-tenants", document=zt.to_dict()
)
1 change: 0 additions & 1 deletion tests/test_views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ def test_views_reachable(flask_client, endpoint, expected):


def test_detail_view(flask_client, es_client):

# Build a simple ES response, containing a minimal role result
response = {
"hits": {
Expand Down
6 changes: 3 additions & 3 deletions tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ commands =
python setup.py sdist bdist_wheel

[testenv:update-requirements]
whitelist_externals =
allowlist_externals =
bash
deps =
pipenv
Expand All @@ -35,8 +35,8 @@ commands =
bash -c 'pipenv requirements --dev > "{toxinidir}/test-requirements.txt"'

[testenv:pipenv]
whitelist_externals =
{[testenv:update-requirements]whitelist_externals}
allowlist_externals =
{[testenv:update-requirements]allowlist_externals}
deps =
{[testenv:update-requirements]deps}
commands =
Expand Down
52 changes: 31 additions & 21 deletions zubbi/doc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,42 +20,57 @@
import sys
import tempfile

from docutils.parsers.rst import Directive
from readme_renderer import markdown
from sphinx.application import Sphinx
from sphinx.util import logging as sphinx_logging
from sphinx.util.console import nocolor
from sphinx.util.docutils import docutils_namespace, patch_docutils
from sphinx.util.docutils import docutils_namespace, patch_docutils, SphinxDirective


LOGGER = logging.getLogger(__name__)


class ZubbiDirective(Directive):
class ZubbiDirective(SphinxDirective):
has_content = True
required_arguments = 0
optional_arguments = 1
final_argument_whitespace = True
logger = sphinx_logging.getLogger("ZubbiDirective")

def run(self):

class SupportedOS(ZubbiDirective):
directive_name = "supported_os"

def run(self):
if len(self.arguments) > 0:
values = self.arguments[0].split(",")
for value in values:
# Log the value with directive_name for later extraction
self.logger.info("%s: %s", self.directive_name, value)
# Store the platforms in Sphinx' domain data, so we can extract
# them later on during the rendering process.
# NOTE (felix): The "correct" solution might be to define a own
# domain and let the domain create the initial domaindata, e.g.
# https://opendev.org/zuul/zuul-sphinx/src/branch/master/zuul_sphinx/zuul.py#L714
# However, as a simple solution, this should be sufficient.
zubbi_domain_data = self.env.domaindata.setdefault("zubbi", {})
zubbi_domain_data["platforms"] = [v.strip().lower() for v in values]
# We don't want to render anything, so we return an empty list of nodes
return []


class SupportedOS(ZubbiDirective):
directive_name = "supported_os"


class Reusable(ZubbiDirective):
directive_name = "reusable"

def run(self):
reusable = False
if len(self.arguments) > 0:
reusable = self.arguments[0].strip().lower() in ["true", "yes"]
# Store the platforms in Sphinx' domain data, so we can extract
# them later on during the rendering process.
zubbi_domain_data = self.env.domaindata.setdefault("zubbi", {})
zubbi_domain_data["reusable"] = reusable

# We don't want to render anything, so we return an empty list of nodes
return []


class SphinxBuildError(RuntimeError):
pass
Expand Down Expand Up @@ -103,16 +118,11 @@ def render_sphinx(content):
if app.statuscode:
raise SphinxBuildError

# Extract the platforms from the logger output
platforms = []
reusable = False
status_log.seek(0)
for line in status_log.readlines():
prefix, _, value = line.partition(":")
if prefix == SupportedOS.directive_name:
platforms.append(value.strip().lower())
elif prefix == Reusable.directive_name:
reusable = value.strip().lower() in ["true", "yes"]
# Extract the data from our custom directives from the domain data
zubbi_domain_data = app.env.domaindata.get("zubbi", {})
platforms = zubbi_domain_data.get("platforms", [])
reusable = zubbi_domain_data.get("reusable", False)

with build_path.open() as build:
html_parts = json.load(build)

Expand Down
1 change: 0 additions & 1 deletion zubbi/scraper/connections/gerrit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ def build_directory_url(self, repo_name, directory_path):


class GerritConnection(GitConnection):

WEB_URL_BUILDERS = {"cgit": CGitUrlBuilder, "gitweb": GitwebUrlBuilder}

# TODO (felix): Should we ensure that only the user that started zubbi has access rights
Expand Down
1 change: 0 additions & 1 deletion zubbi/scraper/connections/github.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@

class GitHubConnection:
def __init__(self, url, app_id, app_key):

self.base_url = url
self.api_url = urljoin(url, "api/v3")
self.graphql_url = urljoin(url, "api/graphql")
Expand Down
Loading