Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev plugin visibility #268

Merged
merged 46 commits into from
Sep 24, 2021
Merged
Show file tree
Hide file tree
Changes from 37 commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
d871e65
Step 1: Add reading and default behavior of visibility flag from the …
klai95 Aug 25, 2021
c7428c1
Code Review Updates
klai95 Aug 26, 2021
04cb04f
STEP 2 and STEP 3
klai95 Aug 26, 2021
4032b97
Test Error and Spacing Issue
klai95 Aug 26, 2021
3491e51
Added default behavior for white_list parameter in filter_excluded_pl…
klai95 Aug 27, 2021
801a754
Refactored code
klai95 Aug 27, 2021
7855f6d
Updated s3 file and refactored parameter name
klai95 Aug 31, 2021
3fb9c6e
Added comments and refactored read/write operations in endpoints
klai95 Sep 14, 2021
1b0f499
Added code in update_index() to update and write plugins.json and exc…
klai95 Sep 14, 2021
5f66e01
Fixed test errors
klai95 Sep 14, 2021
4e873f9
Refactored methods and wrote comments
klai95 Sep 16, 2021
d4004da
Modify test file to change version to visibility parameter
klai95 Sep 16, 2021
3bdd831
In process of fixing test error from assert len(result) == 2 due to g…
klai95 Sep 17, 2021
8883fc9
Modified test_get_plugins() to have expected test code due to changes…
klai95 Sep 17, 2021
a9927ac
Merge branch 'main' into hidden-plugin-feature
klai95 Sep 17, 2021
068a532
Some PR comment changes
klai95 Sep 20, 2021
7298de2
Merge branch 'hidden-plugin-feature' of github.com:chanzuckerberg/nap…
klai95 Sep 20, 2021
22c5c7f
Refactored comment and added todo per pr comment
klai95 Sep 20, 2021
b7ea3b8
update plugin visibility feature
Sep 22, 2021
3091cd9
correct boto3 api
Sep 22, 2021
1efdd63
correct visibility check
Sep 23, 2021
bd7ef33
correct exclusion update
Sep 23, 2021
d4bea94
ignore exclusion stats when visibility not present
Sep 23, 2021
b608781
add no value return
Sep 23, 2021
0f6199a
refactored code structure
Sep 23, 2021
dac8d76
address import structure
Sep 23, 2021
95e6bc8
address import structure
Sep 23, 2021
b885e80
address project url override
Sep 23, 2021
fbfe761
cache plugin metadata after build
Sep 23, 2021
4598376
correct github repo url
Sep 23, 2021
9d64572
reformat
Sep 23, 2021
c93be46
update documentation
Sep 23, 2021
58cbfe3
update documentation
Sep 23, 2021
87af444
fix github project url reading
Sep 23, 2021
1fad480
remove unused variable
Sep 23, 2021
4bcc6f0
update documentation
Sep 23, 2021
7eef38f
address comments
Sep 23, 2021
22f725e
use slack url
Sep 23, 2021
67a45db
update s3 to use client only
Sep 23, 2021
4f55728
update s3 to use client only
Sep 23, 2021
6b960c0
reinstate bucket path
Sep 24, 2021
fd8b531
correct s3 API
Sep 24, 2021
6247a91
use bucket path to separate data in dev environment
Sep 24, 2021
07bf5ac
also verify empty url
Sep 24, 2021
e40a2a8
add alert for exceptions
Sep 24, 2021
367fce3
address comments on citation dict
Sep 24, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/backend-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,5 @@ jobs:
pip install pytest

- name: Run tests
run : pytest
working-directory: backend
run : python -m pytest
2 changes: 1 addition & 1 deletion .happy/terraform/modules/ecs-stack/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ resource "aws_cloudwatch_event_target" "update_target" {
rule = aws_cloudwatch_event_rule.update_rule.name
arn = module.backend_lambda.function_arn
input_transformer {
input_template = jsonencode({path = "/plugins/index/update", httpMethod = "GET"})
input_template = jsonencode({path = "/update", httpMethod = "POST"})
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we refactoring the path for the sake of simplicity?"

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

originally the update is only updating the index file, however right now the update is responsible for keeping content up to date across all resources, so this change is to would reflect the actual logical level of the endpoint

}
}

Expand Down
Empty file removed backend/_tests/__init__.py
Empty file.
62 changes: 62 additions & 0 deletions backend/_tests/test_github.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import unittest
from unittest.mock import patch

from github import get_github_repo_url, get_license, get_citations
from test_utils import FakeResponse, license_response, no_license_response, citation_string


class TestGithub(unittest.TestCase):

def test_github_get_url(self):
project_urls = {"Source Code": "test1"}
assert ("test1" == get_github_repo_url(project_urls))

project_urls = {"Random": "https://random.com"}
assert (get_github_repo_url(project_urls) is None)

project_urls = {"Random": "https://github.com/org"}
assert (get_github_repo_url(project_urls) is None)

project_urls = {"Random": "https://github.com/org/repo/random"}
assert ("https://github.com/org/repo" == get_github_repo_url(project_urls))

@patch(
'requests.get', return_value=FakeResponse(data=license_response)
)
def test_github_license(self, mock_get):
result = get_license("test_website")
assert result == "BSD-3-Clause"

@patch(
'requests.get', return_value=FakeResponse(data=no_license_response)
)
def test_github_no_assertion_license(self, mock_get):
result = get_license("test_website")
assert result is None

def test_valid_citation(self):
citation = get_citations(citation_string)
assert citation['APA'] == "Fa G.N., Family G. (2019). testing (version 0.0.0). " \
"DOI: 10.0000/something.0000000 URL: https://example.com/example\n"
assert citation['BibTex'] == "@misc{YourReferenceHere,\nauthor = {Fa, Gi N. and Family, Given},\n" \
"doi = {10.0000/something.0000000},\nmonth = {11},\ntitle = {testing},\n" \
"url = {https://example.com/example},\nyear = {2019}\n}\n"
assert citation['RIS'] == "TY - GEN\nAB - Test\nAU - Fa, Gi N.\nAU - Family, Given\nDA - 2019-11-12\n" \
"DO - 10.0000/something.0000000\nKW - citation\nKW - test\nKW - cff\n" \
"KW - CITATION.cff\nPY - 2019\nTI - testing\n" \
"UR - https://example.com/example\nER\n"
assert citation['citation'] == "# YAML 1.2\n---\nabstract: \"Test\"\nauthors:\n -\n " \
"affiliation: \"Test Center\"\n family-names: Fa\n " \
"given-names: Gi N.\n orcid: https://orcid.org/0000-0000-0000-0000\n " \
"-\n affiliation: \"Test Center 2\"\n family-names: Family\n " \
"given-names: Given\ncff-version: \"1.0.3\"\ndate-released: 2019-11-12\n" \
"doi: 10.0000/something.0000000\nkeywords:\n - \"citation\"\n - \"test\"\n" \
" - \"cff\"\n - \"CITATION.cff\"\nlicense: Apache-2.0\n" \
"message: \"If you use this software, please cite it using these metadata.\"\n" \
"repository-code: \"https://example.com/example\"\ntitle: testing\n" \
"version: \"0.0.0\"\n"

def test_invalid_citation(self):
citation_str = """Ha? What is this?"""
citations = get_citations(citation_str)
assert citations['citation'] is None
49 changes: 49 additions & 0 deletions backend/_tests/test_pypi.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import unittest
from unittest.mock import patch

from requests import HTTPError

from pypi import query_pypi, get_plugin_pypi_metadata
from test_utils import FakeResponse, plugin, plugin_list


class TestPypi(unittest.TestCase):

@patch(
'requests.get', return_value=FakeResponse(data=plugin_list)
)
def test_query_pypi(self, mock_get):
result = query_pypi()
assert len(result) == 2
assert result['package1'] == "0.2.7"
assert result['package2'] == "0.1.0"

@patch(
'requests.get', return_value=FakeResponse(data=plugin)
)
def test_get_plugin_pypi_metadata(self, mock_request_get):
result = get_plugin_pypi_metadata("test", "0.0.1")
assert (result["name"] == "test")
assert (result["summary"] == "A test plugin")
assert (result["description"] == "# description [example](http://example.com)")
assert (result["description_content_type"] == "")
assert (result["authors"] == [{'email': '[email protected]', 'name': 'Test Author'}])
assert (result["license"] == "BSD-3")
assert (result["python_version"] == ">=3.6")
assert (result["operating_system"] == ['Operating System :: OS Independent'])
assert (result["release_date"] == '2020-04-13T03:37:20.169990Z')
assert (result["version"] == "0.0.1")
assert (result["first_released"] == "2020-04-13T03:37:20.169990Z")
assert (result["development_status"] == ['Development Status :: 4 - Beta'])
assert (result["requirements"] is None)
assert (result["project_site"] == "https://github.com/test/test")
assert (result["documentation"] == "")
assert (result["support"] == "")
assert (result["report_issues"] == "")
assert (result["twitter"] == "")

@patch(
'requests.get', side_effect=HTTPError()
)
def test_get_plugin_error(self, mock_get):
assert ({} == get_plugin_pypi_metadata("test", "0.0.1"))
24 changes: 24 additions & 0 deletions backend/_tests/test_shield.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import unittest
from unittest.mock import patch


@patch(
'model.get_valid_plugins', return_value={"package1": "0.0.1"}
)
class TestShield(unittest.TestCase):

def test_get_shield(self, mock_get_valid_plugins):
from shield import get_shield
result = get_shield('package1')
assert result['message'] == 'package1'
assert 'label' in result
assert 'schemaVersion' in result
assert 'color' in result

def test_get_shield_for_non_plugin(self, mock_get_valid_plugins):
from shield import get_shield
result = get_shield('not-a-package')
assert result['message'] == 'plugin not found'
assert 'label' in result
assert 'schemaVersion' in result
assert 'color' in result
144 changes: 1 addition & 143 deletions backend/_tests/test_lambda.py → backend/_tests/test_utils.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,6 @@
from unittest import mock
import requests
from requests.exceptions import HTTPError

from backend.napari import get_plugin, get_shield
from backend.napari import get_plugins
from backend.napari import get_download_url
from backend.napari import get_license
from backend.napari import get_citations


class FakeResponse:
def __init__(self, *, data: str):
Expand Down Expand Up @@ -97,86 +90,6 @@ def raise_for_status(self):
"upload_time_iso_8601":"2020-04-20T15:28:53.386281Z",
"url":"","yanked":false,"yanked_reason":null}]}}"""

@mock.patch(
'requests.get', return_value=FakeResponse(data=plugin_list)
)
def test_get_plugins(mock_get):
result = get_plugins()
assert len(result) == 2
assert result['package1'] == "0.2.7"
assert result['package2'] == "0.1.0"

@mock.patch(
'requests.get', return_value=FakeResponse(data=plugin)
)
@mock.patch(
'backend.napari.get_plugins', return_value={'test': '0.0.1'}
)
def test_get_plugin(mock_get, mock_plugins):
result = get_plugin("test")
assert(result["name"] == "test")
assert(result["summary"] == "A test plugin")
assert(result["description"] == "# description [example](http://example.com)")
assert(result["description_text"] == "description example")
assert(result["description_content_type"] == "")
assert(result["authors"] == [{'email': '[email protected]', 'name': 'Test Author'}])
assert(result["license"] == "BSD-3")
assert(result["python_version"] == ">=3.6")
assert(result["operating_system"] == ['Operating System :: OS Independent'])
assert(result["release_date"] == '2020-04-13T03:37:20.169990Z')
assert(result["version"] == "0.0.1")
assert(result["first_released"] == "2020-04-13T03:37:20.169990Z")
assert(result["development_status"] == ['Development Status :: 4 - Beta'])
assert(result["requirements"] is None)
assert(result["project_site"] == "https://github.com/test/test")
assert(result["documentation"] == "")
assert(result["support"] == "")
assert(result["report_issues"] == "")
assert(result["twitter"] == "")
assert(result["code_repository"] == "https://github.com/test/test")


@mock.patch(
'requests.get', return_value=FakeResponse(data=plugin)
)
@mock.patch(
'backend.napari.get_plugins', return_value={'not_test': '0.0.1'}
)
def test_get_invalid_plugin(mock_get, mock_plugins):
assert({} == get_plugin("test"))


@mock.patch(
'backend.napari.get_plugins', return_value=plugin_list
)
def test_get_shield(mock_plugins):
result = get_shield('package1')
assert result['message'] == 'package1'
assert 'label' in result
assert 'schemaVersion' in result
assert 'color' in result

result = get_shield('not-a-package')
assert result['message'] == 'plugin not found'
assert 'label' in result
assert 'schemaVersion' in result
assert 'color' in result


def test_github_get_url():
plugins = {"info": {"project_urls": {"Source Code": "test1"}}}
assert("test1" == get_download_url(plugins))

plugins = {"info": {"project_urls": {"Random": "https://random.com"}}}
assert(get_download_url(plugins) is None)

plugins = {"info": {"project_urls": {"Random": "https://github.com/org"}}}
assert(get_download_url(plugins) is None)

plugins = {"info": {"project_urls": {"Random": "https://github.com/org/repo/random"}}}
assert("https://github.com/org/repo" == get_download_url(plugins))


license_response = """
{
"name": "LICENSE",
Expand All @@ -190,15 +103,6 @@ def test_github_get_url():
}
"""


@mock.patch(
'requests.get', return_value=FakeResponse(data=license_response)
)
def test_github_license(mock_get):
result = get_license("test_website")
assert result == "BSD-3-Clause"


no_license_response = """
{
"name": "LICENSE",
Expand All @@ -212,17 +116,7 @@ def test_github_license(mock_get):
}
"""


@mock.patch(
'requests.get', return_value=FakeResponse(data=no_license_response)
)
def test_github_no_assertion_license(mock_get):
result = get_license("test_website")
assert result is None


def test_valid_citation():
citation_string = """# YAML 1.2
citation_string = """# YAML 1.2
---
abstract: "Test"
authors:
Expand All @@ -249,39 +143,3 @@ def test_valid_citation():
title: testing
version: "0.0.0"
"""
citation = get_citations(citation_string)
assert citation['citation'] == citation_string
assert citation['RIS'] == """TY - COMP
AU - Fa, Gi N.
AU - Family, Given
DO - 10.0000/something.0000000
KW - citation
KW - test
KW - cff
KW - CITATION.cff
M3 - software
PB - GitHub Inc.
PP - San Francisco, USA
PY - 2019/11/12
T1 - testing
UR - https://example.com/example
ER -
"""
assert citation['BibTex'] == """@misc{YourReferenceHere,
author = {
Gi N. Fa and
Given Family
},
title = {testing},
month = {11},
year = {2019},
doi = {10.0000/something.0000000},
url = {https://example.com/example}
}
"""


def test_invalid_citation():
citation_str = """Ha? What is this?"""
citations = get_citations(citation_str)
assert citations['citation'] is None
Loading