Merge pull request #272 from arXiv/develop

Pre-release merge for search v0.5.6, search-api v0.2

.coveragerc

@@ -1,13 +1,20 @@
 [run]
 omit =
+    api.py
     app.py
+    audit.py
+    classic_api.py
     setup.py
     docs/*
-    test*
+    *test*
     wsgi.py
+    wsgi-api.py
+    wsgi-classic-api.py
+    wsgi-app.py
     populate_test_metadata.py
     upload_static_assets.py
     create_index.py
+    reindex.py
     bulk_index.py
     shard_ids_for_index.py
     search/config.py

.gitignore

@@ -101,6 +101,9 @@ ENV/
 .vscode
 settings.json
 
+# PyCharm
+.idea
+
 # mypy
 .mypy_cache/
 
@@ -110,3 +113,4 @@ temp/
 
 to_index/
 
+.pytest_cache/

.pre-commit-config.yaml

@@ -0,0 +1,35 @@
+- repo: https://github.com/ambv/black
+  rev: stable
+  hooks:
+    - id: black
+      name: Format Python Code
+      language: python
+      entry: black
+      args:
+        - --safe
+        - --line-length=79
+        - --target-version=py37
+        - .
+
+- repo: https://github.com/PyCQA/flake8
+  rev: 3.7.9
+  hooks:
+    - id: flake8
+      name: Flake8 Check
+      language: python
+      entry: flake8
+      args:
+        - search
+        - tests
+
+- repo: https://github.com/pycqa/pydocstyle
+  rev: master
+  hooks:
+    - id: pydocstyle
+      name: Python Documentation Style Check
+      language: python
+      entry: pydocstyle
+      args:
+        - search
+        - tests
+        - --add-ignore=D401,D202

.pylintrc

@@ -3,7 +3,8 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code
-extension-pkg-whitelist=
+extension-pkg-whitelist=lxml.etree
+# lxml.etree is unfortunately a dependency of feedgen, and isn't great with pylint
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.

.travis.yml

@@ -11,13 +11,12 @@ python:
   - "3.6"
 script:
   - pip install pipenv
-  - pipenv install
-  - pipenv install --dev --skip-lock
+  - pipenv sync --dev 
+  - pipenv run nose2 -vvv tests.base_app_tests
   - pipenv run nose2 -vvv --with-coverage
+  - "./lintstats.sh"
 after_success:
   - coveralls
-  - pipenv install pylint pydocstyle mypy
-  - ./lintstats.sh
   - docker login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_PASSWORD"
   - docker build . -t arxiv/search:${TRAVIS_COMMIT};
     docker push arxiv/search:${TRAVIS_COMMIT}

Dockerfile

@@ -37,7 +37,7 @@ ENV FLASK_APP /opt/arxiv/app.py
 
 ENV ELASTICSEARCH_SERVICE_HOST 127.0.0.1
 ENV ELASTICSEARCH_SERVICE_PORT 9200
-ENV ELASTICSEARCH_PORT_9200_PROTO http
+ENV ELASTICSEARCH_SERVICE_PORT_9200_PROTO http
 ENV ELASTICSEARCH_PASSWORD changeme
 ENV METADATA_ENDPOINT https://arxiv.org/docmeta_bulk/
 

Dockerfile-agent

@@ -4,13 +4,13 @@
 #  article metadata becomes available. Subscribes to a Kinesis stream for
 #  notifications about new metadata.
 
-FROM arxiv/search:0.5.1
+FROM arxiv/search:0.5.6
 
 WORKDIR /opt/arxiv
 
 ENV ELASTICSEARCH_SERVICE_HOST 127.0.0.1
 ENV ELASTICSEARCH_SERVICE_PORT 9200
-ENV ELASTICSEARCH_PORT_9200_PROTO http
+ENV ELASTICSEARCH_SERVICE_PORT_9200_PROTO http
 ENV ELASTICSEARCH_INDEX arxiv
 ENV ELASTICSEARCH_USER elastic
 ENV ELASTICSEARCH_PASSWORD changeme

Dockerfile-api

@@ -32,7 +32,7 @@ ENV FLASK_APP /opt/arxiv/app.py
 
 ENV ELASTICSEARCH_SERVICE_HOST 127.0.0.1
 ENV ELASTICSEARCH_SERVICE_PORT 9200
-ENV ELASTICSEARCH_PORT_9200_PROTO http
+ENV ELASTICSEARCH_SERVICE_PORT_9200_PROTO http
 ENV ELASTICSEARCH_INDEX arxiv
 ENV ELASTICSEARCH_USER elastic
 ENV ELASTICSEARCH_PASSWORD changeme

Dockerfile-index

@@ -11,7 +11,7 @@
 #    $ cp arxiv_id_dump.txt /tmp/to_index
 #    $ docker run -it --network=arxivsearch_es_stack \
 #    >   -v /tmp/to_index:/to_index \
-#    >   -e ELASTICSEARCH_HOST=elasticsearch \
+#    >   -e ELASTICSEARCH_SERVICE_HOST=elasticsearch \
 #    >   arxiv/search-index /to_index/arxiv_id_dump.txt
 #
 # See also ELASTICSEARCH_* and METADATA_ENDPOINT parameters, below.
@@ -31,7 +31,7 @@ ENV FLASK_APP /opt/arxiv/app.py
 
 ENV ELASTICSEARCH_SERVICE_HOST 127.0.0.1
 ENV ELASTICSEARCH_SERVICE_PORT 9200
-ENV ELASTICSEARCH_PORT_9200_PROTO http
+ENV ELASTICSEARCH_SERVICE_PORT_9200_PROTO http
 ENV ELASTICSEARCH_USER elastic
 ENV ELASTICSEARCH_PASSWORD changeme
 ENV METADATA_ENDPOINT https://arxiv.org/docmeta_bulk/

Pipfile

@@ -5,7 +5,7 @@ name = "pypi"
 
 [packages]
 arxiv-auth = "==0.2.7"
-arxiv-base = "==0.16.4"
+arxiv-base = "==0.16.6"
 boto = "==2.48.0"
 "boto3" = "==1.6.6"
 botocore = "==1.9.6"
@@ -15,20 +15,22 @@ click = "==6.7"
 coverage = "==4.4.2"
 dataclasses = "==0.4"
 docutils = "==0.14"
-elasticsearch = ">=6.0.0,<7.0.0"
-elasticsearch-dsl = ">=6.0.0,<7.0.0"
-flask = "*"
+elasticsearch = "==6.3.0"
+elasticsearch-dsl = "==6.4.0"
+feedgen = "==0.9.0"
+flask = "==1.0.2"
 "flask-s3" = "==0.3.3"
 idna = "==2.6"
 ipaddress = "==1.0.19"
 itsdangerous = "==0.24"
 "jinja2" = ">=2.10.1"
 jmespath = "==0.9.3"
 jsonschema = "==2.6.0"
+lark-parser = "==0.8.1"
 markupsafe = "==1.0"
 mccabe = "==0.6.1"
 mock = "==2.0.0"
-mypy = "==0.670"
+mypy = "==0.720"
 "nose2" = "==0.7.3"
 pbr = "==3.1.1"
 psutil = "==5.4.3"
@@ -44,7 +46,7 @@ snowballstemmer = "==1.2.1"
 thrift = "==0.11.0"
 thrift-connector = "==0.23"
 "urllib3" = ">=1.23"
-werkzeug = "*"
+werkzeug = "~=0.14"
 wtforms = "==2.1"
 bleach = "*"
 lxml = "*"
@@ -57,3 +59,10 @@ sphinx = "*"
 sphinxcontrib-websupport = "*"
 sphinx-autodoc-typehints = "*"
 pylint = "*"
+pytest = "*"
+nose = "*"
+mypy = "==0.720"
+pre-commit = "==2.0.1"
+
+[requires]
+python_version = "3.6"