Web application scaffolding (#154)

* #131 start refactoring application generation
* #131 backend application template-based creation added
* #131 backend application with gunicorn
* #131 updated samples application with gunicorn
* #132 Logging errors fixed
* #131 samples application missing files
* #131 samples application backend + frontend
* #131 Webapp base images
* #131 Webapp application generation logic and templates
* #131 Sample application new files
* #131 Database application generation
* #132 Add cluster initialization script
* #132 Move client library
* #131 improve interactive generation
* #132 fix base images
* #132 Temporarily revert ingress proxy
* #132 Workflows refactoring
* #131 namespaced argo instance
* #132 Reduce minimum applications requests
* #132 Fix application retrieval
* #132 Reduce minimum applications requests
* #132 samples api link
* #132 cleaning useless files
* #132 Improve extract-download chmod
* #132 Improve logging
* #132 small improvement on application generation
* #154 PR fixes

applications/argo/deploy/charts/values.yaml

@@ -30,7 +30,7 @@ workflow:
     annotations: {}
     name: "argo-workflow"  # Service account which is used to run workflows
   rbac:
-    create: false  # adds Role and RoleBinding for the above specified service account to be able to run workflows
+    create: true  # adds Role and RoleBinding for the above specified service account to be able to run workflows
 
 controller:
   image:
@@ -94,10 +94,10 @@ controller:
   instanceID:
     # `instanceID.enabled` configures the controller to filter workflow submissions
     # to only those which have a matching instanceID attribute.
-    enabled: false
+    enabled: true
     # NOTE: If `instanceID.enabled` is set to `true` then either `instanceID.userReleaseName`
     # or `instanceID.explicitID` must be defined.
-    # useReleaseName: true
+    useReleaseName: true
     # explicitID: unique-argo-controller-identifier
   logging:
     level: info

applications/argo/deploy/values.yaml

@@ -1,7 +1,7 @@
 harness:
   subdomain: argo
   secured: true
-  name: argo-server
+  name: argo
   service:
     port: 2746
     auto: false

applications/common/server/common/config.py

@@ -1,5 +1,5 @@
 import os
-import logging
+from cloudharness import log
 
 from cloudharness.utils.config import CloudharnessConfig as conf
 
@@ -19,7 +19,7 @@ class Config(object):
         SENTRY_APP = conf.get_application_by_filter(name='sentry')[0].name
         SQLALCHEMY_DATABASE_URI = f'postgresql+psycopg2://{SENTRY_POSTGRES_APP.user}:{SENTRY_POSTGRES_APP.password}@{SENTRY_POSTGRES_APP.name}:{SENTRY_POSTGRES_APP.port}/{SENTRY_POSTGRES_APP.initialdb}'
     except:
-        logging.error("Cannot configure SENTRY")
+        log.error("Cannot configure SENTRY")
 
 
 

applications/samples/.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+dist

applications/samples/Dockerfile

@@ -0,0 +1,33 @@
+ARG REGISTRY
+ARG TAG=latest
+FROM ${REGISTRY}cloudharness-frontend-build:${TAG} as frontend
+
+ENV APP_DIR=/app
+
+WORKDIR ${APP_DIR}
+COPY frontend/package.json ${APP_DIR}
+COPY frontend/package-lock.json ${APP_DIR}
+RUN npm ci
+
+COPY frontend ${APP_DIR}
+RUN npm run build
+
+#####
+
+ARG REGISTRY
+ARG TAG=latest
+FROM ${REGISTRY}cloudharness-flask:${TAG}
+
+ENV MODULE_NAME=samples
+
+ENV WORKERS=2
+ENV PORT=8080
+
+COPY backend/requirements.txt /usr/src/app/
+
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+COPY backend/ /usr/src/app
+
+COPY --from=frontend app/dist/ /usr/src/app/www
+

applications/samples/README.md

@@ -1,3 +0,0 @@
-Temporary: we are using it to test the token generation.
-
-TODO move into neuroimaging

applications/samples/api/config.json

@@ -1,3 +1 @@
-{
-  "packageName": "api_samples"
-}
+{"packageName": "cloudharness_cli.samples"}

applications/samples/api/samples.yaml

@@ -40,7 +40,7 @@ paths:
       summary: test sentry is working
       operationId: error
       tags:
-        - workflows
+        - test
       responses:
         "500":
           description: Sentry entry should come!
@@ -50,7 +50,21 @@ paths:
             application/json:
               schema:
                 type: string
-
+  /ping:
+    get:
+      summary: test the application is up
+      operationId: ping
+      tags:
+        - test
+      responses:
+        "500":
+          description: This shouldn't happen
+        "200":
+          description: What we want
+          content:
+            application/json:
+              schema:
+                type: string
 
   /operation_sync:
     get:
@@ -117,7 +131,7 @@ paths:
                         type: string
                         example: my-op
 servers:
-  - url: https://samples.cloudharness.metacell.us/api
+  - url: /api
 components:
   securitySchemes:
     bearerAuth:

applications/samples/server/.gitignore → applications/samples/backend/.gitignore

@@ -64,3 +64,5 @@ target/
 
 #Ipython Notebook
 .ipynb_checkpoints
+
+www

applications/samples/server/.openapi-generator-ignore → applications/samples/backend/.openapi-generator-ignore

@@ -23,5 +23,5 @@
 #!docs/README.md
 setup.py
 */controllers/*
-*/models/*
-Dockerfile
+Dockerfile
+src/__main__.py

applications/samples/server/README.md → applications/samples/backend/README.md

@@ -15,7 +15,7 @@ To run the server, please execute the following from the root directory:
 
 ```
 pip3 install -r requirements.txt
-python3 -m api_samples
+python3 -m cloudharness_cli.samples
 ```
 
 and open your browser to here:
@@ -42,8 +42,8 @@ To run the server on a Docker container, please execute the following from the r
 
 ```bash
 # building the image
-docker build -t api_samples .
+docker build -t cloudharness_cli.samples .
 
 # starting up a container
-docker run -p 8080:8080 api_samples
+docker run -p 8080:8080 cloudharness_cli.samples
 ```

applications/samples/backend/requirements.txt

@@ -0,0 +1,10 @@
+connexion[swagger-ui] >= 2.6.0; python_version>="3.6"
+# 2.3 is the last version that supports python 3.4-3.5
+connexion[swagger-ui] <= 2.3.0; python_version=="3.5" or python_version=="3.4"
+# connexion requires werkzeug but connexion < 2.4.0 does not install werkzeug
+# we must peg werkzeug versions below to fix connexion
+# https://github.com/zalando/connexion/pull/1044
+werkzeug == 0.16.1; python_version=="3.5" or python_version=="3.4"
+swagger-ui-bundle >= 0.0.2
+python_dateutil >= 2.6.0
+setuptools >= 21.0.0

applications/samples/backend/samples/__main__.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import logging
+
+import connexion
+import flask
+from samples import encoder
+
+
+# setup connection app
+connexion_app = connexion.App(__name__, specification_dir="./openapi/", debug=True)
+app = connexion_app.app
+app.json_encoder = encoder.JSONEncoder
+
+with app.app_context():
+    # setup logging
+    gunicorn_logger = logging.getLogger("gunicorn.error")
+    app.logger.handlers = gunicorn_logger.handlers
+    app.logger.setLevel(gunicorn_logger.level)
+
+    connexion_app.add_api("openapi.yaml", arguments={"title": "samples"}, pythonic_params=True)
+    try:
+        # init_app can be defined to add behaviours to the wsgi app
+        from samples import init_app
+        init_app(app)
+    except ImportError:
+        pass
+
+@app.route('/', methods=['GET'])
+def index():
+    return flask.send_from_directory('../www', 'index.html')
+
+@app.route('/<path:path>')
+def send_webapp(path):
+    return flask.send_from_directory('../www', path)
+
+@app.route('/static/<path:path>')
+def send_static(path):
+    return flask.send_from_directory('../www/static', path)
+
+def main():
+    connexion_app.debug=True
+    connexion_app.run(port=5001)
+
+
+if __name__ == "__main__":
+    main()

applications/samples/server/api_samples/controllers/auth_controller.py → applications/samples/backend/samples/controllers/auth_controller.py

@@ -1,8 +1,8 @@
 import connexion
 import six
 
-from api_samples.models.valid import Valid  # noqa: E501
-from api_samples import util
+from samples.models.valid import Valid  # noqa: E501
+from samples import util
 
 
 def valid_token():  # noqa: E501

applications/samples/backend/samples/controllers/security_controller_.py

@@ -0,0 +1,17 @@
+from typing import List
+
+
+def info_from_bearerAuth(token):
+    """
+    Check and retrieve authentication information from custom bearer token.
+    Returned value will be passed in 'token_info' parameter of your operation function, if there is one.
+    'sub' or 'uid' will be set in 'user' parameter of your operation function, if there is one.
+
+    :param token Token provided by Authorization header
+    :type token: str
+    :return: Decoded token information or None if token is invalid
+    :rtype: dict | None
+    """
+    return {'uid': 'user_id'}
+
+

applications/samples/backend/samples/controllers/test_controller.py

@@ -0,0 +1,27 @@
+import connexion
+import six
+
+from samples import util
+
+
+def error():  # noqa: E501
+    """test sentry is working
+
+     # noqa: E501
+
+
+    :rtype: str
+    """
+    raise Exception("The error we supposed to find here")
+
+
+def ping():  # noqa: E501
+    """test the application is up
+
+     # noqa: E501
+
+
+    :rtype: str
+    """
+    import time
+    return time.time()

applications/samples/server/api_samples/controllers/workflows_controller.py → applications/samples/backend/samples/controllers/workflows_controller.py

@@ -1,18 +1,18 @@
 import connexion
 import six
 
-from api_samples.models.inline_response202 import InlineResponse202  # noqa: E501
-from api_samples import util
-from api_samples.models import InlineResponse202
-from api_samples.models.inline_response202_task import InlineResponse202Task
+from samples.models.inline_response202 import InlineResponse202  # noqa: E501
+from samples import util
+from samples.models import InlineResponse202
+from samples.models.inline_response202_task import InlineResponse202Task
 from flask.json import jsonify
 
 from cloudharness import log
 
 try:
     from cloudharness.workflows import operations, tasks
 except Exception as e:
-    log.error("Cannot start workflows module. Probably this is related some problem with the kubectl configuration", e)
+    log.error("Cannot start workflows module. Probably this is related some problem with the kubectl configuration", exc_info=True)
 
 
 def submit_async():  # noqa: E501
@@ -47,7 +47,7 @@ def submit_sync():  # noqa: E501
 
     op = operations.DistributedSyncOperation('test-sync-op-', task)
     workflow = op.execute()
-    return workflow.raw
+    return workflow.raw.to_dict()
 
 
 def submit_sync_with_results(a=1, b=2):  # noqa: E501
@@ -71,13 +71,3 @@ def submit_sync_with_results(a=1, b=2):  # noqa: E501
         return jsonify(str(e)), 200
 
 
-
-def error():  # noqa: E501
-    """test sentry is working
-
-     # noqa: E501
-
-
-    :rtype: str
-    """
-    return "a"[2]

applications/samples/server/api_samples/encoder.py → applications/samples/backend/samples/encoder.py

@@ -1,7 +1,7 @@
 from connexion.apps.flask_app import FlaskJSONEncoder
 import six
 
-from api_samples.models.base_model_ import Model
+from samples.models.base_model_ import Model
 
 
 class JSONEncoder(FlaskJSONEncoder):

applications/samples/backend/samples/models/__init__.py

@@ -0,0 +1,8 @@
+# coding: utf-8
+
+# flake8: noqa
+from __future__ import absolute_import
+# import models into model package
+from samples.models.inline_response202 import InlineResponse202
+from samples.models.inline_response202_task import InlineResponse202Task
+from samples.models.valid import Valid

applications/samples/server/api_samples/models/base_model_.py → applications/samples/backend/samples/models/base_model_.py

@@ -3,7 +3,7 @@
 import six
 import typing
 
-from api_samples import util
+from samples import util
 
 T = typing.TypeVar('T')
 

applications/samples/server/api_samples/models/inline_response202.py → applications/samples/backend/samples/models/inline_response202.py

@@ -5,11 +5,11 @@
 
 from typing import List, Dict  # noqa: F401
 
-from api_samples.models.base_model_ import Model
-from api_samples.models.inline_response202_task import InlineResponse202Task
-from api_samples import util
-
+from samples.models.base_model_ import Model
+from samples.models.inline_response202_task import InlineResponse202Task
+from samples import util
 
+from samples.models.inline_response202_task import InlineResponse202Task  # noqa: E501
 
 class InlineResponse202(Model):
     """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

applications/samples/server/api_samples/models/inline_response202_task.py → applications/samples/backend/samples/models/inline_response202_task.py

@@ -5,8 +5,8 @@
 
 from typing import List, Dict  # noqa: F401
 
-from api_samples.models.base_model_ import Model
-from api_samples import util
+from samples.models.base_model_ import Model
+from samples import util
 
 
 class InlineResponse202Task(Model):