diff --git a/.pylintrc b/.pylintrc index d4d7d62d6..a99c2cf65 100644 --- a/.pylintrc +++ b/.pylintrc @@ -98,6 +98,7 @@ disable=C0111,missing-docstring, R0902,too-many-instance-attributes, R0904,too-many-public-methods, R0912,too-many-branches, + R0913,too-many-arguments, R0914,too-many-locals, R0915,too-many-statements, R0917,too-many-positional-arguments, diff --git a/CHANGES.rst b/CHANGES.rst index aa9d53b3a..eeae25142 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,6 +12,25 @@ Changes Changes: -------- +- Add support of *OGC API - Processes - Part 4: Job Management* endpoints for `Job` creation and execution + (fixes `#716 `_). +- Add ``headers``, ``mode`` and ``response`` parameters along the ``inputs`` and ``outputs`` returned by + the ``GET /jobs/{jobID}/inputs`` endpoint to better describe the expected resolution strategy of the + multiple `Job` execution options according to submitted request parameters. +- Increase flexible auto-resolution of *synchronous* vs *asynchronous* `Job` execution when no explicit strategy + is specified by ``mode`` body parameter or ``Prefer`` header. Situations where such flexible resolution can occur + will be reflected by a ``mode: auto`` and the absence of ``wait``/``respond-async`` in the ``Prefer`` header + within the response of the ``GET /jobs/{jobID}/inputs`` endpoint. +- Add support "on-trigger" `Job` submission using the ``status: create`` request body parameter. + Such a `Job` will be pending, and can be modified by ``PATCH /jobs/{jobID}`` requests, until execution is triggered + by a subsequent ``POST /jobs/{jobID}/results`` request. +- Align ``GET /jobs/{jobID}/outputs`` with requirements of *OGC API - Processes - Part 4: Job Management* endpoints + such that omitting the ``schema`` query parameter will automatically apply the `OGC` mapping representation by + default. Previous behavior was to return whichever representation that was used by the internal `Process` interface. +- Align `Job` status and update operations with some of the `openEO` behaviors, such as supporting a `Job` ``title`` + and allowing ``status`` to return `openEO` values when using ``profile=openeo`` in the ``Content-Type`` or using + the query parameter ``profile``/``schema``. The ``Content-Schema`` will also reflect the resolved representation + in the `Job` status response. - Add support of ``response: raw`` execution request body parameter as alternative to ``response: document``, which allows directly returning the result contents or ``Link`` headers rather then embedding them in a `JSON` response (fixes `#376 `_). diff --git a/docs/examples/job_status_ogcapi.json b/docs/examples/job_status_ogcapi.json new file mode 100644 index 000000000..a6b03157d --- /dev/null +++ b/docs/examples/job_status_ogcapi.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/statusInfo.yaml", + "jobID": "a305ef3e-3220-4d43-b1be-301f5ef13c23", + "processID": "example-process", + "providerID": null, + "type": "process", + "status": "succeeded", + "message": "Job succeeded.", + "created": "2024-10-02T14:21:12.380000+00:00", + "started": "2024-10-02T14:21:12.990000+00:00", + "finished": "2024-10-02T14:21:23.629000+00:00", + "updated": "2024-10-02T14:21:23.630000+00:00", + "duration": "00:00:10", + "runningDuration": "PT11S", + "runningSeconds": 10.639, + "percentCompleted": 100, + "progress": 100, + "links": [ + { + "title": "Job status.", + "hreflang": "en-CA", + "href": "https://example.com/processes/download-band-sentinel2-product-safe/jobs/a305ef3e-3220-4d43-b1be-301f5ef13c23", + "type": "application/json", + "rel": "status" + }, + { + "title": "Job monitoring location.", + "hreflang": "en-CA", + "href": "https://example.com/processes/download-band-sentinel2-product-safe/jobs/a305ef3e-3220-4d43-b1be-301f5ef13c23", + "type": "application/json", + "rel": "monitor" + }, + { + "title": "Job results of successful process execution (direct output values mapping).", + "hreflang": "en-CA", + "href": "https://example.com/processes/download-band-sentinel2-product-safe/jobs/a305ef3e-3220-4d43-b1be-301f5ef13c23/results", + "type": "application/json", + "rel": "http://www.opengis.net/def/rel/ogc/1.0/results" + } + ] +} diff --git a/docs/examples/job_status_wps.xml b/docs/examples/job_status_wps.xml new file mode 100644 index 000000000..be1a0d38a --- /dev/null +++ b/docs/examples/job_status_wps.xml @@ -0,0 +1,29 @@ + + + + example-process + + + Package operations complete. + + + + output + output + + + + diff --git a/docs/source/package.rst b/docs/source/package.rst index a41ace285..77557094d 100644 --- a/docs/source/package.rst +++ b/docs/source/package.rst @@ -194,7 +194,7 @@ When the above code is saved in a |jupyter-notebook|_ and committed to a Git rep utility can automatically clone the repository, parse the Python code, extract the :term:`CWL` annotations, and generate the :term:`Application Package` with a :term:`Docker` container containing all of their respective definitions. All of this is accomplished with a single call to obtain a deployable :term:`CWL` in `Weaver`, which can then take over -from the :ref:`Process Deployment ` to obtain an :term:`OGC API - Process` definition. +from the :ref:`Process Deployment ` to obtain an :term:`OGC API - Processes` definition. Jupyter Notebook to CWL Example: NCML to STAC Application ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1408,7 +1408,7 @@ However, the :term:`Vault` approach as potential drawbacks. .. note:: For more details about the :term:`Vault`, refer to sections :ref:`file_vault_inputs`, :ref:`vault_upload`, - and the corresponding capabilities in :term:`cli_example_upload`. + and the corresponding capabilities in :ref:`cli_example_upload`. .. _app_pkg_secret_cwltool: diff --git a/docs/source/processes.rst b/docs/source/processes.rst index 3d204ef04..98d2037ba 100644 --- a/docs/source/processes.rst +++ b/docs/source/processes.rst @@ -613,6 +613,11 @@ Execution of a Process (Execute) For backward compatibility, the |exec-req-job|_ request is also supported as alias to the above :term:`OGC API - Processes` compliant endpoint. +.. seealso:: + Alternatively, the |job-exec-req|_ request can also be used to submit a :term:`Job` for later execution, + as well as enabling other advanced :ref:`proc_job_management` capabilities. + See :ref:`proc_op_job_create` for more details. + This section will first describe the basics of this request format (:ref:`proc_exec_body`), and after go into further details for specific use cases and parametrization of various input/output combinations (:ref:`proc_exec_mode`, :ref:`proc_exec_results`, etc.). @@ -1694,7 +1699,7 @@ the ``POST /search`` or the ``POST /collections/dataset-features/search`` could Alternatively, if an array of ``image/tiff; application=geotiff`` was expected by the :term:`Process` while targeting the ``collection`` on a :term:`STAC` server, the |stac-assets|_ matching the requested :term:`Media-Types` could -potentially be retrieved as input for the :term:`Process Execution `. +potentially be retrieved as input for the :ref:`Process Execution `. In summary, the |ogc-api-proc-part3-collection-input|_ offers a lot of flexibility with its resolution compared to the typical :ref:`Input Types ` (i.e.: ``Literal``, ``BoundingBox``, ``Complex``) that must be explicitly @@ -1997,17 +2002,93 @@ of the polling-based method on the :ref:`Job Status ` endpoint o .. seealso:: Refer to the |oas-rtd|_ of the |exec-req|_ request for all available ``subscribers`` properties. +.. _proc_job_management: + +Job Management +================================================== + +This section presents capabilities related to :term:`Job` management. +The endpoints and related operations are defined in a mixture of |ogc-api-proc|_ *Core* requirements, +some official extensions, and further `Weaver`-specific capabilities. + +.. seealso:: + - |ogc-api-proc-part1-spec-html|_ + - |ogc-api-proc-part4|_ + +.. _proc_op_job_create: + +Submitting a Job Creation +--------------------------------------------------------------------- + +.. important:: + All concepts introduced in the :ref:`Execution of a Process ` also apply in this section. + Consider reading the subsections for more specific details. + + This section will only cover *additional* concepts and parameters applicable only for this feature. + +Rather than using the |exec-req|_ request, the |job-exec-req|_ request can be used to submit a :term:`Job`. +When doing so, all parameters typically required for :term:`Process` execution must also be provided, including +any relevant :ref:`proc_exec_body` contents (:term:`I/O`), the desired :ref:`proc_exec_mode`, and +the :ref:`proc_exec_results` options. However, an *additional* ``process`` :term:`URL` in the request body is required, +to indicate which :term:`Process` should be executed by the :term:`Job`. + +The |job-exec-req|_ operation allows interoperability alignement with other execution strategies, such as defined +by the |openeo-api|_ and the |ogc-tb20-gdc|_ *GDC API Profile*. It also opens the door for advanced :term:`Workflow` +definitions from a common :term:`Job` endpoint interface, as described by the |ogc-api-proc-part4|_ extension. + +Furthermore, an optional ``"status": "create"`` request body parameter can be supplied to indicate to the :term:`Job` +that it should remain in *pending* state, until a later :ref:`Job Execution Trigger ` is performed +to start its execution. This allows the user to apply any desired :ref:`Job Updates ` or reviewing +the resolved :ref:`proc_op_job_inputs` prior to submitting the :term:`Job`. This acts in contrast to +the *Core* |exec-req|_ operation that *immediately* places the :term:`Job` in queue, locking it from any update. + +.. _proc_op_job_update: + +Updating a Job +--------------------------------------------------------------------- + +The |job-update-req|_ request allows updating the :term:`Job` and its underlying parameters prior to execution. +For this reason, it has for pre-requirement to be in ``created`` :ref:`Job Status `, such that +it is pending a :ref:`Job Execution Trigger ` before being sent to the worker execution queue. +For any other ``status`` than ``created``, attempts to modify the :term:`Job` will return an *HTTP 423 Locked* error +response. + +Potential parameters that can be updated are: + +- Submitted :term:`Process` ``inputs`` +- Desired ``outputs`` formats and representations, as per :ref:`proc_exec_results` +- Applicable ``headers``, ``response`` and ``mode`` options as per :ref:`proc_exec_mode` +- Additional metadata such as a custom :term:`Job` ``title`` + +After updating the :term:`Job`, the :ref:`Job Status ` and :ref:`Job Inputs ` +operations can further be performed to review the *pending* :term:`Job` state. Using all those operations allows the +user to iteratively adjust the :term:`Job` until it is ready for execution, for which +the :ref:`Job Execution Trigger ` would then be employed. + +.. _proc_op_job_trigger: + +Triggering Job Execution +--------------------------------------------------------------------- + +The |job-trigger-req|_ request allows submitting a *pending* :term:`Job` to the worker execution queue. Once performed, +the typical :ref:`proc_op_monitor` operation can be employed, until eventual success or failure of the :term:`Job`. + +If the :term:`Job` was already submitted, is already in queue, is actively running, or already finished execution, +this operation will return a *HTTP 423 Locked* error response. + .. _proc_op_job_status: .. _proc_op_status: .. _proc_op_monitor: -Monitoring of a Process Execution (GetStatus) +Monitoring a Job Execution (GetStatus) --------------------------------------------------------------------- Monitoring the execution of a :term:`Job` consists of polling the status ``Location`` provided from the -:ref:`Execute ` operation and verifying the indicated ``status`` for the expected result. -The ``status`` can correspond to any of the value defined by :data:`weaver.status.JOB_STATUS_VALUES` -accordingly to the internal state of the workers processing their execution. +:ref:`Execute ` or :ref:`Trigger ` operation and verifying the +indicated ``status`` for the expected result. +The ``status`` can correspond to any of the value defined by :class:`weaver.status.Status` +accordingly to the internal state of the workers processing their execution, and the +negotiated :ref:`proc_op_job_status_alt` representation. When targeting a :term:`Job` submitted to a `Weaver` instance, monitoring is usually accomplished through the :term:`OGC API - Processes` endpoint using |status-req|_, which will return a :term:`JSON` body. @@ -2029,21 +2110,58 @@ format is employed according to the chosen location. - Location * - :term:`OGC API - Processes` - :term:`JSON` - - ``{WEAVER_URL}/jobs/{JobUUID}`` + - ``{WEAVER_URL}/jobs/{jobID}`` * - :term:`WPS` - :term:`XML` - - ``{WEAVER_WPS_OUTPUTS}/{JobUUID}.xml`` + - ``{WEAVER_WPS_OUTPUTS}/{jobID}.xml`` .. seealso:: For the :term:`WPS` endpoint, refer to :ref:`conf_settings`. -.. fixme: add example -.. fixme: describe minimum fields and extra fields +Following are examples for both representations. Note that results might vary according to other parameters such +as when using :ref:`proc_op_job_status_alt`, or when different :term:`Process` references or :term:`Workflow` +definitions are involved. + +.. literalinclude:: ../examples/job_status_ogcapi.json + :language: json + :caption: :term:`Job` Status in :term:`JSON` using the :term:`OGC API - Processes` interface + :name: job-status-ogcapi + +.. literalinclude:: ../examples/job_status_wps.xml + :language: xml + :caption: :term:`Job` Status in :term:`XML` using the :term:`WPS` interface + :name: job-status-wps + +.. _proc_op_job_status_alt: + +Alternate Job Status +~~~~~~~~~~~~~~~~~~~~ + +In order to support alternate :term:`Job` status representations, the following approaches can be used when performing +the |status-req|_ request. + +- Specify either a ``profile`` or ``schema`` query parameter (e.g.: ``/jobs/{jobID}?profile=openeo``). +- Specify a ``profile`` parameter within the ``Accept`` header (e.g.: ``Accept: application/json; profile=openeo``). + +Using the |openeo|_ profile for example, will allow returning ``status`` values that are appropriate +as per the |openeo-api|_ definition. + +When performing :ref:`Job Status ` requests, the received response should +contain a ``Content-Schema`` header indicating which of the applied ``profile`` is being represented. +This header is employed because multiple ``Content-Type: application/json`` headers are applicable +across multiple :term:`API` implementations and status representations. .. _proc_op_result: +.. _proc_op_job_detail: -Obtaining Job Outputs, Results, Logs or Errors ---------------------------------------------------------------------- +Obtaining Job Details and Metadata +---------------------------------- + +All endpoints to retrieve any of the following information about a :term:`Job` can either be requested directly +(i.e.: ``/jobs/{jobID}/...``) or with equivalent :term:`Provider` and/or :term:`Process` prefixed endpoints, +if the requested :term:`Job` did refer to those :term:`Provider` and/or :term:`Process`. +A *local* :term:`Process` would have its :term:`Job` references as ``/processes/{processId}/jobs/{jobID}/...`` +while a :ref:`proc_remote_provider` will use ``/provider/{providerName}/processes/{processId}/jobs/{jobID}/...``. .. _proc_op_job_outputs: @@ -2219,7 +2337,8 @@ Job Inputs In order to better understand the parameters that were *originally* submitted during :term:`Job` creation, the |inputs-req|_ can be employed. This will return both the data and reference ``inputs`` that were submitted, as well as the *requested* ``outputs`` [#outN]_ to retrieve any relevant ``transmissionMode``, ``format``, etc. -parameters that where specified during submission of the :ref:`proc_exec_body`. +parameters that where specified during submission of the :ref:`proc_exec_body`, and any other relevant ``headers`` +that can affect the :ref:`proc_exec_mode` and :ref:`proc_exec_results`. For convenience, this endpoint also returns relevant ``links`` applicable for the requested :term:`Job`. .. literalinclude:: ../examples/job_inputs.json @@ -2230,6 +2349,9 @@ For convenience, this endpoint also returns relevant ``links`` applicable for th .. note:: The ``links`` presented above are not an exhaustive list to keep the example relatively small. +If the :term:`Job` is still pending execution, the parameters returned by this endpoint can be modified +using the :ref:`proc_op_job_update` operation before submitting it. + .. _proc_op_job_error: .. _proc_op_job_exceptions: @@ -2278,12 +2400,33 @@ Note again that the more the :term:`Process` is verbose, the more tracking will :caption: Example :term:`JSON` Representation of :term:`Job` Logs Response :name: job-logs +.. _proc_op_job_prov: + +Job Provenance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. fixme: CWL and Job Prov (https://github.com/crim-ca/weaver/issues/673) +.. todo:: + implement ``GET /jobs/{jobID}/run`` and/or ``GET /jobs/{jobID}/prov`` + (see https://github.com/crim-ca/weaver/issues/673) + +.. _proc_op_job_stats: + +Job Statistics +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + .. note:: - All endpoints to retrieve any of the above information about a :term:`Job` can either be requested directly - (i.e.: ``/jobs/{jobID}/...``) or with equivalent :term:`Provider` and/or :term:`Process` prefixed endpoints, - if the requested :term:`Job` did refer to those :term:`Provider` and/or :term:`Process`. - A *local* :term:`Process` would have its :term:`Job` references as ``/processes/{processId}/jobs/{jobID}/...`` - while a :ref:`proc_remote_provider` will use ``/provider/{providerName}/processes/{processId}/jobs/{jobID}/...``. + This feature is specific to `Weaver`. + +The |job-stats-req|_ request can be performed to obtain runtime statistics from the :term:`Job`. +This content is only available when a :term:`Job` has successfully completed. +Below is a sample of possible response. Some parts might be omitted according to the +internal :term:`Application Package` of the :term:`Process` represented by the :term:`Job` execution. + +.. literalinclude:: ../../weaver/wps_restapi/examples/job_statistics.json + :language: json + :caption: Example :term:`JSON` of :term:`Job` Statistics Response + :name: job-statistics .. _vault_upload: diff --git a/docs/source/references.rst b/docs/source/references.rst index a2461a14c..9b0c3ffec 100644 --- a/docs/source/references.rst +++ b/docs/source/references.rst @@ -149,6 +149,10 @@ .. _ogc-api-proc-part3: https://docs.ogc.org/DRAFTS/21-009.html .. |ogc-api-proc-part3-collection-input| replace:: *Collection Input* .. _ogc-api-proc-part3-collection-input: https://docs.ogc.org/DRAFTS/21-009.html#section_collection_input +.. |ogc-api-proc-part4| replace:: *OGC API - Processes* - Part 4: Job Management +.. _ogc-api-proc-part4: https://docs.ogc.org/DRAFTS/24-051.html +.. |ogc-tb20-gdc| replace:: *OGC Testbed-20 - GeoDataCubes* +.. _ogc-tb20-gdc: https://www.ogc.org/initiatives/ogc-testbed-20/ .. |ogc-proc-ext-billing| replace:: *OGC API - Processes* - Billing extension .. _ogc-proc-ext-billing: https://github.com/opengeospatial/ogcapi-processes/tree/master/extensions/billing .. |ogc-proc-ext-quotation| replace:: *OGC API - Processes* - Quotation extension @@ -162,6 +166,10 @@ .. _ONNX-long: `ONNX`_ .. |ONNX| replace:: ONNX .. _ONNX: https://onnx.ai/ +.. |openeo| replace:: openEO +.. _openeo: https://openeo.org/ +.. |openeo-api| replace:: openEO API +.. _openeo-api: https://openeo.org/documentation/1.0/developers/api/reference.html .. |OpenAPI-spec| replace:: OpenAPI Specification .. _OpenAPI-spec: https://spec.openapis.org/oas/v3.1.0 .. |pywps| replace:: PyWPS @@ -246,6 +254,14 @@ .. _exec-req: https://pavics-weaver.readthedocs.io/en/latest/api.html#tag/Processes/paths/~1processes~1{process_id}~1execution/post .. |exec-req-job| replace:: ``POST {WEAVER_URL}/processes/{processID}/jobs`` (Execute) .. _exec-req-job: https://pavics-weaver.readthedocs.io/en/latest/api.html#tag/Processes%2Fpaths%2F~1processes~1{process_id}~1jobs%2Fpost +.. |job-exec-req| replace:: ``POST {WEAVER_URL}/jobs`` (Create) +.. _job-exec-req: https://pavics-weaver.readthedocs.io/en/latest/api.html#tag/Jobs/paths/~1jobs/post +.. |job-update-req| replace:: ``PATCH {WEAVER_URL}/jobs/{jobID}`` (Update) +.. _job-update-req: https://pavics-weaver.readthedocs.io/en/latest/api.html#tag/Jobs/paths/~1jobs~1{job_id}/patch +.. |job-trigger-req| replace:: ``POST {WEAVER_URL}/jobs{jobID}/results`` (Trigger) +.. _job-trigger-req: https://pavics-weaver.readthedocs.io/en/latest/api.html#tag/Jobs/paths/~1jobs~1{job_id}~1results/post +.. |job-stats-req| replace:: ``GET {WEAVER_URL}/jobs{jobID}/statistics`` +.. _job-stats-req: https://pavics-weaver.readthedocs.io/en/latest/api.html#tag/Jobs/paths/~1jobs~1{job_id}~1statistics/get .. |vis-req| replace:: ``PUT {WEAVER_URL}/processes/{processID}/visibility`` (Visibility) .. _vis-req: https://pavics-weaver.readthedocs.io/en/latest/api.html#tag/Processes%2Fpaths%2F~1processes~1%7Bprocess_id%7D~1visibility%2Fput .. |pkg-req| replace:: ``GET {WEAVER_URL}/processes/{processID}/package`` (Package) diff --git a/setup.cfg b/setup.cfg index 4b450a14b..35c398a31 100644 --- a/setup.cfg +++ b/setup.cfg @@ -60,6 +60,12 @@ markers = slow: mark test to be slow remote: mark test with remote Weaver instance requirement vault: mark test with Vault file feature validation + html: mark test as related to HTML rendering + oap_part1: mark test as 'OGC API - Processes - Part 1: Core' functionalities + oap_part2: mark test as 'OGC API - Processes - Part 2: Deploy, Replace, Undeploy (DRU)' functionalities + oap_part3: mark test as 'OGC API - Processes - Part 3: Workflows and Chaining' functionalities + oap_part4: mark test as 'OGC API - Processes - Part 4: Job Management' functionalities + openeo: mark test as evaluating 'openEO' functionalities filterwarnings = ignore:No file specified for WPS-1 providers registration:RuntimeWarning ignore:.*configuration setting.*weaver\.cwl_processes_dir.*:RuntimeWarning diff --git a/tests/functional/test_builtin.py b/tests/functional/test_builtin.py index 1b69ce723..dd41a5270 100644 --- a/tests/functional/test_builtin.py +++ b/tests/functional/test_builtin.py @@ -21,8 +21,9 @@ from weaver.execute import ExecuteControlOption, ExecuteMode, ExecuteResponse, ExecuteTransmissionMode from weaver.formats import ContentEncoding, ContentType, get_format, repr_json from weaver.processes.builtin import file_index_selector, jsonarray2netcdf, metalink2netcdf, register_builtin_processes +from weaver.processes.constants import JobInputsOutputsSchema from weaver.status import Status -from weaver.utils import create_metalink, fully_qualified_name +from weaver.utils import create_metalink, fully_qualified_name, get_path_kvp from weaver.wps.utils import map_wps_output_location from weaver.wps_restapi import swagger_definitions as sd @@ -209,7 +210,7 @@ def test_jsonarray2netcdf_execute_invalid_file_local(self): assert resp.status_code == 201 job_url = resp.json["location"] - job_res = self.monitor_job(job_url, expect_failed=True) + job_res = self.monitor_job(job_url, expect_failed=True, return_status=True) assert job_res["status"] == Status.FAILED job_logs = self.app.get(f"{job_url}/logs").json assert any("ValueError: Not a valid file URL reference" in log for log in job_logs) @@ -242,7 +243,7 @@ def test_jsonarray2netcdf_execute_async(self): assert resp.headers["Location"] == job_url results = self.monitor_job(job_url) - output_url = f"{job_url}/outputs" + output_url = get_path_kvp(f"{job_url}/outputs", schema=JobInputsOutputsSchema.OLD) resp = self.app.get(output_url, headers=self.json_headers) assert resp.status_code == 200, f"Error job outputs:\n{repr_json(resp.text, indent=2)}" outputs = resp.json @@ -265,6 +266,7 @@ def test_jsonarray2netcdf_execute_async_output_by_reference_response_document(se with contextlib.ExitStack() as stack_exec: body, nc_data = self.setup_jsonarray2netcdf_inputs(stack_exec) body.update({ + "mode": ExecuteMode.ASYNC, "response": ExecuteResponse.DOCUMENT, # by value/reference doesn't matter because of this "outputs": [{"id": "output", "transmissionMode": ExecuteTransmissionMode.REFERENCE}], }) @@ -288,7 +290,7 @@ def test_jsonarray2netcdf_execute_async_output_by_reference_response_document(se # even though results are requested by Link reference, # Weaver still offers them with document on outputs endpoint - output_url = f"{job_url}/outputs" + output_url = get_path_kvp(f"{job_url}/outputs", schema=JobInputsOutputsSchema.OLD) resp = self.app.get(output_url, headers=self.json_headers) assert resp.status_code == 200, f"Error job outputs:\n{resp.text}" outputs = resp.json @@ -305,6 +307,7 @@ def test_jsonarray2netcdf_execute_async_output_by_value_response_raw(self): with contextlib.ExitStack() as stack_exec: body, nc_data = self.setup_jsonarray2netcdf_inputs(stack_exec) body.update({ + "mode": ExecuteMode.ASYNC, "response": ExecuteResponse.RAW, # by value/reference important here # NOTE: quantity of outputs important as well # since single output, content-type is directly that output (otherwise should be multipart) @@ -332,7 +335,7 @@ def test_jsonarray2netcdf_execute_async_output_by_value_response_raw(self): # even though results are requested by raw data, # Weaver still offers them with document on outputs endpoint - output_url = f"{job_url}/outputs" + output_url = get_path_kvp(f"{job_url}/outputs", schema=JobInputsOutputsSchema.OLD) resp = self.app.get(output_url, headers=self.json_headers) assert resp.status_code == 200, f"Error job outputs:\n{resp.text}" outputs = resp.json @@ -351,6 +354,7 @@ def test_jsonarray2netcdf_execute_async_output_by_reference_response_raw(self): with contextlib.ExitStack() as stack_exec: body, nc_data = self.setup_jsonarray2netcdf_inputs(stack_exec) body.update({ + "mode": ExecuteMode.ASYNC, "response": ExecuteResponse.RAW, # by value/reference important here "outputs": [{"id": "output", "transmissionMode": ExecuteTransmissionMode.REFERENCE}], # Link header }) @@ -374,7 +378,8 @@ def test_jsonarray2netcdf_execute_async_output_by_reference_response_raw(self): # even though results are requested by Link reference, # Weaver still offers them with document on outputs endpoint - resp = self.app.get(f"{job_url}/outputs", headers=self.json_headers) + output_url = get_path_kvp(f"{job_url}/outputs", schema=JobInputsOutputsSchema.OLD) + resp = self.app.get(output_url, headers=self.json_headers) assert resp.status_code == 200, f"Error job outputs:\n{repr_json(resp.text, indent=2)}" outputs = resp.json @@ -442,7 +447,7 @@ def test_jsonarray2netcdf_execute_sync(self): assert resp.content_type == ContentType.APP_JSON results = resp.json - output_url = f"{job_url}/outputs" + output_url = get_path_kvp(f"{job_url}/outputs", schema=JobInputsOutputsSchema.OLD) resp = self.app.get(output_url, headers=self.json_headers) assert resp.status_code == 200, f"Error job outputs:\n{repr_json(resp.text, indent=2)}" outputs = resp.json diff --git a/tests/functional/test_celery.py b/tests/functional/test_celery.py index ad810f155..d705b8e75 100644 --- a/tests/functional/test_celery.py +++ b/tests/functional/test_celery.py @@ -50,10 +50,12 @@ def test_celery_registry_resolution(): settings = get_settings_from_testapp(webapp) wps_url = get_wps_url(settings) job_store = get_db(settings).get_store("jobs") - job1 = job_store.save_job(task_id="tmp", process="jsonarray2netcdf", - inputs={"input": {"href": "http://random-dont-care.com/fake.json"}}) - job2 = job_store.save_job(task_id="tmp", process="jsonarray2netcdf", - inputs={"input": {"href": "http://random-dont-care.com/fake.json"}}) + job1 = job_store.save_job( + task_id="tmp", process="jsonarray2netcdf", inputs={"input": {"href": "http://random-dont-care.com/fake.json"}} + ) + job2 = job_store.save_job( + task_id="tmp", process="jsonarray2netcdf", inputs={"input": {"href": "http://random-dont-care.com/fake.json"}} + ) with contextlib.ExitStack() as stack: celery_mongo_broker = f"""mongodb://{settings["mongodb.host"]}:{settings["mongodb.port"]}/celery-test""" diff --git a/tests/functional/test_cli.py b/tests/functional/test_cli.py index 6a157bc67..19dc75084 100644 --- a/tests/functional/test_cli.py +++ b/tests/functional/test_cli.py @@ -770,8 +770,9 @@ def test_jobs_search_multi_status(self): class TestWeaverCLI(TestWeaverClientBase): def setUp(self): super(TestWeaverCLI, self).setUp() - job = self.job_store.save_job(task_id="12345678-1111-2222-3333-111122223333", process="fake-process", - access=Visibility.PUBLIC) + job = self.job_store.save_job( + task_id="12345678-1111-2222-3333-111122223333", process="fake-process", access=Visibility.PUBLIC + ) job.status = Status.SUCCEEDED self.test_job = self.job_store.update_job(job) @@ -985,7 +986,7 @@ def test_deploy_docker_auth_username_password_valid(self): the expected authentication credentials. Re-running this test by itself validates if this case happened. Find a way to make it work seamlessly. Retries sometime works, but it is not guaranteed. """ - p_id = self.fully_qualified_test_process_name() + p_id = self.fully_qualified_test_name() docker_reg = "fake.repo" docker_img = "org/project/private-image:latest" docker_ref = f"{docker_reg}/{docker_img}" @@ -1030,7 +1031,7 @@ def test_deploy_docker_auth_token_valid(self): .. seealso:: :meth:`tests.wps_restapi.test_processes.WpsRestApiProcessesTest.test_deploy_process_CWL_DockerRequirement_auth_header_format` """ - p_id = self.fully_qualified_test_process_name() + p_id = self.fully_qualified_test_name() docker_reg = "fake.repo" docker_img = "org/project/private-image:latest" docker_ref = f"{docker_reg}/{docker_img}" @@ -1072,7 +1073,7 @@ def test_deploy_docker_auth_username_or_password_with_token_invalid(self): All parameter values are themselves valid, only their combination that are not. """ - p_id = self.fully_qualified_test_process_name() + p_id = self.fully_qualified_test_name() docker_reg = "fake.repo" docker_img = "org/project/private-image:latest" docker_ref = f"{docker_reg}/{docker_img}" @@ -1150,7 +1151,7 @@ def test_deploy_docker_auth_username_or_password_missing_invalid(self): All parameter values are themselves valid, only their combination that are not. """ - p_id = self.fully_qualified_test_process_name() + p_id = self.fully_qualified_test_name() docker_reg = "fake.repo" docker_img = "org/project/private-image:latest" docker_ref = f"{docker_reg}/{docker_img}" @@ -2279,37 +2280,36 @@ def test_job_statistics(self): body = json.loads(text) assert body == job.statistics - def test_job_info_wrong_status(self): + @parameterized.expand([ + ("results", Status.FAILED, "JobResultsFailed", True), + ("statistics", Status.FAILED, "NoJobStatistics", True), + ("exceptions", Status.FAILED, repr_json(["failed"], force_string=True, indent=2), False), + ]) + def test_job_info_status_dependant(self, operation, status, expect, expect_error): # results/statistics must be in success status job = self.job_store.save_job(task_id=uuid.uuid4(), process="test-process", access=Visibility.PUBLIC) job.statistics = resources.load_example("job_statistics.json") job.save_log(message="Some info", status=Status.ACCEPTED, errors=ValueError("failed")) job = self.job_store.update_job(job) - - for operation, status, expect in [ - ("results", Status.FAILED, "JobResultsFailed"), - ("statistics", Status.FAILED, "404 Not Found"), - # ("exceptions", Status.SUCCEEDED, "404 Not Found"), # no error, just irrelevant or empty - ]: - job.status = status - job = self.job_store.update_job(job) - lines = mocked_sub_requests( - self.app, run_command, - [ - # "weaver", - operation, - "-u", self.url, - "-j", str(job.id), - "-nL", - ], - trim=False, - entrypoint=weaver_cli, - only_local=True, - expect_error=True, - ) - assert len(lines) - text = "".join(lines) - assert expect in text + job.status = status + job = self.job_store.update_job(job) + lines = mocked_sub_requests( + self.app, run_command, + [ + # "weaver", + operation, + "-u", self.url, + "-j", str(job.id), + "-nL", + ], + trim=False, + entrypoint=weaver_cli, + only_local=True, + expect_error=expect_error, + ) + assert len(lines) + text = "\n".join(lines) + assert expect in text def test_execute_remote_input(self): """ diff --git a/tests/functional/test_workflow.py b/tests/functional/test_workflow.py index c0ef941d9..5b9d12bca 100644 --- a/tests/functional/test_workflow.py +++ b/tests/functional/test_workflow.py @@ -37,7 +37,7 @@ ) from weaver import WEAVER_ROOT_DIR from weaver.config import WeaverConfiguration -from weaver.execute import ExecuteResponse, ExecuteReturnPreference, ExecuteTransmissionMode +from weaver.execute import ExecuteMode, ExecuteResponse, ExecuteReturnPreference, ExecuteTransmissionMode from weaver.formats import ContentType from weaver.processes.constants import ( CWL_REQUIREMENT_MULTIPLE_INPUT, @@ -919,6 +919,7 @@ def workflow_runner( # execute workflow execute_body = override_execute_body or workflow_info.execute_payload + execute_body.setdefault("mode", ExecuteMode.ASYNC) execute_path = f"{process_path}/jobs" self.assert_test(lambda: execute_body is not None, message="Cannot execute workflow without a request body!") diff --git a/tests/functional/test_wps_package.py b/tests/functional/test_wps_package.py index cdbbcd10d..9a34fc448 100644 --- a/tests/functional/test_wps_package.py +++ b/tests/functional/test_wps_package.py @@ -83,7 +83,14 @@ from responses import RequestsMock - from weaver.typedefs import CWL_AnyRequirements, CWL_RequirementsDict, JSON, Number + from weaver.typedefs import ( + CWL_AnyRequirements, + CWL_RequirementsDict, + JSON, + Number, + ProcessOfferingListing, + ProcessOfferingMapping + ) EDAM_PLAIN = f"{EDAM_NAMESPACE}:{EDAM_MAPPING[ContentType.TEXT_PLAIN]}" OGC_NETCDF = f"{OGC_NAMESPACE}:{OGC_MAPPING[ContentType.APP_NETCDF]}" @@ -169,7 +176,7 @@ def test_deploy_ogc_schema(self): # even if deployed as OGC schema, OLD schema can be converted back desc = self.describe_process(self._testMethodName, ProcessSchema.OLD) - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing assert "inputs" in proc and isinstance(proc["inputs"], list) and len(proc["inputs"]) == 1 assert "outputs" in proc and isinstance(proc["outputs"], list) and len(proc["outputs"]) == 1 assert proc["inputs"][0]["id"] == "url" @@ -608,7 +615,7 @@ def test_deploy_merge_literal_io_from_package(self): "executionUnit": [{"unit": cwl}], } desc, _ = self.deploy_process(body, describe_schema=ProcessSchema.OLD) - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing assert proc["id"] == self._testMethodName assert proc["title"] == "some title" @@ -707,7 +714,7 @@ def test_deploy_merge_literal_io_from_package_and_offering(self): "executionUnit": [{"unit": cwl}], } desc, pkg = self.deploy_process(body, describe_schema=ProcessSchema.OLD) - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing assert proc["id"] == self._testMethodName assert proc["title"] == "some title" @@ -865,7 +872,7 @@ def test_deploy_merge_complex_io_format_references(self): }}], } desc, pkg = self.deploy_process(body, describe_schema=ProcessSchema.OLD) - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing assert proc["inputs"][0]["id"] == "wps_only_format_exists" assert len(proc["inputs"][0]["formats"]) == 1 @@ -989,7 +996,7 @@ def test_deploy_merge_mediatype_io_format_references(self): }] } desc, _ = self.deploy_process(body, describe_schema=ProcessSchema.OLD) - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing assert proc["inputs"][0]["id"] == "wps_format_mimeType" assert proc["inputs"][0]["formats"][0]["mediaType"] == ContentType.APP_JSON assert proc["inputs"][1]["id"] == "wps_format_mediaType" @@ -1418,7 +1425,7 @@ def test_deploy_merge_complex_io_with_multiple_formats_and_defaults(self): "executionUnit": [{"unit": cwl}], } desc, pkg = self.deploy_process(body, describe_schema=ProcessSchema.OLD) - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing # process description input validation assert proc["inputs"][0]["id"] == "single_value_single_format" @@ -1660,7 +1667,7 @@ def test_deploy_merge_resolution_io_min_max_occurs(self): "executionUnit": [{"unit": cwl}], } desc, pkg = self.deploy_process(body, describe_schema=ProcessSchema.OLD) - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing assert proc["inputs"][0]["id"] == "required_literal" assert proc["inputs"][0]["minOccurs"] == 1 @@ -1795,7 +1802,7 @@ def test_deploy_merge_valid_io_min_max_occurs_as_str_or_int(self): self.fail("MinOccurs/MaxOccurs values defined as valid int/str should not raise an invalid schema error") inputs = body["processDescription"]["inputs"] # type: List[JSON] - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing assert isinstance(proc["inputs"], list) assert len(proc["inputs"]) == len(inputs) for i, process_input in enumerate(inputs): @@ -1843,23 +1850,24 @@ def test_deploy_merge_wps_io_as_mappings(self): "executionUnit": [{"unit": cwl}], } desc, _ = self.deploy_process(body, describe_schema=ProcessSchema.OGC) + proc = desc # type: ProcessOfferingMapping - assert isinstance(desc["inputs"], dict) - assert len(desc["inputs"]) == len(body["processDescription"]["process"]["inputs"]) - assert isinstance(desc["outputs"], dict) - assert len(desc["outputs"]) == len(body["processDescription"]["process"]["outputs"]) + assert isinstance(proc["inputs"], dict) + assert len(proc["inputs"]) == len(body["processDescription"]["process"]["inputs"]) + assert isinstance(proc["outputs"], dict) + assert len(proc["outputs"]) == len(body["processDescription"]["process"]["outputs"]) # following inputs metadata were correctly parsed from WPS mapping entries if defined and not using defaults - assert desc["inputs"]["input_num"]["title"] == "Input numbers" - assert desc["inputs"]["input_num"]["maxOccurs"] == 20 - assert desc["inputs"]["input_num"]["literalDataDomains"][0]["dataType"]["name"] == "float" - assert desc["inputs"]["input_file"]["title"] == "Test File" - assert desc["inputs"]["input_file"]["formats"][0]["mediaType"] == ContentType.APP_ZIP - assert desc["outputs"]["values"]["title"] == "Test Output" - assert desc["outputs"]["values"]["description"] == "CSV raw values" - assert desc["outputs"]["values"]["literalDataDomains"][0]["dataType"]["name"] == "string" - assert desc["outputs"]["out_file"]["title"] == "Result File" - assert desc["outputs"]["out_file"]["formats"][0]["mediaType"] == "text/csv" + assert proc["inputs"]["input_num"]["title"] == "Input numbers" + assert proc["inputs"]["input_num"]["maxOccurs"] == 20 + assert proc["inputs"]["input_num"]["literalDataDomains"][0]["dataType"]["name"] == "float" + assert proc["inputs"]["input_file"]["title"] == "Test File" + assert proc["inputs"]["input_file"]["formats"][0]["mediaType"] == ContentType.APP_ZIP + assert proc["outputs"]["values"]["title"] == "Test Output" + assert proc["outputs"]["values"]["description"] == "CSV raw values" + assert proc["outputs"]["values"]["literalDataDomains"][0]["dataType"]["name"] == "string" + assert proc["outputs"]["out_file"]["title"] == "Result File" + assert proc["outputs"]["out_file"]["formats"][0]["mediaType"] == "text/csv" def test_execute_job_with_accept_languages(self): """ @@ -2240,7 +2248,7 @@ def test_execute_job_with_inline_input_values(self): def test_execute_job_with_bbox(self): body = self.retrieve_payload("EchoBoundingBox", "deploy", local=True) - proc = self.fully_qualified_test_process_name(self._testMethodName) + proc = self.fully_qualified_test_name(self._testMethodName) self.deploy_process(body, describe_schema=ProcessSchema.OGC, process_id=proc) data = self.retrieve_payload("EchoBoundingBox", "execute", local=True) @@ -2276,7 +2284,7 @@ def test_execute_job_with_bbox(self): def test_execute_job_with_collection_input_geojson_feature_collection(self): name = "EchoFeatures" body = self.retrieve_payload(name, "deploy", local=True) - proc = self.fully_qualified_test_process_name(self._testMethodName) + proc = self.fully_qualified_test_name(self._testMethodName) self.deploy_process(body, describe_schema=ProcessSchema.OGC, process_id=proc) with contextlib.ExitStack() as stack: @@ -2331,7 +2339,7 @@ def test_execute_job_with_collection_input_geojson_feature_collection(self): def test_execute_job_with_collection_input_ogc_features(self, filter_method, filter_lang, filter_value): name = "EchoFeatures" body = self.retrieve_payload(name, "deploy", local=True) - proc = self.fully_qualified_test_process_name(self._testMethodName) + proc = self.fully_qualified_test_name(self._testMethodName) self.deploy_process(body, describe_schema=ProcessSchema.OGC, process_id=proc) with contextlib.ExitStack() as stack: @@ -3014,7 +3022,7 @@ def test_deploy_merge_complex_io_from_package(self): "executionUnit": [{"unit": cwl}], } desc, _ = self.deploy_process(body, describe_schema=ProcessSchema.OLD) - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing assert proc["id"] == self._testMethodName assert proc["title"] == "some title" assert proc["description"] == "this is a test" @@ -3111,7 +3119,7 @@ def test_deploy_merge_complex_io_from_package_and_offering(self): "executionUnit": [{"unit": cwl}], } desc, pkg = self.deploy_process(body, describe_schema=ProcessSchema.OLD) - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing assert proc["id"] == self._testMethodName assert proc["title"] == "some title" @@ -3185,7 +3193,7 @@ def test_deploy_literal_and_complex_io_from_wps_xml_reference(self): # basic contents validation assert "cwlVersion" in pkg assert "process" in desc - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing assert proc["id"] == self._testMethodName # package I/O validation @@ -3278,7 +3286,7 @@ def test_deploy_enum_array_and_multi_format_inputs_from_wps_xml_reference(self): # basic contents validation assert "cwlVersion" in pkg assert "process" in desc - proc = desc["process"] + proc = desc["process"] # type: ProcessOfferingListing assert proc["id"] == self._testMethodName # package I/O validation @@ -3598,9 +3606,10 @@ def fix_result_multipart_indent(results): res_dedent = res_dedent.rstrip("\n ") # last line often indented less because of closing multiline string return res_dedent + @pytest.mark.oap_part1 def test_execute_single_output_prefer_header_return_representation_literal(self): proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -3644,9 +3653,10 @@ def test_execute_single_output_prefer_header_return_representation_literal(self) }, } + @pytest.mark.oap_part1 def test_execute_single_output_prefer_header_return_representation_complex(self): proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -3693,12 +3703,13 @@ def test_execute_single_output_prefer_header_return_representation_complex(self) }, } + @pytest.mark.oap_part1 def test_execute_single_output_prefer_header_return_minimal_literal_accept_default(self): """ For single requested output, without ``Accept`` content negotiation, its default format is returned directly. """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -3746,12 +3757,13 @@ def test_execute_single_output_prefer_header_return_minimal_literal_accept_defau }, } + @pytest.mark.oap_part1 def test_execute_single_output_prefer_header_return_minimal_literal_accept_json(self): """ For single requested output, with ``Accept`` :term:`JSON` content negotiation, document response is returned. """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -3801,6 +3813,7 @@ def test_execute_single_output_prefer_header_return_minimal_literal_accept_json( }, } + @pytest.mark.oap_part1 def test_execute_single_output_prefer_header_return_minimal_complex_accept_default(self): """ For single requested output, without ``Accept`` content negotiation, its default format is returned by link. @@ -3818,7 +3831,7 @@ def test_execute_single_output_prefer_header_return_minimal_complex_accept_defau - :func:`test_execute_single_output_prefer_header_return_representation_complex` """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -3880,6 +3893,7 @@ def test_execute_single_output_prefer_header_return_minimal_complex_accept_defau }, } + @pytest.mark.oap_part1 def test_execute_single_output_prefer_header_return_minimal_complex_accept_json(self): """ For single requested output, with ``Accept`` :term:`JSON` content negotiation, document response is returned. @@ -3900,7 +3914,7 @@ def test_execute_single_output_prefer_header_return_minimal_complex_accept_json( using the ``response`` parameter at :term:`Job` execution time, as alternative method to ``Prefer``. """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -3955,9 +3969,10 @@ def test_execute_single_output_prefer_header_return_minimal_complex_accept_json( }, } + @pytest.mark.oap_part1 def test_execute_single_output_response_raw_value_literal(self): proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -3980,7 +3995,7 @@ def test_execute_single_output_response_raw_value_literal(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" # request status instead of results since not expecting 'document' JSON in this case status_url = resp.json["location"] @@ -3999,6 +4014,7 @@ def test_execute_single_output_response_raw_value_literal(self): }, } + @pytest.mark.oap_part1 def test_execute_single_output_response_raw_value_complex(self): """ Since value transmission is requested for a single output, its :term:`JSON` contents are returned directly. @@ -4007,7 +4023,7 @@ def test_execute_single_output_response_raw_value_complex(self): - :func:`test_execute_single_output_prefer_header_return_minimal_complex_accept_json` """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4031,7 +4047,7 @@ def test_execute_single_output_response_raw_value_complex(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -4054,9 +4070,10 @@ def test_execute_single_output_response_raw_value_complex(self): }, } + @pytest.mark.oap_part1 def test_execute_single_output_response_raw_reference_literal(self): proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4080,7 +4097,7 @@ def test_execute_single_output_response_raw_reference_literal(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -4113,9 +4130,10 @@ def test_execute_single_output_response_raw_reference_literal(self): }, } + @pytest.mark.oap_part1 def test_execute_single_output_response_raw_reference_complex(self): proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4139,7 +4157,7 @@ def test_execute_single_output_response_raw_reference_complex(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -4172,6 +4190,7 @@ def test_execute_single_output_response_raw_reference_complex(self): }, } + @pytest.mark.oap_part1 def test_execute_single_output_multipart_accept_data(self): """ Validate that requesting multipart for a single output is permitted. @@ -4185,7 +4204,7 @@ def test_execute_single_output_multipart_accept_data(self): - :func:`test_execute_single_output_multipart_accept_alt_format` """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4211,7 +4230,7 @@ def test_execute_single_output_multipart_accept_data(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" not in resp.headers # rely on location that should be provided to find the job ID @@ -4250,6 +4269,7 @@ def test_execute_single_output_multipart_accept_data(self): }, } + @pytest.mark.oap_part1 def test_execute_single_output_multipart_accept_link(self): """ Validate that requesting multipart for a single output is permitted. @@ -4261,7 +4281,7 @@ def test_execute_single_output_multipart_accept_link(self): - :func:`test_execute_single_output_multipart_accept_alt_format` """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4287,7 +4307,7 @@ def test_execute_single_output_multipart_accept_link(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" not in resp.headers # rely on location that should be provided to find the job ID @@ -4326,6 +4346,7 @@ def test_execute_single_output_multipart_accept_link(self): } # FIXME: implement (https://github.com/crim-ca/weaver/pull/548) + @pytest.mark.oap_part1 @pytest.mark.xfail(reason="not implemented") def test_execute_single_output_multipart_accept_alt_format(self): """ @@ -4335,7 +4356,7 @@ def test_execute_single_output_multipart_accept_alt_format(self): output representation, based on the ``format`` definition. """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4361,7 +4382,7 @@ def test_execute_single_output_multipart_accept_alt_format(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" not in resp.headers # rely on location that should be provided to find the job ID @@ -4403,15 +4424,16 @@ def test_execute_single_output_multipart_accept_alt_format(self): # validate the results can be obtained with the "real" representation result_json = self.app.get(f"/jobs/{job_id}/results/output_json", headers=self.json_headers) - assert result_json.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert result_json.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert result_json.content_type == ContentType.APP_JSON assert result_json.text == "{\"data\":\"test\"}" # FIXME: implement (https://github.com/crim-ca/weaver/pull/548) + @pytest.mark.oap_part1 @pytest.mark.xfail(reason="not implemented") def test_execute_single_output_response_document_alt_format_yaml(self): proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4437,7 +4459,7 @@ def test_execute_single_output_response_document_alt_format_yaml(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" not in resp.headers # rely on location that should be provided to find the job ID @@ -4480,13 +4502,14 @@ def test_execute_single_output_response_document_alt_format_yaml(self): # FIXME: implement (https://github.com/crim-ca/weaver/pull/548) # validate the results can be obtained with the "real" representation result_json = self.app.get(f"/jobs/{job_id}/results/output_json", headers=self.json_headers) - assert result_json.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert result_json.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert result_json.content_type == ContentType.APP_JSON assert result_json.text == "{\"data\":\"test\"}" + @pytest.mark.oap_part1 def test_execute_single_output_response_document_alt_format_json_raw_literal(self): proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4517,7 +4540,7 @@ def test_execute_single_output_response_document_alt_format_json_raw_literal(sel path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" not in resp.headers # rely on location that should be provided to find the job ID @@ -4555,6 +4578,7 @@ def test_execute_single_output_response_document_alt_format_json_raw_literal(sel # assert result_json.content_type == ContentType.APP_JSON # assert result_json.json == {"data": "test"} + @pytest.mark.oap_part1 def test_execute_single_output_response_document_default_format_json_special(self): """ Validate that a :term:`JSON` output is directly embedded in a ``document`` response also using :term:`JSON`. @@ -4569,7 +4593,7 @@ def test_execute_single_output_response_document_default_format_json_special(sel - :func:`test_execute_single_output_response_document_alt_format_json` """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4600,7 +4624,7 @@ def test_execute_single_output_response_document_default_format_json_special(sel path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" not in resp.headers # rely on location that should be provided to find the job ID @@ -4631,6 +4655,7 @@ def test_execute_single_output_response_document_default_format_json_special(sel }, } + @pytest.mark.oap_part1 @parameterized.expand([ ContentType.MULTIPART_ANY, ContentType.MULTIPART_MIXED, @@ -4644,7 +4669,7 @@ def test_execute_multi_output_multipart_accept(self, multipart_header): - :func:`test_execute_multi_output_multipart_accept_async_not_acceptable` """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4676,7 +4701,7 @@ def test_execute_multi_output_multipart_accept(self, multipart_header): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -4724,6 +4749,7 @@ def test_execute_multi_output_multipart_accept(self, multipart_header): }, } + @pytest.mark.oap_part1 def test_execute_multi_output_multipart_accept_async_not_acceptable(self): """ When executing the process asynchronously, ``Accept`` with multipart (strictly) is not acceptable. @@ -4736,7 +4762,7 @@ def test_execute_multi_output_multipart_accept_async_not_acceptable(self): - :func:`test_execute_multi_output_multipart_accept_async_alt_acceptable` """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4757,7 +4783,7 @@ def test_execute_multi_output_multipart_accept_async_not_acceptable(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 406, f"Expected error. Instead got: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 406, f"Expected error. Instead got: [{resp.status_code}]\nReason:\n{resp.text}" assert resp.content_type == ContentType.APP_JSON, "Expect JSON instead of Multipart because of error." assert "Accept header" in resp.json["detail"] assert resp.json["value"] == ContentType.MULTIPART_MIXED @@ -4766,6 +4792,7 @@ def test_execute_multi_output_multipart_accept_async_not_acceptable(self): "in": "headers", } + @pytest.mark.oap_part1 def test_execute_multi_output_multipart_accept_async_alt_acceptable(self): """ When executing the process asynchronously, ``Accept`` with multipart and an alternative is acceptable. @@ -4778,7 +4805,7 @@ def test_execute_multi_output_multipart_accept_async_alt_acceptable(self): - :func:`test_execute_multi_output_multipart_accept_async_not_acceptable` """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4800,15 +4827,16 @@ def test_execute_multi_output_multipart_accept_async_alt_acceptable(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert resp.content_type == ContentType.APP_JSON, "Expect JSON instead of Multipart because of error." assert "status" in resp.json, "Expected a JSON Job Status response." assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") + @pytest.mark.oap_part1 def test_execute_multi_output_prefer_header_return_representation(self): proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4835,7 +4863,7 @@ def test_execute_multi_output_prefer_header_return_representation(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -4881,9 +4909,10 @@ def test_execute_multi_output_prefer_header_return_representation(self): }, } + @pytest.mark.oap_part1 def test_execute_multi_output_response_raw_value(self): proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4908,7 +4937,7 @@ def test_execute_multi_output_response_raw_value(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -4954,6 +4983,7 @@ def test_execute_multi_output_response_raw_value(self): }, } + @pytest.mark.oap_part1 def test_execute_multi_output_response_raw_reference_default_links(self): """ All outputs resolved as reference (explicitly or inferred) with raw representation should be all Link headers. @@ -4964,7 +4994,7 @@ def test_execute_multi_output_response_raw_reference_default_links(self): - :func:`test_execute_multi_output_response_raw_reference_accept_multipart` """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -4989,7 +5019,7 @@ def test_execute_multi_output_response_raw_reference_default_links(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -5028,6 +5058,7 @@ def test_execute_multi_output_response_raw_reference_default_links(self): }, } + @pytest.mark.oap_part1 def test_execute_multi_output_response_raw_reference_accept_multipart(self): """ Requesting ``multipart`` explicitly should return it instead of default ``Link`` headers response. @@ -5038,7 +5069,7 @@ def test_execute_multi_output_response_raw_reference_accept_multipart(self): - :func:`test_execute_multi_output_multipart_accept_async_not_acceptable` """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -5067,7 +5098,7 @@ def test_execute_multi_output_response_raw_reference_accept_multipart(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -5115,9 +5146,10 @@ def test_execute_multi_output_response_raw_reference_accept_multipart(self): }, } + @pytest.mark.oap_part1 def test_execute_multi_output_response_raw_mixed(self): proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -5143,7 +5175,7 @@ def test_execute_multi_output_response_raw_mixed(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -5200,12 +5232,13 @@ def test_execute_multi_output_response_raw_mixed(self): }, } + @pytest.mark.oap_part1 def test_execute_multi_output_prefer_header_return_minimal_defaults(self): """ Test ``Prefer: return=minimal`` with default ``transmissionMode`` resolutions for literal/complex outputs. """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -5232,7 +5265,7 @@ def test_execute_multi_output_prefer_header_return_minimal_defaults(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -5264,6 +5297,7 @@ def test_execute_multi_output_prefer_header_return_minimal_defaults(self): }, } + @pytest.mark.oap_part1 def test_execute_multi_output_prefer_header_return_minimal_override_transmission(self): """ Test ``Prefer: return=minimal`` with ``transmissionMode`` overrides. @@ -5273,7 +5307,7 @@ def test_execute_multi_output_prefer_header_return_minimal_override_transmission embedded inline. However, this respects the *preference* vs *enforced* property requirements. """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -5301,7 +5335,7 @@ def test_execute_multi_output_prefer_header_return_minimal_override_transmission path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -5344,12 +5378,13 @@ def test_execute_multi_output_prefer_header_return_minimal_override_transmission }, } + @pytest.mark.oap_part1 def test_execute_multi_output_response_document_defaults(self): """ Test ``response: document`` with default ``transmissionMode`` resolutions for literal/complex outputs. """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -5376,7 +5411,7 @@ def test_execute_multi_output_response_document_defaults(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -5408,12 +5443,13 @@ def test_execute_multi_output_response_document_defaults(self): }, } + @pytest.mark.oap_part1 def test_execute_multi_output_response_document_mixed(self): """ Test ``response: document`` with ``transmissionMode`` specified to force convertion of literal/complex outputs. """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) @@ -5442,7 +5478,7 @@ def test_execute_multi_output_response_document_mixed(self): path = f"/processes/{p_id}/execution" resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, data=exec_content, headers=exec_headers, only_local=True) - assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" assert "Preference-Applied" in resp.headers assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") @@ -5485,6 +5521,308 @@ def test_execute_multi_output_response_document_mixed(self): }, } + def test_execute_mismatch_process(self): + proc = "EchoResultsTester" + p_id = self.fully_qualified_test_name(proc) + body = self.retrieve_payload(proc, "deploy", local=True) + self.deploy_process(body, process_id=p_id) + + # use non-existing process to ensure this particular situation is handled as well + # a missing process reference must not cause an early "not-found" response + proc = "random-other-process" + proc_other = self.fully_qualified_test_name(proc) + + exec_content = { + "process": f"https://localhost/processes/{proc_other}", + "inputs": {"message": "test"} + } + with contextlib.ExitStack() as stack: + for mock_exec in mocked_execute_celery(): + stack.enter_context(mock_exec) + path = f"/processes/{p_id}/execution" # mismatch on purpose + resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, + data=exec_content, headers=self.json_headers, only_local=True) + assert resp.status_code == 400, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" + assert resp.content_type == ContentType.APP_JSON + assert resp.json["cause"] == {"name": "process", "in": "body"} + + @pytest.mark.oap_part4 + def test_execute_jobs_sync(self): + proc = "EchoResultsTester" + p_id = self.fully_qualified_test_name(proc) + body = self.retrieve_payload(proc, "deploy", local=True) + self.deploy_process(body, process_id=p_id) + + exec_headers = { + "Accept": ContentType.APP_JSON, # response 'document' should be enough to use JSON, but make extra sure + "Content-Type": ContentType.APP_JSON, + } + exec_content = { + "process": f"https://localhost/processes/{p_id}", + "mode": ExecuteMode.SYNC, # force sync to make sure JSON job status is not returned instead + "response": ExecuteResponse.DOCUMENT, + "inputs": { + "message": "test" + }, + "outputs": { + "output_json": { + "transmissionMode": ExecuteTransmissionMode.VALUE, # force convert of the file reference + "format": {"mediaType": ContentType.APP_JSON}, # request output format explicitly + } + } + } + with contextlib.ExitStack() as stack: + for mock_exec in mocked_execute_celery(): + stack.enter_context(mock_exec) + path = "/jobs" + resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, + data=exec_content, headers=exec_headers, only_local=True) + assert resp.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" + assert "Preference-Applied" not in resp.headers + + # rely on location that should be provided to find the job ID + results_url = get_header("Content-Location", resp.headers) + assert results_url, ( + "Content-Location should have been provided in" + "results response pointing at where they can be found." + ) + job_id = results_url.rsplit("/results")[0].rsplit("/jobs/")[-1] + assert is_uuid(job_id), f"Failed to retrieve the job ID: [{job_id}] is not a UUID" + out_url = get_wps_output_url(self.settings) + + # validate the results based on original execution request + results = resp + assert results.content_type.startswith(ContentType.APP_JSON) + assert results.json == { + "output_json": { + "mediaType": ContentType.APP_JSON, + "value": {"data": "test"}, + } + } + outputs = self.app.get(f"/jobs/{job_id}/outputs", params={"schema": JobInputsOutputsSchema.OGC_STRICT}) + assert outputs.content_type.startswith(ContentType.APP_JSON) + assert outputs.json["outputs"] == { + "output_json": { + "href": f"{out_url}/{job_id}/output_json/result.json", + "type": ContentType.APP_JSON, + }, + } + + @pytest.mark.oap_part4 + def test_execute_jobs_async(self): + proc = "EchoResultsTester" + p_id = self.fully_qualified_test_name(proc) + body = self.retrieve_payload(proc, "deploy", local=True) + self.deploy_process(body, process_id=p_id) + + prefer_header = f"return={ExecuteReturnPreference.MINIMAL}, respond-async" + exec_headers = { + "Prefer": prefer_header + } + exec_headers.update(self.json_headers) + exec_content = { + "process": f"https://localhost/processes/{p_id}", + "inputs": { + "message": "test" + }, + "outputs": { + "output_json": {}, + "output_data": {} + } + } + with contextlib.ExitStack() as stack: + for mock_exec in mocked_execute_celery(): + stack.enter_context(mock_exec) + path = "/jobs" + resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, + data=exec_content, headers=exec_headers, only_local=True) + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" + assert "Preference-Applied" in resp.headers + assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") + + status_url = resp.json["location"] + status = self.monitor_job(status_url, return_status=True) + assert status["status"] == Status.SUCCEEDED + + job_id = status["jobID"] + out_url = get_wps_output_url(self.settings) + results = self.app.get(f"/jobs/{job_id}/results") + results_json = self.remove_result_format(results.json) + assert results.content_type.startswith(ContentType.APP_JSON) + assert results_json == { + "output_data": "test", + "output_json": { + "href": f"{out_url}/{job_id}/output_json/result.json", + "type": ContentType.APP_JSON, + }, + } + outputs = self.app.get(f"/jobs/{job_id}/outputs", params={"schema": JobInputsOutputsSchema.OGC_STRICT}) + assert outputs.content_type.startswith(ContentType.APP_JSON) + assert outputs.json["outputs"] == { + "output_data": { + "value": "test" + }, + "output_json": { + "href": f"{out_url}/{job_id}/output_json/result.json", + "type": ContentType.APP_JSON, + }, + } + + @pytest.mark.oap_part4 + def test_execute_jobs_create_trigger(self): + proc = "EchoResultsTester" + p_id = self.fully_qualified_test_name(proc) + body = self.retrieve_payload(proc, "deploy", local=True) + self.deploy_process(body, process_id=p_id) + + prefer_header = f"return={ExecuteReturnPreference.MINIMAL}, respond-async" + exec_headers = { + "Prefer": prefer_header + } + exec_headers.update(self.json_headers) + exec_content = { + "process": f"https://localhost/processes/{p_id}", + "status": "create", # force wait until triggered + "inputs": { + "message": "test" + }, + "outputs": { + "output_json": {}, + "output_data": {} + } + } + with contextlib.ExitStack() as stack: + for mock_exec in mocked_execute_celery(): + stack.enter_context(mock_exec) + + # create the job, with pending status (not in worker processing queue) + path = "/jobs" + resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, + data=exec_content, headers=exec_headers, only_local=True) + assert resp.status_code == 201, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" + assert "Preference-Applied" in resp.headers + assert resp.headers["Preference-Applied"] == prefer_header.replace(",", ";") + + status_url = resp.json["location"] + status = self.monitor_job(status_url, return_status=True, wait_for_status=Status.CREATED) + assert status["status"] == Status.CREATED + + # trigger the execution (submit the task to worker processing queue) + job_id = status["jobID"] + res_path = f"/jobs/{job_id}/results" + res_headers = { + "Accept": ContentType.APP_JSON, + } + resp = mocked_sub_requests(self.app, "post_json", res_path, timeout=5, + data={}, headers=res_headers, only_local=True) + assert resp.status_code == 202, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" + assert resp.json["status"] == Status.ACCEPTED + + # retrieve the execution status + status = self.monitor_job(status_url, return_status=True) + assert status["status"] == Status.SUCCEEDED + + out_url = get_wps_output_url(self.settings) + results = self.app.get(f"/jobs/{job_id}/results") + results_json = self.remove_result_format(results.json) + assert results.content_type.startswith(ContentType.APP_JSON) + assert results_json == { + "output_data": "test", + "output_json": { + "href": f"{out_url}/{job_id}/output_json/result.json", + "type": ContentType.APP_JSON, + }, + } + outputs = self.app.get(f"/jobs/{job_id}/outputs", params={"schema": JobInputsOutputsSchema.OGC_STRICT}) + assert outputs.content_type.startswith(ContentType.APP_JSON) + assert outputs.json["outputs"] == { + "output_data": { + "value": "test" + }, + "output_json": { + "href": f"{out_url}/{job_id}/output_json/result.json", + "type": ContentType.APP_JSON, + }, + } + + @pytest.mark.oap_part4 + def test_execute_jobs_process_not_found(self): + # use non-existing process to ensure this particular situation is handled as well + # a missing process reference must not cause an early "not-found" response + proc = "random-other-process" + proc = self.fully_qualified_test_name(proc) + + exec_content = { + "process": f"https://localhost/processes/{proc}", + "inputs": {"message": "test"} + } + with contextlib.ExitStack() as stack: + for mock_exec in mocked_execute_celery(): + stack.enter_context(mock_exec) + path = "/jobs" + resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, + data=exec_content, headers=self.json_headers, only_local=True) + assert resp.status_code == 404, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" + assert resp.content_type == ContentType.APP_JSON + assert resp.json["type"] == "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process" + + @pytest.mark.oap_part4 + def test_execute_jobs_process_malformed_json(self): + exec_content = { + "process": "xyz", + "inputs": {"message": "test"} + } + with contextlib.ExitStack() as stack: + for mock_exec in mocked_execute_celery(): + stack.enter_context(mock_exec) + path = "/jobs" + resp = mocked_sub_requests(self.app, "post_json", path, timeout=5, + data=exec_content, headers=self.json_headers, only_local=True) + assert resp.status_code == 400, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" + assert resp.content_type == ContentType.APP_JSON + assert resp.json["type"] == "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process" + assert resp.json["cause"] == {"in": "body", "process": "xyz"} + + @pytest.mark.oap_part4 + def test_execute_jobs_process_malformed_xml(self): + exec_content = """ + + + + """ + headers = { + "Accept": ContentType.APP_JSON, + "Content-Type": ContentType.APP_XML, + } + with contextlib.ExitStack() as stack: + for mock_exec in mocked_execute_celery(): + stack.enter_context(mock_exec) + path = "/jobs" + resp = mocked_sub_requests(self.app, "post", path, timeout=5, + data=exec_content, headers=headers, only_local=True) + assert resp.status_code == 400, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" + assert resp.content_type == ContentType.APP_JSON + assert resp.json["type"] == "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process" + assert resp.json["cause"] == {"in": "body", "ows:Identifier": None} + + @pytest.mark.oap_part4 + def test_execute_jobs_unsupported_media_type(self): + headers = { + "Accept": ContentType.APP_JSON, + "Content-Type": ContentType.TEXT_PLAIN, + } + with contextlib.ExitStack() as stack: + for mock_exec in mocked_execute_celery(): + stack.enter_context(mock_exec) + path = "/jobs" + resp = mocked_sub_requests(self.app, "post", path, timeout=5, data="", headers=headers, only_local=True) + assert resp.status_code == 415, f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" + assert resp.content_type == ContentType.APP_JSON + assert resp.json["type"] == ( + "http://www.opengis.net/def/exceptions/ogcapi-processes-4/1.0/unsupported-media-type" + ) + assert resp.json["cause"] == {"in": "headers", "name": "Content-Type", "value": ContentType.TEXT_PLAIN} + @pytest.mark.functional class WpsPackageAppWithS3BucketTest(WpsConfigBase, ResourcesUtil): @@ -5598,7 +5936,7 @@ def test_execute_application_package_process_with_bucket_results(self): proc_url = f"/processes/{self._testMethodName}/jobs" resp = mocked_sub_requests(self.app, "post_json", proc_url, timeout=5, data=exec_body, headers=self.json_headers, only_local=True) - assert resp.status_code in [200, 201], f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code in [200, 201], f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" status_url = resp.json["location"] job_id = resp.json["jobID"] @@ -5708,7 +6046,7 @@ def test_execute_with_directory_output(self): proc_url = f"/processes/{proc}/jobs" resp = mocked_sub_requests(self.app, "post_json", proc_url, timeout=5, data=exec_body, headers=self.json_headers, only_local=True) - assert resp.status_code in [200, 201], f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + assert resp.status_code in [200, 201], f"Failed with: [{resp.status_code}]\nReason:\n{resp.text}" status_url = resp.json["location"] job_id = resp.json["jobID"] @@ -5777,7 +6115,7 @@ def test_execute_with_result_representations(self): .. versionadded:: 6.0 """ proc = "EchoResultsTester" - p_id = self.fully_qualified_test_process_name(proc) + p_id = self.fully_qualified_test_name(proc) body = self.retrieve_payload(proc, "deploy", local=True) self.deploy_process(body, process_id=p_id) diff --git a/tests/functional/utils.py b/tests/functional/utils.py index 2eaea6880..45f3e50de 100644 --- a/tests/functional/utils.py +++ b/tests/functional/utils.py @@ -31,7 +31,7 @@ from weaver.processes.constants import JobInputsOutputsSchema, ProcessSchema from weaver.processes.wps_package import get_application_requirement from weaver.status import Status -from weaver.utils import fully_qualified_name, get_weaver_url, load_file +from weaver.utils import fully_qualified_name, get_path_kvp, get_weaver_url, load_file from weaver.visibility import Visibility if TYPE_CHECKING: @@ -41,6 +41,8 @@ from pyramid.config import Configurator from webtest import TestApp + from weaver.processes.constants import ProcessSchemaOGCType, ProcessSchemaOLDType, ProcessSchemaType + from weaver.status import AnyStatusType from weaver.store.mongodb import MongodbJobStore, MongodbProcessStore, MongodbServiceStore from weaver.typedefs import ( AnyRequestMethod, @@ -61,7 +63,21 @@ ReferenceType = Literal["deploy", "describe", "execute", "package", "quotation", "estimator"] -class ResourcesUtil(object): +class GenericUtils(unittest.TestCase): + def fully_qualified_test_name(self, name=""): + """ + Generates a unique name using the current test method full context name and the provided name, if any. + + Normalizes the generated name such that it can be used as a valid :term:`Process` or :term:`Service` ID. + """ + extra_name = f"-{name}" if name else "" + class_name = fully_qualified_name(self) + test_name = f"{class_name}.{self._testMethodName}{extra_name}" + test_name = test_name.replace(".", "-").replace("-_", "_").replace("_-", "-") + return test_name + + +class ResourcesUtil(GenericUtils): @classmethod def request(cls, method, url, *args, **kwargs): # type: (AnyRequestMethod, str, *Any, **Any) -> AnyResponseType @@ -271,7 +287,7 @@ def get_builtin_process_names(): return proc_names -class JobUtils(object): +class JobUtils(GenericUtils): job_store = None job_info = None # type: Iterable[Job] @@ -316,7 +332,7 @@ def assert_equal_with_jobs_diffs(self, ) -class WpsConfigBase(unittest.TestCase): +class WpsConfigBase(GenericUtils): json_headers = MappingProxyType({"Accept": ContentType.APP_JSON, "Content-Type": ContentType.APP_JSON}) html_headers = MappingProxyType({"Accept": ContentType.TEXT_HTML}) xml_headers = MappingProxyType({"Content-Type": ContentType.TEXT_XML}) @@ -363,7 +379,7 @@ def describe_process(cls, process_id, describe_schema=ProcessSchema.OGC): def deploy_process(cls, payload, # type: JSON process_id=None, # type: Optional[str] - describe_schema=ProcessSchema.OGC, # type: Literal[ProcessSchema.OGC] # noqa + describe_schema=ProcessSchema.OGC, # type: ProcessSchemaOGCType mock_requests_only_local=True, # type: bool add_package_requirement=True, # type: bool ): # type: (...) -> Tuple[ProcessDescriptionMapping, CWL] @@ -374,7 +390,7 @@ def deploy_process(cls, def deploy_process(cls, payload, # type: JSON process_id=None, # type: Optional[str] - describe_schema=ProcessSchema.OGC, # type: Literal[ProcessSchema.OLD] # noqa + describe_schema=ProcessSchema.OGC, # type: ProcessSchemaOLDType mock_requests_only_local=True, # type: bool add_package_requirement=True, # type: bool ): # type: (...) -> Tuple[ProcessDescriptionListing, CWL] @@ -384,7 +400,7 @@ def deploy_process(cls, def deploy_process(cls, payload, # type: JSON process_id=None, # type: Optional[str] - describe_schema=ProcessSchema.OGC, # type: ProcessSchema + describe_schema=ProcessSchema.OGC, # type: ProcessSchemaType mock_requests_only_local=True, # type: bool add_package_requirement=True, # type: bool ): # type: (...) -> Tuple[ProcessDescription, CWL] @@ -440,13 +456,6 @@ def _try_get_logs(self, status_url): return f"Error logs:\n{_text}" return "" - def fully_qualified_test_process_name(self, name=""): - extra_name = f"-{name}" if name else "" - class_name = fully_qualified_name(self) - test_name = f"{class_name}.{self._testMethodName}{extra_name}" - test_name = test_name.replace(".", "-").replace("-_", "_").replace("_-", "-") - return test_name - @overload def monitor_job(self, status_url, **__): # type: (str, **Any) -> ExecutionResults @@ -462,7 +471,7 @@ def monitor_job(self, timeout=None, # type: Optional[int] interval=None, # type: Optional[int] return_status=False, # type: bool - wait_for_status=None, # type: Optional[str] + wait_for_status=None, # type: Optional[AnyStatusType] expect_failed=False, # type: bool ): # type: (...) -> Union[ExecutionResults, JobStatusResponse] """ @@ -485,17 +494,16 @@ def monitor_job(self, :return: result of the successful job, or the status body if requested. :raises AssertionError: when job fails or took too long to complete. """ - wait_for_status = wait_for_status or Status.SUCCEEDED + final_status = Status.FAILED if expect_failed else (wait_for_status or Status.SUCCEEDED) def check_job_status(_resp, running=False): # type: (AnyResponseType, bool) -> bool body = _resp.json pretty = json.dumps(body, indent=2, ensure_ascii=False) - final = Status.FAILED if expect_failed else Status.SUCCEEDED - statuses = [Status.ACCEPTED, Status.RUNNING, final] if running else [final] + statuses = [Status.ACCEPTED, Status.RUNNING, final_status] if running else [final_status] assert _resp.status_code == 200, f"Execution failed:\n{pretty}\n{self._try_get_logs(status_url)}" assert body["status"] in statuses, f"Error job info:\n{pretty}\n{self._try_get_logs(status_url)}" - return body["status"] in {wait_for_status, Status.SUCCEEDED, Status.FAILED} # break condition + return body["status"] in {final_status, Status.SUCCEEDED, Status.FAILED} # break condition time.sleep(1) # small delay to ensure process execution had a chance to start before monitoring left = timeout or self.monitor_timeout @@ -518,7 +526,8 @@ def check_job_status(_resp, running=False): return resp.json def get_outputs(self, status_url): - resp = self.app.get(f"{status_url}/outputs", headers=dict(self.json_headers)) + path = get_path_kvp(f"{status_url}/outputs", schema=JobInputsOutputsSchema.OLD) + resp = self.app.get(path, headers=dict(self.json_headers)) body = resp.json pretty = json.dumps(body, indent=2, ensure_ascii=False) assert resp.status_code == 200, f"Get outputs failed:\n{pretty}\n{self._try_get_logs(status_url)}" diff --git a/tests/test_datatype.py b/tests/test_datatype.py index 051b94f59..418d259a2 100644 --- a/tests/test_datatype.py +++ b/tests/test_datatype.py @@ -1,11 +1,18 @@ import uuid from copy import deepcopy +from datetime import datetime, timedelta import pytest +from visibility import Visibility from tests import resources -from weaver.datatype import Authentication, AuthenticationTypes, DockerAuthentication, Process -from weaver.execute import ExecuteControlOption +from weaver.datatype import Authentication, AuthenticationTypes, DockerAuthentication, Job, Process, Service +from weaver.execute import ExecuteControlOption, ExecuteMode, ExecuteResponse, ExecuteReturnPreference +from weaver.formats import ContentType +from weaver.status import Status +from weaver.utils import localize_datetime, now + +TEST_UUID = uuid.uuid4() def test_package_encode_decode(): @@ -206,3 +213,179 @@ def test_process_io_schema_ignore_uri(): ]) def test_process_split_version(process_id, result): assert Process.split_version(process_id) == result + + +@pytest.mark.parametrize( + ["attribute", "value", "result"], + [ + ("user_id", TEST_UUID, TEST_UUID), + ("user_id", str(TEST_UUID), str(TEST_UUID)), + ("user_id", "not-a-uuid", "not-a-uuid"), + ("user_id", 1234, 1234), + ("user_id", 3.14, TypeError), + ("task_id", TEST_UUID, TEST_UUID), + ("task_id", str(TEST_UUID), TEST_UUID), + ("task_id", "not-a-uuid", "not-a-uuid"), + ("task_id", 1234, TypeError), + ("wps_id", TEST_UUID, TEST_UUID), + ("wps_id", str(TEST_UUID), TEST_UUID), + ("wps_id", 1234, TypeError), + ("wps_url", "https://example.com/wps", "https://example.com/wps"), + ("wps_url", 1234, TypeError), + ("execution_mode", ExecuteMode.ASYNC, ExecuteMode.ASYNC), + ("execution_mode", None, ValueError), # "auto" required if unspecified + ("execution_mode", "abc", ValueError), + ("execution_mode", 12345, ValueError), + ("execution_response", ExecuteResponse.RAW, ExecuteResponse.RAW), + ("execution_response", None, ExecuteResponse.DOCUMENT), # weaver's default + ("execution_response", "abc", ValueError), + ("execution_response", 12345, ValueError), + ("execution_return", ExecuteReturnPreference.REPRESENTATION, ExecuteReturnPreference.REPRESENTATION), + ("execution_return", None, ExecuteReturnPreference.MINIMAL), # weaver's default + ("execution_return", "abc", ValueError), + ("execution_return", 12345, ValueError), + ("execution_wait", 1234, 1234), + ("execution_wait", None, None), + ("execution_wait", "abc", ValueError), + ("is_local", True, True), + ("is_local", 1, TypeError), + ("is_local", None, TypeError), + ("is_workflow", True, True), + ("is_workflow", 1, TypeError), + ("is_workflow", None, TypeError), + ("created", "2024-01-02", localize_datetime(datetime(year=2024, month=1, day=2))), + ("created", datetime(year=2024, month=1, day=2), localize_datetime(datetime(year=2024, month=1, day=2))), + ("created", "abc", ValueError), + ("created", 12345, TypeError), + ("updated", "2024-01-02", localize_datetime(datetime(year=2024, month=1, day=2))), + ("updated", datetime(year=2024, month=1, day=2), localize_datetime(datetime(year=2024, month=1, day=2))), + ("updated", "abc", ValueError), + ("updated", 12345, TypeError), + ("service", Service(name="test", url="https://example.com/wps"), "test"), + ("service", "test", "test"), + ("service", 1234, TypeError), + ("service", None, TypeError), + ("process", Process(id="test", package={}), "test"), + ("process", "test", "test"), + ("process", 1234, TypeError), + ("process", None, TypeError), + ("progress", "test", TypeError), + ("process", None, TypeError), + ("progress", 123, ValueError), + ("progress", -20, ValueError), + ("progress", 50, 50), + ("progress", 2.5, 2.5), + ("statistics", {}, {}), + ("statistics", None, TypeError), + ("statistics", 1234, TypeError), + ("exceptions", [], []), + ("exceptions", {}, TypeError), + ("exceptions", "error", TypeError), + ("exceptions", None, TypeError), + ("exceptions", 1234, TypeError), + ("results", [], []), + ("results", None, TypeError), + ("results", 1234, TypeError), + ("logs", [], []), + ("logs", "info", TypeError), + ("logs", None, TypeError), + ("logs", 1234, TypeError), + ("tags", [], []), + ("tags", "test", TypeError), + ("tags", None, TypeError), + ("tags", 1234, TypeError), + ("title", "test", "test"), + ("title", None, None), + ("title", TypeError, TypeError), + ("title", 1234, TypeError), + ("status", Status.SUCCEEDED, Status.SUCCEEDED), + ("status", 12345678, ValueError), + ("status", "random", ValueError), + ("status_message", None, "no message"), + ("status_message", "test", "test"), + ("status_message", 123456, TypeError), + ("status_location", f"https://example.com/jobs/{TEST_UUID}", f"https://example.com/jobs/{TEST_UUID}"), + ("status_location", None, TypeError), + ("status_location", 123456, TypeError), + ("accept_type", None, TypeError), + ("accept_type", 123456, TypeError), + ("accept_type", ContentType.APP_JSON, ContentType.APP_JSON), + ("accept_language", None, TypeError), + ("accept_language", 123456, TypeError), + ("accept_language", "en", "en"), + ("access", Visibility.PRIVATE, Visibility.PRIVATE), + ("access", 12345678, ValueError), + ("access", "random", ValueError), + ("access", None, ValueError), + ("context", "test", "test"), + ("context", None, None), + ("context", 1234, TypeError), + ] +) +def test_job_attribute_setter(attribute, value, result): + job = Job(task_id="test") + if isinstance(result, type) and issubclass(result, Exception): + with pytest.raises(result): + setattr(job, attribute, value) + else: + setattr(job, attribute, value) + assert job[attribute] == result + + +@pytest.mark.parametrize( + ["value", "result"], + [ + (TEST_UUID, TEST_UUID), + (str(TEST_UUID), TEST_UUID), + ("not-a-uuid", ValueError), + (12345, TypeError), + + ] +) +def test_job_id(value, result): + if isinstance(result, type) and issubclass(result, Exception): + with pytest.raises(result): + Job(task_id="test", id=value) + else: + job = Job(task_id="test", id=value) + assert job.id == result + + +def test_job_updated_auto(): + min_dt = now() + job = Job(task_id="test") + update_dt = job.updated + assert isinstance(update_dt, datetime) + assert update_dt > min_dt + assert update_dt == job.updated, "Updated time auto generated should have been set to avoid always regenerating it." + + +def test_job_updated_status(): + created = now() + started = now() + timedelta(seconds=1) + finished = now() + timedelta(seconds=2) + # date-times cannot be set in advance in job, + # otherwise 'updated' detects and returns them automatically + job = Job(task_id="test") + job.created = created + job.status = Status.ACCEPTED + assert job.updated == created + job["updated"] = None # reset to test auto resolve + job.started = started + job.status = Status.STARTED + assert job.updated == started + job["updated"] = None # reset to test auto resolve + job.finished = finished + job.status = Status.SUCCEEDED + assert job.updated == finished + + +def test_job_execution_wait_ignored_async(): + job = Job(task_id="test", execution_wait=1234, execution_mode=ExecuteMode.ASYNC) + assert job.execution_mode == ExecuteMode.ASYNC + assert job.execution_wait is None, "Because of async explicitly set, wait time does not apply" + + +def test_job_display(): + job = Job(task_id=TEST_UUID, id=TEST_UUID) + assert str(job) == f"Job <{TEST_UUID}>" diff --git a/tests/test_execute.py b/tests/test_execute.py index 69f8685f0..3696d48ff 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -3,7 +3,14 @@ import pytest from pyramid.httpexceptions import HTTPBadRequest -from weaver.execute import ExecuteControlOption, ExecuteMode, ExecuteReturnPreference, parse_prefer_header_execute_mode +from weaver.datatype import Job +from weaver.execute import ( + ExecuteControlOption, + ExecuteMode, + ExecuteReturnPreference, + parse_prefer_header_execute_mode, + update_preference_applied_return_header +) @pytest.mark.parametrize( @@ -20,12 +27,16 @@ for (_headers, _support, _expected), _extra in itertools.product( [ - # no mode + # no mode (API-wide default async) ({"Prefer": "respond-async, wait=4"}, [], (ExecuteMode.ASYNC, None, {})), # both modes supported (sync attempted upto max/specified wait time, unless async requested explicitly) + ({"Prefer": ""}, None, # explicit 'None' or omitting the parameter entirely means "any" mode + (ExecuteMode.SYNC, 10, {})), ({"Prefer": ""}, [ExecuteControlOption.ASYNC, ExecuteControlOption.SYNC], (ExecuteMode.SYNC, 10, {})), + ({"Prefer": "respond-async"}, None, + (ExecuteMode.ASYNC, None, {"Preference-Applied": "respond-async"})), ({"Prefer": "respond-async"}, [ExecuteControlOption.ASYNC, ExecuteControlOption.SYNC], (ExecuteMode.ASYNC, None, {"Preference-Applied": "respond-async"})), ({"Prefer": "respond-async, wait=4"}, [ExecuteControlOption.ASYNC, ExecuteControlOption.SYNC], @@ -98,3 +109,46 @@ def test_parse_prefer_header_execute_mode_invalid(prefer_header): headers = {"Prefer": prefer_header} with pytest.raises(HTTPBadRequest): parse_prefer_header_execute_mode(headers, [ExecuteControlOption.ASYNC]) + + +@pytest.mark.parametrize( + ["job_return", "request_headers", "response_headers", "expect_headers"], + [ + ( + None, + {}, + {}, + {}, + ), + ( + None, + {"Prefer": "respond-async, wait=4"}, + {}, + {}, + ), + ( + None, + {"Prefer": f"return={ExecuteReturnPreference.MINIMAL}"}, + {}, + {"Preference-Applied": f"return={ExecuteReturnPreference.MINIMAL}"}, + ), + ( + ExecuteReturnPreference.MINIMAL, + {"Prefer": f"return={ExecuteReturnPreference.REPRESENTATION}"}, + {}, + {}, + ), + ( + ExecuteReturnPreference.MINIMAL, + {"Prefer": f"return={ExecuteReturnPreference.MINIMAL}"}, + {"Preference-Applied": "respond-async"}, + {"Preference-Applied": f"return={ExecuteReturnPreference.MINIMAL}; respond-async"}, + ), + ] +) +def test_update_preference_applied_return_header(job_return, request_headers, response_headers, expect_headers): + job = Job(task_id="test") + if job_return: + job.execution_return = job_return + update_headers = update_preference_applied_return_header(job, request_headers, response_headers) + assert update_headers == expect_headers diff --git a/tests/test_utils.py b/tests/test_utils.py index d046f9481..1154459e6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -109,6 +109,8 @@ AWS_S3_REGION_SUBSET_WITH_MOCK = {MOCK_AWS_REGION} | AWS_S3_REGION_SUBSET AWS_S3_REGION_NON_DEFAULT = list(AWS_S3_REGION_SUBSET_WITH_MOCK - {MOCK_AWS_REGION})[0] +KNOWN_STATUSES = set(Status.values()) - {Status.UNKNOWN} + # pylint: disable=R1732,W1514 # not using with open + encoding @@ -389,37 +391,30 @@ def test_pass_http_error_raises_other_error_with_multi_pyramid_error(): pass_http_error(ex, [HTTPConflict, HTTPInternalServerError]) -def get_status_variations(status_value): - return [status_value.lower(), - status_value.upper(), - status_value.capitalize(), - f"Process{status_value.capitalize()}"] - - -def test_map_status_ogc_compliant(): - known_statuses = set(Status.values()) - {Status.UNKNOWN} - for sv in known_statuses: - for s in get_status_variations(sv): - assert map_status(s, StatusCompliant.OGC) in JOB_STATUS_CATEGORIES[StatusCompliant.OGC] - - -def test_map_status_pywps_compliant(): - known_statuses = set(Status.values()) - {Status.UNKNOWN} - for sv in known_statuses: - for s in get_status_variations(sv): - assert map_status(s, StatusCompliant.PYWPS) in JOB_STATUS_CATEGORIES[StatusCompliant.PYWPS] - - -def test_map_status_owslib_compliant(): - known_statuses = set(Status.values()) - {Status.UNKNOWN} - for sv in known_statuses: - for s in get_status_variations(sv): - assert map_status(s, StatusCompliant.OWSLIB) in JOB_STATUS_CATEGORIES[StatusCompliant.OWSLIB] +@pytest.mark.parametrize( + ["compliance", "status"], + itertools.product( + list(StatusCompliant), + itertools.chain.from_iterable( + [ + status.lower(), + status.upper(), + status.capitalize(), + f"Process{status.capitalize()}" + ] + for status in KNOWN_STATUSES + ) + ) +) +def test_map_status_compliant(compliance, status): + # type: (StatusCompliant, str) -> None + assert map_status(status, compliance) in JOB_STATUS_CATEGORIES[compliance] def test_map_status_back_compatibility_and_special_cases(): - for c in StatusCompliant: + for c in (set(StatusCompliant.values()) - {StatusCompliant.OPENEO}): # type: ignore assert map_status("successful", c) == Status.SUCCEEDED + assert map_status("successful", StatusCompliant.OPENEO) == Status.FINISHED def test_map_status_pywps_compliant_as_int_statuses(): @@ -659,6 +654,7 @@ def mock_sleep(delay): assert all(called == expect for called, expect in zip(sleep_counter["called_with"], intervals)) +@pytest.mark.flaky(reruns=2, reruns_delay=1) def test_request_extra_zero_values(): """ Test that zero-value ``retries`` and ``backoff`` are not ignored. diff --git a/tests/wps_restapi/test_api.py b/tests/wps_restapi/test_api.py index 16af9d0b9..061c200f4 100644 --- a/tests/wps_restapi/test_api.py +++ b/tests/wps_restapi/test_api.py @@ -160,6 +160,22 @@ def test_openapi_includes_schema(self): assert "$id" in body["components"]["schemas"]["CWL"] assert body["components"]["schemas"]["CWL"]["$id"] == sd.CWL_SCHEMA_URL + def test_openapi_jobs_create_description(self): + """ + Ensure the correct docstring is picked up by multiple service decorators across view functions. + + .. seealso:: + - :func:`weaver.wps_restapi.jobs.jobs.create_job` + - :func:`weaver.wps_restapi.jobs.jobs.create_job_unsupported_media_type` + """ + resp = self.app.get(sd.openapi_json_service.path, headers=self.json_headers) + assert resp.status_code == 200 + body = resp.json + + for field in ["summary", "description"]: + desc = body["paths"][sd.jobs_service.path]["post"].get(field, "") + assert not desc or "Create a new processing job" in desc + def test_status_unauthorized_and_forbidden(self): """ Validates that 401/403 status codes are correctly handled and that the appropriate one is returned. diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index 5d8219831..8efdafa42 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -5,7 +5,6 @@ import os import shutil import tempfile -import unittest import warnings from datetime import date from typing import TYPE_CHECKING @@ -31,10 +30,17 @@ setup_mongodb_servicestore ) from weaver.compat import Version -from weaver.datatype import Job, Service -from weaver.execute import ExecuteMode, ExecuteResponse, ExecuteTransmissionMode +from weaver.datatype import Job, Process, Service +from weaver.execute import ( + ExecuteControlOption, + ExecuteMode, + ExecuteResponse, + ExecuteReturnPreference, + ExecuteTransmissionMode +) from weaver.formats import ContentType from weaver.notify import decrypt_email +from weaver.processes.constants import JobStatusSchema from weaver.processes.wps_testing import WpsTestProcess from weaver.status import JOB_STATUS_CATEGORIES, Status, StatusCategory from weaver.utils import get_path_kvp, now @@ -49,14 +55,14 @@ ) if TYPE_CHECKING: - from typing import Iterable, List, Optional, Tuple, Union + from typing import Any, Iterable, List, Optional, Tuple, Union from weaver.status import AnyStatusType - from weaver.typedefs import AnyLogLevel, JSON, Number, Statistics + from weaver.typedefs import AnyLogLevel, JobResults, JSON, Number, Statistics from weaver.visibility import AnyVisibility -class WpsRestApiJobsTest(unittest.TestCase, JobUtils): +class WpsRestApiJobsTest(JobUtils): settings = {} config = None @@ -87,6 +93,9 @@ def setUp(self): self.process_public = WpsTestProcess(identifier="process-public") self.process_store.save_process(self.process_public) self.process_store.set_visibility(self.process_public.identifier, Visibility.PUBLIC) + proc_pub = self.process_store.fetch_by_id("process-public") + proc_pub["jobControlOptions"] = [ExecuteControlOption.ASYNC, ExecuteControlOption.SYNC] + self.process_store.save_process(proc_pub) self.process_private = WpsTestProcess(identifier="process-private") self.process_store.save_process(self.process_private) self.process_store.set_visibility(self.process_private.identifier, Visibility.PRIVATE) @@ -161,26 +170,31 @@ def setUp(self): user_id=self.user_editor1_id, status=Status.STARTED, progress=99, access=Visibility.PUBLIC) def make_job(self, + *, # force keyword arguments task_id, # type: str process, # type: str service, # type: Optional[str] - user_id, # type: Optional[int] status, # type: AnyStatusType progress, # type: int - access, # type: AnyVisibility + access=None, # type: AnyVisibility + user_id=None, # type: Optional[int] created=None, # type: Optional[Union[datetime.datetime, str]] offset=None, # type: Optional[int] duration=None, # type: Optional[int] exceptions=None, # type: Optional[List[JSON]] logs=None, # type: Optional[List[Union[str, Tuple[str, AnyLogLevel, AnyStatusType, Number]]]] statistics=None, # type: Optional[Statistics] + results=None, # type: Optional[JobResults] tags=None, # type: Optional[List[str]] add_info=True, # type: bool + **job_params, # type: Any ): # type: (...) -> Job if isinstance(created, str): created = date_parser.parse(created) - job = self.job_store.save_job(task_id=task_id, process=process, service=service, is_workflow=False, - execute_async=True, user_id=user_id, access=access, created=created) + job = self.job_store.save_job( + task_id=task_id, process=process, service=service, is_workflow=False, user_id=user_id, + access=access, created=created, **job_params + ) job.status = status if status != Status.ACCEPTED: job.started = job.created + datetime.timedelta(seconds=offset if offset is not None else 0) @@ -198,6 +212,8 @@ def make_job(self, job.exceptions = exceptions if statistics is not None: job.statistics = statistics + if results is not None: + job.results = results if tags is not None: job.tags = tags job = self.job_store.update_job(job) @@ -281,6 +297,7 @@ def check_basic_jobs_grouped_info(response, groups): total += grouped_jobs["count"] assert total == response.json["total"] + @pytest.mark.oap_part1 def test_get_jobs_normal_paged(self): resp = self.app.get(sd.jobs_service.path, headers=self.json_headers) self.check_basic_jobs_info(resp) @@ -322,6 +339,8 @@ def test_get_jobs_detail_grouped(self): for job in grouped_jobs["jobs"]: self.check_job_format(job) + @pytest.mark.html + @pytest.mark.oap_part1 @parameterized.expand([ ({}, ), # detail omitted should apply it for HTML, unlike JSON that returns the simplified listing by default ({"detail": None}, ), @@ -347,6 +366,7 @@ def test_get_jobs_detail_html_enforced(self, params): jobs = [line for line in resp.text.splitlines() if "job-list-item" in line] assert len(jobs) == 6 + @pytest.mark.html def test_get_jobs_groups_html_unsupported(self): groups = ["process", "service"] path = get_path_kvp(sd.jobs_service.path, groups=groups) @@ -424,6 +444,7 @@ def test_get_jobs_valid_grouping_by_provider(self): """ self.template_get_jobs_valid_grouping_by_service_provider("provider") + @pytest.mark.oap_part1 def test_get_jobs_links_navigation(self): """ Verifies that relation links update according to context in order to allow natural navigation between responses. @@ -543,6 +564,7 @@ def test_get_jobs_links_navigation(self): assert links["first"].startswith(jobs_url) and limit_kvp in links["first"] and "page=0" in links["first"] assert links["last"].startswith(jobs_url) and limit_kvp in links["last"] and "page=0" in links["last"] + @pytest.mark.oap_part1 def test_get_jobs_page_out_of_range(self): resp = self.app.get(sd.jobs_service.path, headers=self.json_headers) total = resp.json["total"] @@ -607,8 +629,8 @@ def test_get_jobs_by_encrypted_email(self): # verify the email is not in plain text job = self.job_store.fetch_by_id(job_id) - assert job.notification_email != email and job.notification_email is not None - assert decrypt_email(job.notification_email, self.settings) == email, "Email should be recoverable." + assert job.notification_email != email and job.notification_email is not None # noqa + assert decrypt_email(job.notification_email, self.settings) == email, "Email should be recoverable." # noqa # make sure that jobs searched using email are found with encryption transparently for the user path = get_path_kvp(sd.jobs_service.path, detail="true", notification_email=email) @@ -618,6 +640,7 @@ def test_get_jobs_by_encrypted_email(self): assert resp.json["total"] == 1, "Should match exactly 1 email with specified literal string as query param." assert resp.json["jobs"][0]["jobID"] == job_id + @pytest.mark.oap_part1 def test_get_jobs_by_type_process(self): path = get_path_kvp(sd.jobs_service.path, type="process") resp = self.app.get(path, headers=self.json_headers) @@ -756,6 +779,7 @@ def test_get_jobs_process_unknown_in_query(self): assert resp.status_code == 404 assert resp.content_type == ContentType.APP_JSON + @pytest.mark.oap_part1 @parameterized.expand([ get_path_kvp( sd.jobs_service.path, @@ -859,9 +883,9 @@ def test_get_jobs_private_service_public_process_forbidden_access_in_query(self) def test_get_jobs_public_service_private_process_forbidden_access_in_query(self): """ - NOTE: - it is up to the remote service to hide private processes - if the process is visible, the a job can be executed and it is automatically considered public + .. note:: + It is up to the remote service to hide private processes. + If the process is visible, the job can be executed and it is automatically considered public. """ path = get_path_kvp(sd.jobs_service.path, service=self.service_public.name, @@ -875,9 +899,9 @@ def test_get_jobs_public_service_private_process_forbidden_access_in_query(self) def test_get_jobs_public_service_no_processes(self): """ - NOTE: - it is up to the remote service to hide private processes - if the process is invisible, no job should have been executed nor can be fetched + .. note:: + It is up to the remote service to hide private processes. + If the process is invisible, no job should have been executed nor can be fetched. """ path = get_path_kvp(sd.jobs_service.path, service=self.service_public.name, @@ -962,6 +986,7 @@ def filter_service(jobs): # type: (Iterable[Job]) -> List[Job] test_values = {"path": path, "access": access, "user_id": user_id} self.assert_equal_with_jobs_diffs(job_result, job_expect, test_values, index=i) + @pytest.mark.oap_part1 def test_jobs_list_with_limit_api(self): """ Test handling of ``limit`` query parameter when listing jobs. @@ -980,6 +1005,7 @@ def test_jobs_list_with_limit_api(self): assert resp.json["limit"] == limit_parameter assert len(resp.json["jobs"]) <= limit_parameter + @pytest.mark.oap_part1 def test_jobs_list_schema_not_required_fields(self): """ Test that job listing query parameters for filtering results are marked as optional in OpenAPI schema. @@ -1102,6 +1128,7 @@ def test_get_jobs_datetime_interval(self): assert date_parser.parse(resp.json["created"]) >= date_parser.parse(datetime_after) assert date_parser.parse(resp.json["created"]) <= date_parser.parse(datetime_before) + @pytest.mark.oap_part1 def test_get_jobs_datetime_match(self): """ Test that only filtered jobs at a specific time are returned when ``datetime`` query parameter is provided. @@ -1125,6 +1152,7 @@ def test_get_jobs_datetime_match(self): assert resp.content_type == ContentType.APP_JSON assert date_parser.parse(resp.json["created"]) == date_parser.parse(datetime_match) + @pytest.mark.oap_part1 def test_get_jobs_datetime_invalid(self): """ Test that incorrectly formatted ``datetime`` query parameter value is handled. @@ -1142,6 +1170,7 @@ def test_get_jobs_datetime_invalid(self): resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 400 + @pytest.mark.oap_part1 def test_get_jobs_datetime_interval_invalid(self): """ Test that invalid ``datetime`` query parameter value is handled. @@ -1159,6 +1188,7 @@ def test_get_jobs_datetime_interval_invalid(self): resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 422 + @pytest.mark.oap_part1 def test_get_jobs_datetime_before_invalid(self): """ Test that invalid ``datetime`` query parameter value with a range is handled. @@ -1175,6 +1205,7 @@ def test_get_jobs_datetime_before_invalid(self): resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 400 + @pytest.mark.oap_part1 def test_get_jobs_duration_min_only(self): test = {"minDuration": 35} path = get_path_kvp(sd.jobs_service.path, **test) @@ -1201,6 +1232,7 @@ def test_get_jobs_duration_min_only(self): expect_jobs = [self.job_info[i].id for i in [8]] self.assert_equal_with_jobs_diffs(result_jobs, expect_jobs, test) + @pytest.mark.oap_part1 def test_get_jobs_duration_max_only(self): test = {"maxDuration": 30} path = get_path_kvp(sd.jobs_service.path, **test) @@ -1222,6 +1254,7 @@ def test_get_jobs_duration_max_only(self): expect_jobs = [self.job_info[i].id for i in expect_idx] self.assert_equal_with_jobs_diffs(result_jobs, expect_jobs, test) + @pytest.mark.oap_part1 def test_get_jobs_duration_min_max(self): # note: avoid range <35s for this test to avoid sudden dynamic duration of 9, 10 becoming within min/max test = {"minDuration": 35, "maxDuration": 60} @@ -1247,6 +1280,7 @@ def test_get_jobs_duration_min_max(self): result_jobs = resp.json["jobs"] assert len(result_jobs) == 0 + @pytest.mark.oap_part1 def test_get_jobs_duration_min_max_invalid(self): test = {"minDuration": 30, "maxDuration": 20} path = get_path_kvp(sd.jobs_service.path, **test) @@ -1268,6 +1302,7 @@ def test_get_jobs_duration_min_max_invalid(self): resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code in [400, 422] + @pytest.mark.oap_part1 def test_get_jobs_by_status_single(self): test = {"status": Status.SUCCEEDED} path = get_path_kvp(sd.jobs_service.path, **test) @@ -1285,6 +1320,7 @@ def test_get_jobs_by_status_single(self): result_jobs = resp.json["jobs"] self.assert_equal_with_jobs_diffs(result_jobs, expect_jobs, test) + @pytest.mark.oap_part1 def test_get_jobs_by_status_multi(self): test = {"status": f"{Status.SUCCEEDED},{Status.RUNNING}"} path = get_path_kvp(sd.jobs_service.path, **test) @@ -1310,6 +1346,7 @@ def test_get_jobs_by_status_invalid(self): assert resp.json["value"]["status"] == status assert "status" in resp.json["cause"] + @pytest.mark.oap_part1 def test_get_job_status_response_process_id(self): """ Verify the processID value in the job status response. @@ -1330,6 +1367,7 @@ def test_get_job_status_response_process_id(self): assert resp.json["processID"] == "process-public" + @pytest.mark.oap_part1 def test_get_job_invalid_uuid(self): """ Test handling of invalid UUID reference to search job. @@ -1348,6 +1386,7 @@ def test_get_job_invalid_uuid(self): assert resp.json["type"].endswith("no-such-job") assert "UUID" in resp.json["detail"] + @pytest.mark.oap_part1 @mocked_dismiss_process() def test_job_dismiss_running_single(self): """ @@ -1386,6 +1425,7 @@ def test_job_dismiss_running_single(self): assert resp.status_code == 410, "Job cannot be dismissed again." assert job.id in resp.json["value"] + @pytest.mark.oap_part1 @mocked_dismiss_process() def test_job_dismiss_complete_single(self): """ @@ -1470,7 +1510,7 @@ def test_job_dismiss_batch(self): def test_job_results_errors(self): """ - Validate errors returned for a incomplete, failed or dismissed job when requesting its results. + Validate errors returned for an incomplete, failed or dismissed job when requesting its results. """ job_accepted = self.make_job( task_id="1111-0000-0000-0000", process=self.process_public.identifier, service=None, @@ -1635,6 +1675,7 @@ def test_jobs_inputs_outputs_validations(self): with self.assertRaises(colander.Invalid): sd.Execute().deserialize({"outputs": {"random": {"transmissionMode": "bad"}}}) + @pytest.mark.oap_part4 def test_job_logs_formats(self): path = f"/jobs/{self.job_info[0].id}/logs" resp = self.app.get(path, headers=self.json_headers) @@ -1701,6 +1742,7 @@ def test_job_logs_formats(self): assert "Process" in lines[1] assert "Complete" in lines[2] + @pytest.mark.oap_part4 def test_job_logs_formats_unsupported(self): path = f"/jobs/{self.job_info[0].id}/logs" resp = self.app.get(path, headers={"Accept": ContentType.IMAGE_GEOTIFF}, expect_errors=True) @@ -1740,7 +1782,380 @@ def test_job_statistics_response(self): if job: self.job_store.delete_job(job.id) + @pytest.mark.oap_part4 + def test_job_inputs_response(self): + new_job = self.make_job( + task_id=self.fully_qualified_test_name(), process=self.process_public.identifier, service=None, + status=Status.RUNNING, progress=50, access=Visibility.PRIVATE, context="test/context", + inputs={"test": "data"}, outputs={"test": {"transmissionMode": ExecuteTransmissionMode.VALUE}}, + ) + + path = f"/jobs/{new_job.id}/inputs" + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.json["inputs"] == {"test": "data"} + assert resp.json["outputs"] == {"test": {"transmissionMode": ExecuteTransmissionMode.VALUE}} + assert resp.json["headers"] == { + "Accept": None, + "Accept-Language": None, + "Content-Type": None, + "Prefer": f"return={ExecuteReturnPreference.MINIMAL}", + "X-WPS-Output-Context": "test/context", + } + assert resp.json["mode"] == ExecuteMode.AUTO + assert resp.json["response"] == ExecuteResponse.DOCUMENT + + assert "subscribers" not in resp.json, "Subscribers must not be exposed due to potentially sensible data" + + @pytest.mark.oap_part4 + def test_job_outputs_response(self): + new_job = self.make_job( + task_id=self.fully_qualified_test_name(), process=self.process_public.identifier, service=None, + status=Status.SUCCEEDED, progress=100, access=Visibility.PRIVATE, context="test/context", + results=[{"id": "test", "value": "data"}], + ) + + path = f"/jobs/{new_job.id}/outputs" + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.json["outputs"] == {"test": {"value": "data"}} + + @pytest.mark.oap_part4 + @pytest.mark.xfail(reason="CWL PROV not implemented (https://github.com/crim-ca/weaver/issues/673)") + def test_job_run_response(self): + raise NotImplementedError # FIXME (https://github.com/crim-ca/weaver/issues/673) + + @pytest.mark.oap_part4 + @parameterized.expand([Status.ACCEPTED, Status.RUNNING, Status.FAILED, Status.SUCCEEDED]) + def test_job_update_locked(self, status): + new_job = self.make_job( + task_id=self.fully_qualified_test_name(), process=self.process_public.identifier, service=None, + status=status, progress=100, access=Visibility.PUBLIC, + inputs={"test": "data"}, outputs={"test": {"transmissionMode": ExecuteTransmissionMode.VALUE}}, + ) + path = f"/jobs/{new_job.id}" + body = {"inputs": {"test": 400}} + resp = self.app.patch_json(path, params=body, headers=self.json_headers, expect_errors=True) + assert resp.status_code == 423 + assert resp.json["type"] == "http://www.opengis.net/def/exceptions/ogcapi-processes-4/1.0/locked" + + @pytest.mark.oap_part4 + def test_job_update_unsupported_media_type(self): + new_job = self.make_job( + task_id=self.fully_qualified_test_name(), process=self.process_public.identifier, service=None, + status=Status.CREATED, progress=0, access=Visibility.PUBLIC, + ) + path = f"/jobs/{new_job.id}" + resp = self.app.patch(path, params="data", expect_errors=True) + assert resp.status_code == 415 + assert resp.content_type == ContentType.APP_JSON + assert resp.json["type"] == ( + "http://www.opengis.net/def/exceptions/ogcapi-processes-4/1.0/unsupported-media-type" + ) + + @pytest.mark.oap_part4 + def test_job_update_response_contents(self): + """ + Validate that :term:`Job` metadata and responses are updated with contents as requested. + """ + new_job = self.make_job( + task_id=self.fully_qualified_test_name(), process=self.process_public.identifier, service=None, + status=Status.CREATED, progress=0, access=Visibility.PUBLIC, + inputs={"test": "data"}, outputs={"test": {"transmissionMode": ExecuteTransmissionMode.VALUE}}, + subscribers={"successUri": "https://example.com/random"}, + ) + + # check precondition job setup + path = f"/jobs/{new_job.id}/inputs" + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.json["inputs"] == {"test": "data"} + assert resp.json["outputs"] == {"test": {"transmissionMode": ExecuteTransmissionMode.VALUE}} + assert resp.json["headers"] == { + "Accept": None, + "Accept-Language": None, + "Content-Type": None, + "Prefer": f"return={ExecuteReturnPreference.MINIMAL}", + "X-WPS-Output-Context": None, + } + assert resp.json["mode"] == ExecuteMode.AUTO + assert resp.json["response"] == ExecuteResponse.DOCUMENT + + # modify job definition + path = f"/jobs/{new_job.id}" + body = { + "inputs": {"test": "modified", "new": 123}, + "outputs": {"test": {"transmissionMode": ExecuteTransmissionMode.REFERENCE}}, + "subscribers": { + "successUri": "https://example.com/success", + "failedUri": "https://example.com/failed", + }, + } + headers = { + "Accept": ContentType.APP_JSON, + "Content-Type": ContentType.APP_JSON, + "Prefer": f"return={ExecuteReturnPreference.REPRESENTATION}; wait=5", + } + resp = self.app.patch_json(path, params=body, headers=headers) + assert resp.status_code == 204 + + # validate changes applied and resolved accordingly + path = f"/jobs/{new_job.id}/inputs" + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.json["inputs"] == {"test": "modified", "new": 123} + assert resp.json["outputs"] == {"test": {"transmissionMode": ExecuteTransmissionMode.REFERENCE}} + assert resp.json["headers"] == { + "Accept": None, + "Accept-Language": None, + "Content-Type": None, + "Prefer": f"return={ExecuteReturnPreference.REPRESENTATION}; wait=5", + "X-WPS-Output-Context": None + } + assert resp.json["mode"] == ExecuteMode.SYNC, "Should have been modified from 'wait' preference." + assert resp.json["response"] == ExecuteResponse.RAW, "Should have been modified from 'return' preference." + + assert "subscribers" not in resp.json, "Subscribers must not be exposed due to potentially sensible data" + test_job = self.job_store.fetch_by_id(new_job.id) + assert test_job.subscribers == { + "callbacks": { + Status.SUCCEEDED: "https://example.com/success", + Status.FAILED: "https://example.com/failed", + } + } + + @pytest.mark.oap_part4 + def test_job_update_execution_parameters(self): + """ + Test modification of the execution ``return`` and ``response`` options, going back-and-forth between approaches. + """ + new_job = self.make_job( + task_id=self.fully_qualified_test_name(), process=self.process_public.identifier, service=None, + status=Status.CREATED, progress=0, access=Visibility.PUBLIC, + execute_mode=ExecuteMode.AUTO, + execute_response=ExecuteResponse.DOCUMENT, + ) + + body = {} + path = f"/jobs/{new_job.id}" + headers = { + "Prefer": f"return={ExecuteReturnPreference.REPRESENTATION}", + } + resp = self.app.patch_json(path, params=body, headers=headers) + assert resp.status_code == 204 + + path = f"/jobs/{new_job.id}/inputs" + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.json["mode"] == ExecuteMode.AUTO + assert resp.json["response"] == ExecuteResponse.RAW + assert resp.json["headers"]["Prefer"] == f"return={ExecuteReturnPreference.REPRESENTATION}" + + body = {"response": ExecuteResponse.DOCUMENT} + path = f"/jobs/{new_job.id}" + resp = self.app.patch_json(path, params=body, headers=self.json_headers) + assert resp.status_code == 204 + + path = f"/jobs/{new_job.id}/inputs" + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.json["mode"] == ExecuteMode.AUTO + assert resp.json["response"] == ExecuteResponse.DOCUMENT + assert resp.json["headers"]["Prefer"] == f"return={ExecuteReturnPreference.MINIMAL}" + + body = {} + headers = { + "Prefer": f"return={ExecuteReturnPreference.REPRESENTATION}", + } + path = f"/jobs/{new_job.id}" + resp = self.app.patch_json(path, params=body, headers=headers) + assert resp.status_code == 204 + + path = f"/jobs/{new_job.id}/inputs" + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.json["mode"] == ExecuteMode.AUTO + assert resp.json["response"] == ExecuteResponse.RAW + assert resp.json["headers"]["Prefer"] == f"return={ExecuteReturnPreference.REPRESENTATION}" + + body = {"response": ExecuteResponse.RAW} + path = f"/jobs/{new_job.id}" + resp = self.app.patch_json(path, params=body, headers=self.json_headers) + assert resp.status_code == 204 + + path = f"/jobs/{new_job.id}/inputs" + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.json["mode"] == ExecuteMode.AUTO + assert resp.json["response"] == ExecuteResponse.RAW + assert resp.json["headers"]["Prefer"] == f"return={ExecuteReturnPreference.REPRESENTATION}" + + @pytest.mark.oap_part4 + def test_job_update_subscribers(self): + new_job = self.make_job( + task_id=self.fully_qualified_test_name(), process=self.process_public.identifier, service=None, + status=Status.CREATED, progress=0, access=Visibility.PUBLIC, + subscribers={"successUri": "https://example.com/random"}, + ) + + # check that subscribers can be removed even if not communicated in job status responses + body = {"subscribers": {}} + path = f"/jobs/{new_job.id}" + resp = self.app.patch_json(path, params=body, headers=self.json_headers) + assert resp.status_code == 204 + + test_job = self.job_store.fetch_by_id(new_job.id) + assert test_job.subscribers is None + + @pytest.mark.oap_part4 + @pytest.mark.openeo + def test_job_update_title(self): + new_job = self.make_job( + task_id=self.fully_qualified_test_name(), process=self.process_public.identifier, service=None, + status=Status.CREATED, progress=0, access=Visibility.PUBLIC, + ) + + path = f"/jobs/{new_job.id}" + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert "title" not in resp.json + + title = "The new title!" + body = {"title": title} + resp = self.app.patch_json(path, params=body, headers=self.json_headers) + assert resp.status_code == 204 + + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.json["title"] == title + + body = {"title": None} + resp = self.app.patch_json(path, params=body, headers=self.json_headers) + assert resp.status_code == 204 + + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert "title" not in resp.json + + body = {"title": ""} + resp = self.app.patch_json(path, params=body, headers=self.json_headers, expect_errors=True) + assert resp.status_code == 422 + assert "title.JobTitle" in resp.json["cause"] + + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert "title" not in resp.json + + @pytest.mark.oap_part4 + def test_job_update_response_process_disallowed(self): + proc_id = self.fully_qualified_test_name() + process = WpsTestProcess(identifier=proc_id) + process = Process.from_wps(process) + process["processDescriptionURL"] = f"https://localhost/processes/{proc_id}" + self.process_store.save_process(process) + + new_job = self.make_job( + task_id=self.fully_qualified_test_name(), process=proc_id, service=None, + status=Status.CREATED, progress=0, access=Visibility.PUBLIC, + ) + + path = f"/jobs/{new_job.id}" + body = {"process": "https://localhost/processes/random"} + resp = self.app.patch_json(path, params=body, headers=self.json_headers, expect_errors=True) + assert resp.status_code == 400 + assert resp.json["cause"] == {"name": "process", "in": "body"} + assert resp.json["value"] == { + "body.process": "https://localhost/processes/random", + "job.process": f"https://localhost/processes/{proc_id}", + } + + @pytest.mark.oap_part4 + @pytest.mark.openeo + def test_job_status_alt_openeo_accept_response(self): + """ + Validate retrieval of :term:`Job` status response with alternate value mapping by ``Accept`` header. + """ + job = self.job_info[0] + assert job.status == Status.SUCCEEDED, "Precondition invalid." + headers = {"Accept": f"{ContentType.APP_JSON}; profile={JobStatusSchema.OPENEO}"} + path = f"/jobs/{job.id}" + resp = self.app.get(path, headers=headers) + assert resp.status_code == 200 + assert resp.headers["Content-Type"] == f"{ContentType.APP_JSON}; profile={JobStatusSchema.OPENEO}" + assert resp.headers["Content-Schema"] == sd.OPENEO_API_SCHEMA_JOB_STATUS_URL + assert resp.json["status"] == Status.FINISHED + + job = self.job_info[1] + assert job.status == Status.FAILED, "Precondition invalid." + path = f"/jobs/{job.id}" + resp = self.app.get(path, headers=headers) + assert resp.status_code == 200 + assert resp.headers["Content-Type"] == f"{ContentType.APP_JSON}; profile={JobStatusSchema.OPENEO}" + assert resp.headers["Content-Schema"] == sd.OPENEO_API_SCHEMA_JOB_STATUS_URL + assert resp.json["status"] == Status.ERROR + + job = self.job_info[9] + assert job.status == Status.RUNNING, "Precondition invalid." + path = f"/jobs/{job.id}" + resp = self.app.get(path, headers=headers) + assert resp.status_code == 200 + assert resp.headers["Content-Type"] == f"{ContentType.APP_JSON}; profile={JobStatusSchema.OPENEO}" + assert resp.headers["Content-Schema"] == sd.OPENEO_API_SCHEMA_JOB_STATUS_URL + assert resp.json["status"] == Status.RUNNING + + job = self.job_info[11] + assert job.status == Status.ACCEPTED, "Precondition invalid." + path = f"/jobs/{job.id}" + resp = self.app.get(path, headers=headers) + assert resp.status_code == 200 + assert resp.headers["Content-Type"] == f"{ContentType.APP_JSON}; profile={JobStatusSchema.OPENEO}" + assert resp.headers["Content-Schema"] == sd.OPENEO_API_SCHEMA_JOB_STATUS_URL + assert resp.json["status"] == Status.QUEUED + + @pytest.mark.oap_part4 + @pytest.mark.openeo + def test_job_status_alt_openeo_profile_response(self): + """ + Validate retrieval of :term:`Job` status response with alternate value mapping by ``profile`` query parameter. + """ + job = self.job_info[0] + assert job.status == Status.SUCCEEDED, "Precondition invalid." + path = f"/jobs/{job.id}" + resp = self.app.get(path, headers=self.json_headers, params={"schema": JobStatusSchema.OPENEO}) + assert resp.status_code == 200 + assert resp.headers["Content-Type"] == f"{ContentType.APP_JSON}; profile={JobStatusSchema.OPENEO}" + assert resp.headers["Content-Schema"] == sd.OPENEO_API_SCHEMA_JOB_STATUS_URL + assert resp.json["status"] == Status.FINISHED + + job = self.job_info[1] + assert job.status == Status.FAILED, "Precondition invalid." + path = f"/jobs/{job.id}" + resp = self.app.get(path, headers=self.json_headers, params={"schema": "openeo"}) + assert resp.status_code == 200 + assert resp.headers["Content-Type"] == f"{ContentType.APP_JSON}; profile={JobStatusSchema.OPENEO}" + assert resp.headers["Content-Schema"] == sd.OPENEO_API_SCHEMA_JOB_STATUS_URL + assert resp.json["status"] == Status.ERROR + + job = self.job_info[9] + assert job.status == Status.RUNNING, "Precondition invalid." + path = f"/jobs/{job.id}" + resp = self.app.get(path, headers=self.json_headers, params={"schema": "openeo"}) + assert resp.status_code == 200 + assert resp.headers["Content-Type"] == f"{ContentType.APP_JSON}; profile={JobStatusSchema.OPENEO}" + assert resp.headers["Content-Schema"] == sd.OPENEO_API_SCHEMA_JOB_STATUS_URL + assert resp.json["status"] == Status.RUNNING + + job = self.job_info[11] + assert job.status == Status.ACCEPTED, "Precondition invalid." + path = f"/jobs/{job.id}" + resp = self.app.get(path, headers=self.json_headers, params={"schema": "openeo"}) + assert resp.status_code == 200 + assert resp.headers["Content-Type"] == f"{ContentType.APP_JSON}; profile={JobStatusSchema.OPENEO}" + assert resp.headers["Content-Schema"] == sd.OPENEO_API_SCHEMA_JOB_STATUS_URL + assert resp.json["status"] == Status.QUEUED + +@pytest.mark.oap_part1 @pytest.mark.parametrize( ["results", "expected"], [ diff --git a/tests/wps_restapi/test_processes.py b/tests/wps_restapi/test_processes.py index ef98f7ae2..6abc3e9f5 100644 --- a/tests/wps_restapi/test_processes.py +++ b/tests/wps_restapi/test_processes.py @@ -151,7 +151,7 @@ def get_process_deploy_template(self, process_id=None, cwl=None, schema=ProcessS to avoid extra package content-specific validations. """ if not process_id: - process_id = self.fully_qualified_test_process_name() + process_id = self.fully_qualified_test_name() body = { "processDescription": {}, "deploymentProfileName": "http://www.opengis.net/profiles/eoc/dockerizedApplication", @@ -541,7 +541,7 @@ def test_get_processes_with_providers_error_servers(self, mock_responses): def test_set_jobControlOptions_async_execute(self): path = "/processes" - process_name = self.fully_qualified_test_process_name() + process_name = self.fully_qualified_test_name() process_data = self.get_process_deploy_template(process_name) process_data["processDescription"]["jobControlOptions"] = [ExecuteControlOption.ASYNC] package_mock = mocked_process_package() @@ -557,7 +557,7 @@ def test_set_jobControlOptions_async_execute(self): def test_set_jobControlOptions_sync_execute(self): path = "/processes" - process_name = self.fully_qualified_test_process_name() + process_name = self.fully_qualified_test_name() process_data = self.get_process_deploy_template(process_name) process_data["processDescription"]["jobControlOptions"] = [ExecuteControlOption.SYNC] package_mock = mocked_process_package() @@ -574,7 +574,7 @@ def test_set_jobControlOptions_sync_execute(self): def test_get_processes_invalid_schemas_handled(self): path = "/processes" # deploy valid test process - process_name = self.fully_qualified_test_process_name() + process_name = self.fully_qualified_test_name() process_data = self.get_process_deploy_template(process_name) package_mock = mocked_process_package() with contextlib.ExitStack() as stack: @@ -669,7 +669,7 @@ def test_describe_process_visibility_private(self): assert resp.content_type == ContentType.APP_JSON def test_deploy_process_success(self): - process_name = self.fully_qualified_test_process_name() + process_name = self.fully_qualified_test_name() process_data = self.get_process_deploy_template(process_name) package_mock = mocked_process_package() @@ -684,7 +684,7 @@ def test_deploy_process_success(self): assert isinstance(resp.json["deploymentDone"], bool) and resp.json["deploymentDone"] def test_deploy_process_ogc_schema(self): - process_name = self.fully_qualified_test_process_name() + process_name = self.fully_qualified_test_name() process_data = self.get_process_deploy_template(process_name, schema=ProcessSchema.OGC) process_desc = process_data["processDescription"] package_mock = mocked_process_package() @@ -727,7 +727,7 @@ def test_deploy_process_short_name(self): assert resp.json["process"]["id"] == process_name def test_deploy_process_bad_name(self): - process_name = f"{self.fully_qualified_test_process_name()}..." + process_name = f"{self.fully_qualified_test_name()}..." process_data = self.get_process_deploy_template(process_name) package_mock = mocked_process_package() @@ -753,7 +753,7 @@ def test_deploy_process_conflict(self): assert resp.content_type == ContentType.APP_JSON def test_deploy_process_missing_or_invalid_components(self): - process_name = self.fully_qualified_test_process_name() + process_name = self.fully_qualified_test_name() process_data = self.get_process_deploy_template(process_name) package_mock = mocked_process_package() @@ -787,7 +787,7 @@ def test_deploy_process_default_endpoint_wps1(self): """ Validates that the default (localhost) endpoint to execute WPS requests are saved during deployment. """ - process_name = self.fully_qualified_test_process_name() + process_name = self.fully_qualified_test_name() process_data = self.get_process_deploy_template(process_name) package_mock = mocked_process_package() @@ -2223,14 +2223,14 @@ def test_delete_process_not_accessible(self): assert resp.content_type == ContentType.APP_JSON def test_delete_process_not_found(self): - name = self.fully_qualified_test_process_name() + name = self.fully_qualified_test_name() path = f"/processes/{name}" resp = self.app.delete_json(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 404, f"Error: {resp.text}" assert resp.content_type == ContentType.APP_JSON def test_delete_process_bad_name(self): - name = f"{self.fully_qualified_test_process_name()}..." + name = f"{self.fully_qualified_test_name()}..." path = f"/processes/{name}" resp = self.app.delete_json(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 400, f"Error: {resp.text}" @@ -2438,7 +2438,7 @@ def test_get_process_visibility_expected_response(self): assert "value" not in resp.json def test_get_process_visibility_not_found(self): - path = f"/processes/{self.fully_qualified_test_process_name()}/visibility" + path = f"/processes/{self.fully_qualified_test_name()}/visibility" resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 404 assert resp.content_type == ContentType.APP_JSON diff --git a/tests/wps_restapi/test_providers.py b/tests/wps_restapi/test_providers.py index 4cb249a6a..f3f1d248d 100644 --- a/tests/wps_restapi/test_providers.py +++ b/tests/wps_restapi/test_providers.py @@ -1,10 +1,9 @@ -import unittest - import owslib import pytest from pyramid.httpexceptions import HTTPNotFound from tests import resources +from tests.functional.utils import GenericUtils from tests.utils import ( get_test_weaver_app, mocked_remote_server_requests_wps1, @@ -19,17 +18,13 @@ from weaver.execute import ExecuteControlOption, ExecuteTransmissionMode from weaver.formats import ContentType from weaver.processes.constants import ProcessSchema -from weaver.utils import fully_qualified_name -class WpsProviderBase(unittest.TestCase): +class WpsProviderBase(GenericUtils): remote_provider_name = None settings = {} config = None - def fully_qualified_test_process_name(self): - return fully_qualified_name(self).replace(".", "-") - def register_provider(self, clear=True, error=False, data=None): if clear: self.service_store.clear_services() diff --git a/weaver/datatype.py b/weaver/datatype.py index 8fd31182b..5b79d8f2e 100644 --- a/weaver/datatype.py +++ b/weaver/datatype.py @@ -111,6 +111,7 @@ ExecutionInputs, ExecutionOutputs, ExecutionSubscribers, + JobResults, JSON, Link, Metadata, @@ -318,6 +319,9 @@ def __get__(self, instance, *_): if instance is None: # allow access to the descriptor as class attribute 'getattr(type(instance), property-name)' return self # noqa + # ensure that any 'fget' specified at property creation is employed + if self.fget != self.__get__: # pylint: disable=W0143 + return self.fget(instance) dt = instance.get(self.name, None) if not dt: if self.default_now: @@ -329,7 +333,10 @@ def __get__(self, instance, *_): def __set__(self, instance, value): # type: (Any, Union[datetime, str]) -> None - if isinstance(str, datetime): + # ensure that any 'fset' specified at property creation is employed + if self.fset != self.__set__: # pylint: disable=W0143 + return self.fset(instance, value) + if isinstance(value, str): value = dt_parse(value) if not isinstance(value, datetime): name = fully_qualified_name(instance) @@ -859,6 +866,25 @@ def wps_id(self, wps_id): raise TypeError(f"Type 'str' or 'UUID' is required for '{self.__name__}.wps_id'") self["wps_id"] = wps_id + @property + def wps_url(self): + # type: () -> Optional[str] + """ + Service URL reference for :term:`WPS` interface. + + .. seealso:: + - :attr:`Process.processEndpointWPS1` + - :attr:`Service.url` + """ + return self.get("wps_url", None) + + @wps_url.setter + def wps_url(self, service): + # type: (Optional[str]) -> None + if not isinstance(service, str): + raise TypeError(f"Type 'str' is required for '{self.__name__}.wps_url'") + self["wps_url"] = service + @property def service(self): # type: () -> Optional[str] @@ -913,6 +939,18 @@ def type(self): return "process" return "provider" + @property + def title(self): + # type: () -> Optional[str] + return self.get("title", None) + + @title.setter + def title(self, title): + # type: (Optional[str]) -> None + if not (isinstance(title, str) or not title): # disallow empty title as well + raise TypeError(f"Type 'str' or 'None' is required for '{self.__name__}.title'") + self["title"] = title + def _get_inputs(self): # type: () -> ExecutionInputs if self.get("inputs") is None: @@ -946,14 +984,14 @@ def _set_outputs(self, outputs): @property def user_id(self): - # type: () -> Optional[str] + # type: () -> Optional[Union[AnyUUID, int]] return self.get("user_id", None) @user_id.setter def user_id(self, user_id): - # type: (Optional[str]) -> None - if not isinstance(user_id, int) or user_id is None: - raise TypeError(f"Type 'int' is required for '{self.__name__}.user_id'") + # type: (Optional[Union[AnyUUID, int]]) -> None + if not isinstance(user_id, (int, str, uuid.UUID)) or user_id is None: + raise TypeError(f"Type 'int', 'str' or a UUID is required for '{self.__name__}.user_id'") self["user_id"] = user_id @property @@ -1060,7 +1098,7 @@ def execute_sync(self): @property def execution_mode(self): # type: () -> AnyExecuteMode - return ExecuteMode.get(self.get("execution_mode"), ExecuteMode.ASYNC) + return ExecuteMode.get(self.get("execution_mode"), ExecuteMode.AUTO) @execution_mode.setter def execution_mode(self, mode): @@ -1071,6 +1109,23 @@ def execution_mode(self, mode): raise ValueError(f"Invalid value for '{self.__name__}.execution_mode'. Must be one of {modes}") self["execution_mode"] = mode + @property + def execution_wait(self): + # type: () -> Optional[int] + """ + Execution time (in seconds) to wait for a synchronous response. + """ + if self.execute_async: + return None + return self.get("execution_wait") + + @execution_wait.setter + def execution_wait(self, wait): + # type: (Optional[int]) -> None + if not (wait is None or isinstance(wait, int)): + raise ValueError(f"Invalid value for '{self.__name__}.execution_wait'. Must be None or an integer.") + self["execution_wait"] = wait + @property def execution_response(self): # type: () -> AnyExecuteResponse @@ -1212,13 +1267,13 @@ def statistics(self, stats): self["statistics"] = stats def _get_results(self): - # type: () -> List[Optional[Dict[str, JSON]]] + # type: () -> JobResults if self.get("results") is None: self["results"] = [] return dict.__getitem__(self, "results") def _set_results(self, results): - # type: (List[Optional[Dict[str, JSON]]]) -> None + # type: (JobResults) -> None if not isinstance(results, list): raise TypeError(f"Type 'list' is required for '{self.__name__}.results'") self["results"] = results @@ -1429,7 +1484,7 @@ def links(self, container=None, self_link=None): job_path = base_url + sd.job_service.path.format(job_id=self.id) job_exec = f"{job_url.rsplit('/', 1)[0]}/execution" job_list = base_url + sd.jobs_service.path - job_links = [ + job_links = [ # type: List[Link] {"href": job_url, "rel": "status", "title": "Job status."}, # OGC {"href": job_url, "rel": "monitor", "title": "Job monitoring location."}, # IANA {"href": get_path_kvp(job_path, f=OutputFormat.JSON), "type": ContentType.APP_JSON, @@ -1506,6 +1561,7 @@ def json(self, container=None): # pylint: disable=W0221,arguments-differ "processID": self.process, "providerID": self.service, "type": self.type, + "title": self.title, "status": map_status(self.status), "message": self.status_message, "created": self.created, @@ -1533,8 +1589,10 @@ def params(self): "id": self.id, "task_id": self.task_id, "wps_id": self.wps_id, + "wps_url": self.wps_url, "service": self.service, "process": self.process, + "title": self.title, "inputs": self.inputs, "outputs": self.outputs, "user_id": self.user_id, @@ -1544,6 +1602,7 @@ def params(self): "execution_response": self.execution_response, "execution_return": self.execution_return, "execution_mode": self.execution_mode, + "execution_wait": self.execution_wait, "is_workflow": self.is_workflow, "created": self.created, "started": self.started, diff --git a/weaver/execute.py b/weaver/execute.py index 01c0b7581..0d0cae1f8 100644 --- a/weaver/execute.py +++ b/weaver/execute.py @@ -119,6 +119,7 @@ def parse_prefer_header_execute_mode( header_container, # type: AnyHeadersContainer supported_modes=None, # type: Optional[List[AnyExecuteControlOption]] wait_max=10, # type: int + return_auto=False, # type: bool ): # type: (...) -> Tuple[AnyExecuteMode, Optional[int], HeadersType] """ Obtain execution preference if provided in request headers. @@ -141,6 +142,10 @@ def parse_prefer_header_execute_mode( :param wait_max: Maximum wait time enforced by the server. If requested wait time is greater, ``wait`` preference will not be applied and will fall back to asynchronous response. + :param return_auto: + If the resolution ends up being an "auto" selection, the auto-resolved mode, wait-time, etc. are returned + by default. Using this option, the "auto" mode will be explicitly returned instead, allowing a mixture of + execution mode to be "auto" handled at another time. This is mostly for reporting purposes. :return: Tuple of resolved execution mode, wait time if specified, and header of applied preferences if possible. Maximum wait time indicates duration until synchronous response should fall back to asynchronous response. @@ -148,8 +153,9 @@ def parse_prefer_header_execute_mode( """ prefer = get_header("prefer", header_container) - relevant_modes = {ExecuteControlOption.ASYNC, ExecuteControlOption.SYNC} - supported_modes = list(set(supported_modes or []).intersection(relevant_modes)) + relevant_modes = [ExecuteControlOption.ASYNC, ExecuteControlOption.SYNC] # order important, async default + supported_modes = relevant_modes if supported_modes is None else supported_modes + supported_modes = [mode for mode in supported_modes if mode in relevant_modes] if not prefer: # /req/core/process-execute-default-execution-mode (A & B) @@ -160,7 +166,8 @@ def parse_prefer_header_execute_mode( wait = None if mode == ExecuteMode.ASYNC else wait_max return mode, wait, {} # /req/core/process-execute-default-execution-mode (C) - return ExecuteMode.SYNC, wait_max, {} + mode = ExecuteMode.AUTO if return_auto else ExecuteMode.SYNC + return mode, wait_max, {} # allow both listing of multiple 'Prefer' headers and single 'Prefer' header with multi-param ';' separated params = parse_kvp(prefer.replace(";", ","), pair_sep=",", multi_value_sep=None) @@ -191,7 +198,7 @@ def parse_prefer_header_execute_mode( LOGGER.info("Requested Prefer wait header too large (%ss > %ss), revert to async execution.", wait, wait_max) return ExecuteMode.ASYNC, None, {} - auto = ExecuteMode.ASYNC if "respond-async" in params else ExecuteMode.SYNC + auto = ExecuteMode.ASYNC if "respond-async" in params else ExecuteMode.AUTO applied_preferences = [] # /req/core/process-execute-auto-execution-mode (A & B) if len(supported_modes) == 1: @@ -199,7 +206,7 @@ def parse_prefer_header_execute_mode( # otherwise, server is allowed to discard preference since it cannot be honoured mode = ExecuteMode.ASYNC if supported_modes[0] == ExecuteControlOption.ASYNC else ExecuteMode.SYNC wait = None if mode == ExecuteMode.ASYNC else wait - if auto == mode: + if auto in [mode, ExecuteMode.AUTO]: if auto == ExecuteMode.ASYNC: applied_preferences.append("respond-async") if wait and "wait" in params: @@ -218,11 +225,36 @@ def parse_prefer_header_execute_mode( return ExecuteMode.ASYNC, None, {"Preference-Applied": "respond-async"} if wait and "wait" in params: return ExecuteMode.SYNC, wait, {"Preference-Applied": f"wait={wait}"} + if auto == ExecuteMode.AUTO and return_auto: + return ExecuteMode.AUTO, None, {} if wait: # default used, not a supplied preference return ExecuteMode.SYNC, wait, {} return ExecuteMode.ASYNC, None, {} +def rebuild_prefer_header(job): + # type: (Job) -> Optional[str] + """ + Rebuilds the expected ``Prefer`` header value from :term:`Job` parameters. + """ + def append_header(header_value, new_value): + # type: (str, str) -> str + if header_value and new_value: + header_value += "; " + header_value += new_value + return header_value + + header = "" + if job.execution_return: + header = append_header(header, f"return={job.execution_return}") + if job.execution_wait: + header = append_header(header, f"wait={job.execution_wait}") + if job.execute_async: + header = append_header(header, "respond-async") + + return header or None + + def update_preference_applied_return_header( job, # type: Job request_headers, # type: Optional[AnyHeadersContainer] diff --git a/weaver/processes/constants.py b/weaver/processes/constants.py index 827ed4c3f..263a25da0 100644 --- a/weaver/processes/constants.py +++ b/weaver/processes/constants.py @@ -365,6 +365,9 @@ class OpenSearchField(Constants): JobInputsOutputsSchemaAnyOGCType = Union[JobInputsOutputsSchemaType_OGC, JobInputsOutputsSchemaType_OGC_STRICT] JobInputsOutputsSchemaAnyOLDType = Union[JobInputsOutputsSchemaType_OLD, JobInputsOutputsSchemaType_OLD_STRICT] JobInputsOutputsSchemaType = Union[JobInputsOutputsSchemaAnyOGCType, JobInputsOutputsSchemaAnyOLDType] +JobStatusSchemaType_OGC = Literal["OGC", "ogc"] +JobStatusSchemaType_OpenEO = Literal["OPENEO", "openeo", "openEO", "OpenEO"] +JobStatusSchemaType = Union[JobStatusSchemaType_OGC, JobStatusSchemaType_OpenEO] class ProcessSchema(Constants): @@ -386,6 +389,14 @@ class JobInputsOutputsSchema(Constants): OLD = "old" # type: JobInputsOutputsSchemaType_OLD +class JobStatusSchema(Constants): + """ + Schema selector to represent a :term:`Job` status response. + """ + OGC = "ogc" # type: JobStatusSchemaType_OGC + OPENEO = "openeo" # type: JobStatusSchemaType_OpenEO + + if TYPE_CHECKING: # pylint: disable=invalid-name CWL_RequirementNames = Literal[ diff --git a/weaver/processes/convert.py b/weaver/processes/convert.py index 2810635cc..ed0f8c297 100644 --- a/weaver/processes/convert.py +++ b/weaver/processes/convert.py @@ -1911,17 +1911,25 @@ def convert_input_values_schema(inputs, schema): @overload -def convert_output_params_schema(inputs, schema): +def convert_output_params_schema(outputs, schema): # type: (Optional[ExecutionOutputs], JobInputsOutputsSchemaAnyOGCType) -> Optional[ExecutionOutputsMap] ... @overload -def convert_output_params_schema(inputs, schema): +def convert_output_params_schema(outputs, schema): # type: (Optional[ExecutionOutputs], JobInputsOutputsSchemaAnyOLDType) -> Optional[ExecutionOutputsList] ... +# FIXME: workaround typing duplicate +# (https://youtrack.jetbrains.com/issue/PY-76786/Typing-literal-with-overload-fails-to-consider-non-overloaded-type) +@overload +def convert_output_params_schema(outputs, schema): + # type: (Optional[ExecutionOutputs], JobInputsOutputsSchemaType) -> Optional[ExecutionOutputs] + ... + + def convert_output_params_schema(outputs, schema): # type: (Optional[ExecutionOutputs], JobInputsOutputsSchemaType) -> Optional[ExecutionOutputs] """ diff --git a/weaver/processes/execution.py b/weaver/processes/execution.py index 2ffae24de..3747364db 100644 --- a/weaver/processes/execution.py +++ b/weaver/processes/execution.py @@ -11,15 +11,26 @@ from celery.utils.log import get_task_logger from owslib.util import clean_ows_url from owslib.wps import BoundingBoxDataInput, ComplexDataInput -from pyramid.httpexceptions import HTTPBadRequest, HTTPNotAcceptable +from pyramid.httpexceptions import ( + HTTPAccepted, + HTTPBadRequest, + HTTPCreated, + HTTPNotAcceptable, + HTTPUnprocessableEntity, + HTTPUnsupportedMediaType +) from pyramid_celery import celery_app as app +from werkzeug.wrappers.request import Request as WerkzeugRequest from weaver.database import get_db from weaver.datatype import Process, Service from weaver.execute import ( ExecuteControlOption, ExecuteMode, + ExecuteResponse, + ExecuteReturnPreference, parse_prefer_header_execute_mode, + parse_prefer_header_return, update_preference_applied_return_header ) from weaver.formats import AcceptLanguage, ContentType, clean_media_type_format, map_cwl_media_type, repr_json @@ -35,15 +46,18 @@ ows2json_output_data ) from weaver.processes.types import ProcessType +from weaver.processes.utils import get_process from weaver.status import JOB_STATUS_CATEGORIES, Status, StatusCategory, map_status from weaver.store.base import StoreJobs, StoreProcesses from weaver.utils import ( apply_number_with_unit, as_int, + extend_instance, fully_qualified_name, get_any_id, get_any_value, get_header, + get_path_kvp, get_registry, get_settings, now, @@ -53,6 +67,7 @@ wait_secs ) from weaver.visibility import Visibility +from weaver.wps.service import get_pywps_service from weaver.wps.utils import ( check_wps_status, get_wps_client, @@ -61,6 +76,7 @@ get_wps_output_dir, get_wps_output_path, get_wps_output_url, + get_wps_path, load_pywps_config ) from weaver.wps_restapi import swagger_definitions as sd @@ -69,7 +85,7 @@ LOGGER = logging.getLogger(__name__) if TYPE_CHECKING: - from typing import Dict, List, Optional, Tuple, Union + from typing import Any, Dict, List, Optional, Tuple, Type, Union from uuid import UUID from celery.app.task import Task @@ -86,7 +102,9 @@ AnyProcessRef, AnyResponseType, AnyServiceRef, + AnySettingsContainer, AnyValueType, + AnyViewResponse, CeleryResult, HeaderCookiesType, HeadersType, @@ -123,8 +141,6 @@ def execute_process(task, job_id, wps_url, headers=None): """ Celery task that executes the WPS process job monitoring as status updates (local and remote). """ - from weaver.wps.service import get_pywps_service - LOGGER.debug("Job execute process called.") task_process = get_celery_process() @@ -667,8 +683,31 @@ def map_locations(job, settings): os.symlink(wps_ref, job_ref) -def submit_job(request, reference, tags=None): - # type: (Request, Union[Service, Process], Optional[List[str]]) -> AnyResponseType +def submit_job_dispatch_wps(request, process): + # type: (Request, Process) -> AnyViewResponse + """ + Dispatch a :term:`XML` request to the relevant :term:`Process` handler using the :term:`WPS` endpoint. + + Sends the :term:`XML` request to the :term:`WPS` endpoint which knows how to parse it properly. + Execution will end up in the same :func:`submit_job_handler` function as for :term:`OGC API - Processes` + :term:`JSON` execution. + + .. warning:: + The function assumes that :term:`XML` was pre-validated as present in the :paramref:`request`. + """ + service = get_pywps_service() + wps_params = {"version": "1.0.0", "request": "Execute", "service": "WPS", "identifier": process.id} + request.path_info = get_wps_path(request) + request.query_string = get_path_kvp("", **wps_params)[1:] + location = request.application_url + request.path_info + request.query_string + LOGGER.warning("Route redirection [%s] -> [%s] for WPS-XML support.", request.url, location) + http_request = extend_instance(request, WerkzeugRequest) + http_request.shallow = False + return service.call(http_request) + + +def submit_job(request, reference, tags=None, process_id=None): + # type: (Request, Union[Service, Process], Optional[List[str]], Optional[str]) -> AnyResponseType """ Generates the job submission from details retrieved in the request. @@ -676,27 +715,17 @@ def submit_job(request, reference, tags=None): :func:`submit_job_handler` to provide elements pre-extracted from requests or from other parsing. """ # validate body with expected JSON content and schema - if ContentType.APP_JSON not in request.content_type: - raise HTTPBadRequest(json={ - "code": "InvalidHeaderValue", - "name": "Content-Type", - "description": f"Request 'Content-Type' header other than '{ContentType.APP_JSON}' not supported.", - "value": str(request.content_type) - }) - try: - json_body = request.json_body - except Exception as ex: - raise HTTPBadRequest(f"Invalid JSON body cannot be decoded for job submission. [{ex}]") + json_body = validate_job_json(request) # validate context if needed later on by the job for early failure context = get_wps_output_context(request) - provider_id = None # None OK if local - process_id = None # None OK if remote, but can be found as well if available from WPS-REST path # noqa + prov_id = None # None OK if local + proc_id = None # None OK if remote, but can be found as well if available from WPS-REST path # noqa tags = tags or [] lang = request.accept_language.header_value # can only preemptively check if local process if isinstance(reference, Process): service_url = reference.processEndpointWPS1 - process_id = reference.identifier # explicit 'id:version' process revision if available, otherwise simply 'id' + proc_id = reference.identifier # explicit 'id:version' process revision if available, otherwise simply 'id' visibility = reference.visibility is_workflow = reference.type == ProcessType.WORKFLOW is_local = True @@ -718,8 +747,8 @@ def submit_job(request, reference, tags=None): lang = matched_lang elif isinstance(reference, Service): service_url = reference.url - provider_id = reference.id - process_id = resolve_process_tag(request) + prov_id = reference.id + proc_id = process_id or resolve_process_tag(request) visibility = Visibility.PUBLIC is_workflow = False is_local = False @@ -732,13 +761,13 @@ def submit_job(request, reference, tags=None): user = request.authenticated_userid # FIXME: consider other methods to provide the user headers = dict(request.headers) settings = get_settings(request) - return submit_job_handler(json_body, settings, service_url, provider_id, process_id, is_workflow, is_local, + return submit_job_handler(json_body, settings, service_url, prov_id, proc_id, is_workflow, is_local, visibility, language=lang, headers=headers, tags=tags, user=user, context=context) def submit_job_handler(payload, # type: ProcessExecution settings, # type: SettingsType - service_url, # type: str + wps_url, # type: str provider=None, # type: Optional[AnyServiceRef] process=None, # type: AnyProcessRef is_workflow=False, # type: bool @@ -751,25 +780,13 @@ def submit_job_handler(payload, # type: ProcessExecution context=None, # type: Optional[str] ): # type: (...) -> AnyResponseType """ - Submits the job to the Celery worker with provided parameters. + Parses parameters that defines the submitted :term:`Job`, and responds accordingly with the selected execution mode. - Assumes that parameters have been pre-fetched and validated, except for the input payload. + Assumes that parameters have been pre-fetched and validated, except for the :paramref:`payload` containing the + desired inputs and outputs from the :term:`Job`. The selected execution mode looks up the various combinations + of headers and body parameters available across :term:`API` implementations and revisions. """ - try: - json_body = sd.Execute().deserialize(payload) - except colander.Invalid as ex: - raise HTTPBadRequest( - json=sd.ErrorJsonResponseBodySchema(schema_include=True).deserialize({ - "type": "InvalidSchema", - "title": "Execute", - "detail": "Execution body failed schema validation.", - "status": HTTPBadRequest.code, - "error": ex.msg, - "cause": ex.asdict(), - "value": repr_json(ex.value), - }) - ) - + json_body = validate_job_schema(payload) db = get_db(settings) # non-local is only a reference, no actual process object to validate @@ -779,6 +796,7 @@ def submit_job_handler(payload, # type: ProcessExecution process = proc_store.fetch_by_id(process) if process and is_local: validate_process_io(process, json_body) + validate_process_id(process, json_body) else: LOGGER.warning( "Skipping validation of execution parameters for remote process [%s] on provider [%s]", @@ -792,80 +810,302 @@ def submit_job_handler(payload, # type: ProcessExecution job_ctl_opts = ExecuteControlOption.values() exec_max_wait = settings.get("weaver.execute_sync_max_wait", settings.get("weaver.exec_sync_max_wait")) exec_max_wait = as_int(exec_max_wait, default=20) - mode, wait, applied = parse_prefer_header_execute_mode(headers, job_ctl_opts, exec_max_wait) + mode, wait, applied = parse_prefer_header_execute_mode(headers, job_ctl_opts, exec_max_wait, return_auto=True) if not applied: # whatever returned is a default, consider 'mode' in body as alternative - is_execute_async = ExecuteMode.get(json_body.get("mode")) != ExecuteMode.SYNC # convert auto to async + execute_mode = ExecuteMode.get(json_body.get("mode"), default=ExecuteMode.AUTO) else: # as per https://datatracker.ietf.org/doc/html/rfc7240#section-2 # Prefer header not resolved with a valid value should still resume without error - is_execute_async = mode != ExecuteMode.SYNC + execute_mode = mode accept_type = validate_job_accept_header(headers, mode) exec_resp, exec_return = get_job_return(job=None, body=json_body, headers=headers) # job 'None' since still parsing req_headers = copy.deepcopy(headers or {}) get_header("prefer", headers, pop=True) # don't care about value, just ensure removed with any header container + job_pending_created = json_body.get("status") == "create" + if job_pending_created: + job_status = Status.CREATED + job_message = "Job created with pending trigger." + else: + job_status = Status.ACCEPTED + job_message = "Job task submitted for execution." + subscribers = map_job_subscribers(json_body, settings) job_inputs = json_body.get("inputs") job_outputs = json_body.get("outputs") store = db.get_store(StoreJobs) # type: StoreJobs - job = store.save_job(task_id=Status.ACCEPTED, process=process, service=provider_id, + job = store.save_job(task_id=job_status, process=process, service=provider_id, status=job_status, inputs=job_inputs, outputs=job_outputs, is_workflow=is_workflow, is_local=is_local, - execute_async=is_execute_async, execute_response=exec_resp, execute_return=exec_return, + execute_mode=execute_mode, execute_wait=wait, + execute_response=exec_resp, execute_return=exec_return, custom_tags=tags, user_id=user, access=visibility, context=context, subscribers=subscribers, accept_type=accept_type, accept_language=language) - job.save_log(logger=LOGGER, message="Job task submitted for execution.", status=Status.ACCEPTED, progress=0) + job.save_log(logger=LOGGER, message=job_message, status=job_status, progress=0) + job.wps_url = wps_url job = store.update_job(job) - location_url = job.status_url(settings) + + return submit_job_dispatch_task(job, headers=req_headers, container=settings) + + +def submit_job_dispatch_task( + job, # type: Job + *, # force named keyword arguments after + container, # type: AnySettingsContainer + headers=None, # type: AnyHeadersContainer + force_submit=False, # type: bool +): # type: (...) -> AnyResponseType + """ + Submits the :term:`Job` to the :mod:`celery` worker with provided parameters. + + Assumes that parameters have been pre-fetched, validated, and can be resolved from the :term:`Job`. + """ + db = get_db(container) + store = db.get_store(StoreJobs) + + location_url = job.status_url(container) resp_headers = {"Location": location_url} - resp_headers.update(applied) + req_headers = copy.deepcopy(headers or {}) - wps_url = clean_ows_url(service_url) - result = execute_process.delay(job_id=job.id, wps_url=wps_url, headers=headers) # type: CeleryResult - LOGGER.debug("Celery pending task [%s] for job [%s].", result.id, job.id) - if not is_execute_async: - LOGGER.debug("Celery task requested as sync if it completes before (wait=%ss)", wait) + task_result = None # type: Optional[CeleryResult] + job_pending_created = job.status == Status.CREATED + if job_pending_created and force_submit: + # preemptively update job status to avoid next + # dispatch steps ignoring submission to the worker + job.status = Status.ACCEPTED + job = store.update_job(job) + job_pending_created = False + response_class = HTTPAccepted + else: + response_class = HTTPCreated + + if not job_pending_created: + wps_url = clean_ows_url(job.wps_url) + task_result = execute_process.delay(job_id=job.id, wps_url=wps_url, headers=headers) + LOGGER.debug("Celery pending task [%s] for job [%s].", task_result.id, job.id) + + execute_sync = not job_pending_created and not job.execute_async + if execute_sync: + LOGGER.debug("Celery task requested as sync if it completes before (wait=%ss)", job.execution_wait) try: - result.wait(timeout=wait) + task_result.wait(timeout=job.execution_wait) except CeleryTaskTimeoutError: pass - if result.ready(): + if task_result.ready(): job = store.fetch_by_id(job.id) # when sync is successful, it must return the results direct instead of status info # see: https://docs.ogc.org/is/18-062r2/18-062r2.html#sc_execute_response if job.status == Status.SUCCEEDED: + _, _, sync_applied = parse_prefer_header_execute_mode(req_headers, [ExecuteControlOption.SYNC]) + if sync_applied: + resp_headers.update(sync_applied) return get_job_results_response( job, request_headers=req_headers, response_headers=resp_headers, - container=settings, + container=container, ) # otherwise return the error status - body = job.json(container=settings) + body = job.json(container=container) body["location"] = location_url resp = get_job_submission_response(body, resp_headers, error=True) return resp else: - LOGGER.debug("Celery task requested as sync took too long to complete (wait=%ss). Continue in async.", wait) - # sync not respected, therefore must drop it - # since both could be provided as alternative preferences, drop only async with limited subset - prefer = get_header("Preference-Applied", headers, pop=True) - _, _, async_applied = parse_prefer_header_execute_mode({"Prefer": prefer}, [ExecuteControlOption.ASYNC]) - if async_applied: - resp_headers.update(async_applied) + job.save_log( + logger=LOGGER, + level=logging.WARNING, + message=( + f"Job requested as synchronous execution took too long to complete (wait={job.execution_wait}s). " + "Will resume with asynchronous execution." + ) + ) + job = store.update_job(job) + execute_sync = False + + if not execute_sync: + # either sync was not respected, therefore must drop it, or it was not requested at all + # since both could be provided as alternative preferences, drop only sync with limited subset + _, _, async_applied = parse_prefer_header_execute_mode(req_headers, [ExecuteControlOption.ASYNC]) + if async_applied: + resp_headers.update(async_applied) LOGGER.debug("Celery task submitted to run async.") body = { "jobID": job.id, "processID": job.process, - "providerID": provider_id, # dropped by validator if not applicable - "status": map_status(Status.ACCEPTED), - "location": location_url + "providerID": job.service, # dropped by validator if not applicable + "status": map_status(job.status), + "location": location_url, # for convenience/backward compatibility, but official is Location *header* } resp_headers = update_preference_applied_return_header(job, req_headers, resp_headers) - resp = get_job_submission_response(body, resp_headers) + resp = get_job_submission_response(body, resp_headers, response_class=response_class) return resp +def update_job_parameters(job, request): + # type: (Job, Request) -> None + """ + Updates an existing :term:`Job` with new request parameters. + """ + body = validate_job_json(request) + body = validate_job_schema(body, sd.PatchJobBodySchema) + + value = field = loc = None + job_process = get_process(job.process) + validate_process_id(job_process, body) + try: + loc = "body" + + # used to avoid possible attribute name conflict + # (e.g.: 'job.response' vs 'job.execution_response') + execution_fields = ["response", "mode"] + + for node in sd.PatchJobBodySchema().children: + field = node.name + if not field or field not in body: + continue + if field in ["subscribers", "notification_email"]: + continue # will be handled simultaneously after + + value = body[field] # type: ignore + if field not in execution_fields and field in job: + setattr(job, field, value) + elif field in execution_fields: + field = f"execution_{field}" + if field == "execution_mode": + if value == ExecuteMode.AUTO: + continue # don't override previously set value that resolved with default value by omission + if value in [ExecuteMode.ASYNC, ExecuteMode.SYNC]: + job_ctrl_exec = ExecuteControlOption.get(f"{value}-execute") + if job_ctrl_exec not in job_process.jobControlOptions: + raise HTTPBadRequest( + json=sd.ErrorJsonResponseBodySchema(schema_include=True).deserialize({ + "type": "InvalidJobUpdate", + "title": "Invalid Job Execution Update", + "detail": ( + "Update of the job execution mode is not permitted " + "by supported jobControlOptions of the process description." + ), + "status": HTTPBadRequest.code, + "cause": {"name": "mode", "in": loc}, + "value": repr_json( + { + "process.jobControlOptions": job_process.jobControlOptions, + "job.mode": job_ctrl_exec, + }, force_string=False + ), + }) + ) + + # 'response' will take precedence, but (somewhat) align 'Prefer: return' value to match intention + # they are not 100% compatible because output 'transmissionMode' must be considered when + # resolving 'response', but given both 'response' and 'transmissionMode' override 'Prefer', + # this is an "acceptable" compromise (see docs 'Execution Response' section for more details) + if field == "execution_response": + if value == ExecuteResponse.RAW: + job.execution_return = ExecuteReturnPreference.REPRESENTATION + else: + job.execution_return = ExecuteReturnPreference.MINIMAL + + setattr(job, field, value) + + settings = get_settings(request) + subscribers = map_job_subscribers(body, settings=settings) + if not subscribers and body.get("subscribers") == {}: + subscribers = {} # asking to remove all subscribers explicitly + if subscribers is not None: + job.subscribers = subscribers + + # for both 'mode' and 'response' + # if provided both in body and corresponding 'Prefer' header parameter, + # the body parameter takes precedence (set in code above) + # however, if provided only in header, allow override of the body parameter considered as "higher priority" + loc = "header" + if ExecuteMode.get(body.get("mode"), default=ExecuteMode.AUTO) == ExecuteMode.AUTO: + mode, wait, _ = parse_prefer_header_execute_mode( + request.headers, + job_process.jobControlOptions, + return_auto=True, + ) + job.execution_mode = mode + job.execution_wait = wait if mode == ExecuteMode.SYNC else job.execution_wait + if "response" not in body: + job_return = parse_prefer_header_return(request.headers) + if job_return: + job.execution_return = job_return + if job_return == ExecuteReturnPreference.REPRESENTATION: + job.execution_response = ExecuteResponse.RAW + else: + job.execution_response = ExecuteResponse.DOCUMENT + + except ValueError as exc: + raise HTTPUnprocessableEntity( + json=sd.ErrorJsonResponseBodySchema(schema_include=True).deserialize({ + "type": "InvalidJobUpdate", + "title": "Invalid Job Execution Update", + "detail": "Could not update the job execution definition using specified parameters.", + "status": HTTPUnprocessableEntity.code, + "error": type(exc), + "cause": {"name": field, "in": loc}, + "value": repr_json(value, force_string=False), + }) + ) + + LOGGER.info("Updating %s", job) + db = get_db(request) + store = db.get_store(StoreJobs) + store.update_job(job) + + +def validate_job_json(request): + # type: (Request) -> JSON + """ + Validates that the request contains valid :term:`JSON` contents, but not necessary valid against expected schema. + + .. seealso:: + :func:`validate_job_schema` + """ + if ContentType.APP_JSON not in request.content_type: + raise HTTPUnsupportedMediaType(json={ + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-4/1.0/unsupported-media-type", + "title": "Unsupported Media-Type", + "detail": f"Request 'Content-Type' header other than '{ContentType.APP_JSON}' is not supported.", + "code": "InvalidHeaderValue", + "name": "Content-Type", + "value": str(request.content_type) + }) + try: + json_body = request.json_body + except Exception as ex: + raise HTTPBadRequest(json={ + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-4/1.0/unsupported-media-type", + "title": "Bad Request", + "detail": f"Invalid JSON body cannot be decoded for job submission. [{ex}]", + }) + return json_body + + +def validate_job_schema(payload, body_schema=sd.Execute): + # type: (Any, Union[Type[sd.Execute], Type[sd.PatchJobBodySchema]]) -> ProcessExecution + """ + Validates that the input :term:`Job` payload is valid :term:`JSON` for an execution request. + """ + try: + json_body = body_schema().deserialize(payload) + except colander.Invalid as ex: + raise HTTPUnprocessableEntity( + json=sd.ErrorJsonResponseBodySchema(schema_include=True).deserialize({ + "type": "InvalidSchema", + "title": "Invalid Job Execution Schema", + "detail": "Execution body failed schema validation.", + "status": HTTPUnprocessableEntity.code, + "error": ex.msg, + "cause": ex.asdict(), + "value": repr_json(ex.value), + }) + ) + return json_body + + def validate_job_accept_header(headers, execution_mode): # type: (AnyHeadersContainer, AnyExecuteMode) -> Optional[str] """ @@ -878,7 +1118,7 @@ def validate_job_accept_header(headers, execution_mode): if ContentType.APP_JSON in accept: return ContentType.APP_JSON # anything always allowed in sync, since results returned directly - if execution_mode == ExecuteMode.SYNC: + if execution_mode in [ExecuteMode.SYNC, ExecuteMode.AUTO]: return accept if ContentType.ANY in accept: return @@ -898,6 +1138,40 @@ def validate_job_accept_header(headers, execution_mode): ) +def validate_process_id(job_process, payload): + # type: (Process, ProcessExecution) -> None + """ + Validates that the specified ``process`` in the payload corresponds to the referenced :term:`Job` :term:`Process`. + + If not ``process```is specified, no check is performed. The :term:`Job` is assumed to have pre-validated that + the :term:`Process` is appropriate from another reference, such as using the ID from the path or a query parameter. + + :raises HTTPException: Corresponding error for detected invalid combination of process references. + """ + if "process" in payload: + # note: don't use 'get_process' for input process, as it might not even exist! + req_process_url = payload["process"] + req_process_id = payload["process"].rsplit("/processes/", 1)[-1] + if req_process_id != job_process.id or req_process_url != job_process.processDescriptionURL: + raise HTTPBadRequest( + json=sd.ErrorJsonResponseBodySchema(schema_include=True).deserialize( + { + "type": "InvalidJobUpdate", + "title": "Invalid Job Execution Update", + "detail": "Update of the reference process for the job execution is not permitted.", + "status": HTTPBadRequest.code, + "cause": {"name": "process", "in": "body"}, + "value": repr_json( + { + "body.process": payload["process"], + "job.process": job_process.processDescriptionURL, + }, force_string=False + ), + } + ) + ) + + def validate_process_io(process, payload): # type: (Process, ProcessExecution) -> None """ diff --git a/weaver/processes/utils.py b/weaver/processes/utils.py index 331df523f..13f7acaa8 100644 --- a/weaver/processes/utils.py +++ b/weaver/processes/utils.py @@ -17,9 +17,11 @@ HTTPCreated, HTTPException, HTTPForbidden, + HTTPInternalServerError, HTTPNotFound, HTTPOk, - HTTPUnprocessableEntity + HTTPUnprocessableEntity, + HTTPUnsupportedMediaType ) from pyramid.settings import asbool @@ -70,7 +72,7 @@ from weaver.wps.utils import get_wps_client from weaver.wps_restapi import swagger_definitions as sd from weaver.wps_restapi.processes.utils import resolve_process_tag -from weaver.wps_restapi.utils import get_wps_restapi_base_url, parse_content +from weaver.wps_restapi.utils import get_wps_restapi_base_url LOGGER = logging.getLogger(__name__) if TYPE_CHECKING: @@ -107,9 +109,6 @@ UpdateFields = List[Union[str, UpdateFieldListMethod]] -# FIXME: -# https://github.com/crim-ca/weaver/issues/215 -# define common Exception classes that won't require this type of conversion def get_process(process_id=None, request=None, settings=None, store=None, revision=True): # type: (Optional[str], Optional[PyramidRequest], Optional[SettingsType], Optional[StoreProcesses], bool) -> Process """ @@ -329,6 +328,65 @@ def resolve_cwl_graph(package): return package +def parse_process_deploy_content( + request=None, # type: Optional[AnyRequestType] + content=None, # type: Optional[Union[JSON, str]] + content_schema=None, # type: Optional[colander.SchemaNode] + content_type=sd.RequestContentTypeHeader.default, # type: Optional[ContentType] + content_type_schema=sd.RequestContentTypeHeader, # type: Optional[colander.SchemaNode] +): # type: (...) -> Union[JSON, CWL] + """ + Load the request content with validation of expected content type and their schema. + """ + if request is None and content is None: # pragma: no cover # safeguard for early detect invalid implementation + raise HTTPInternalServerError(json={ + "title": "Internal Server Error", + "type": "InternalServerError", + "detail": "Cannot parse undefined contents.", + "status": HTTPInternalServerError.code, + "cause": "Request content and content argument are undefined.", + }) + try: + if request is not None: + content = request.text + content_type = request.content_type + if content_type is not None and content_type_schema is not None: + content_type = content_type_schema().deserialize(content_type) + if isinstance(content, str): + content = yaml.safe_load(content) + if not isinstance(content, dict): + raise TypeError("Not a valid JSON body for process deployment.") + except colander.Invalid as exc: + raise HTTPUnsupportedMediaType(json={ + "title": "Unsupported Media Type", + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-2/1.0/unsupported-media-type", + "detail": str(exc), + "status": HTTPUnsupportedMediaType.code, + "cause": {"Content-Type": None if content_type is None else str(content_type)}, + }) + except Exception as exc: + raise HTTPBadRequest(json={ + "title": "Bad Request", + "type": "BadRequest", + "detail": "Unable to parse contents.", + "status": HTTPBadRequest.code, + "cause": str(exc), + }) + try: + if content_schema is not None: + content = content_schema().deserialize(content) + except colander.Invalid as exc: + raise HTTPUnprocessableEntity(json={ + "type": "InvalidParameterValue", + "title": "Failed schema validation.", + "status": HTTPUnprocessableEntity.code, + "error": colander.Invalid.__name__, + "cause": exc.msg, + "value": repr_json(exc.value, force_string=False), + }) + return content + + def deploy_process_from_payload(payload, container, overwrite=False): # pylint: disable=R1260,too-complex # type: (Union[JSON, str], Union[AnySettingsContainer, AnyRequestType], Union[bool, Process]) -> HTTPException """ @@ -354,7 +412,7 @@ def deploy_process_from_payload(payload, container, overwrite=False): # pylint: c_type = ContentType.get(get_header("Content-Type", headers), default=ContentType.APP_OGC_PKG_JSON) # use deepcopy of to remove any circular dependencies before writing to mongodb or any updates to the payload - payload = parse_content( + payload = parse_process_deploy_content( request=None, content=payload, content_type=c_type, @@ -839,7 +897,7 @@ def update_process_metadata(request): Desired new version can be eiter specified explicitly in request payload, or will be guessed accordingly to detected changes to be applied. """ - data = parse_content(request, content_schema=sd.PatchProcessBodySchema) + data = parse_process_deploy_content(request, content_schema=sd.PatchProcessBodySchema) old_process = get_process(request=request) new_process = copy.deepcopy(old_process) update_level = _apply_process_metadata(new_process, data) diff --git a/weaver/status.py b/weaver/status.py index 45042655f..6ecf64ca9 100644 --- a/weaver/status.py +++ b/weaver/status.py @@ -9,22 +9,29 @@ class StatusCompliant(ExtendedEnum): OGC = "OGC" PYWPS = "PYWPS" OWSLIB = "OWSLIB" + OPENEO = "OPENEO" class StatusCategory(ExtendedEnum): FINISHED = "FINISHED" RUNNING = "RUNNING" + PENDING = "PENDING" FAILED = "FAILED" class Status(Constants): + CREATED = "created" + QUEUED = "queued" ACCEPTED = "accepted" STARTED = "started" PAUSED = "paused" SUCCEEDED = "succeeded" SUCCESSFUL = "successful" FAILED = "failed" + ERROR = "error" + FINISHED = "finished" RUNNING = "running" + CANCELED = "canceled" DISMISSED = "dismissed" EXCEPTION = "exception" UNKNOWN = "unknown" # don't include in any below collections @@ -33,19 +40,21 @@ class Status(Constants): JOB_STATUS_CATEGORIES = { # note: # OGC compliant (old): [Accepted, Running, Succeeded, Failed] - # OGC compliant (new): [accepted, running, successful, failed, dismissed] + # OGC compliant (new): [accepted, running, successful, failed, dismissed, created] ('created' in Part 4 only) # PyWPS uses: [Accepted, Started, Succeeded, Failed, Paused, Exception] - # OWSLib users: [Accepted, Running, Succeeded, Failed, Paused] (with 'Process' in front) + # OWSLib uses: [Accepted, Running, Succeeded, Failed, Paused] (with 'Process' in front) + # OpenEO uses: [queued, running, finished, error, canceled, created] # https://github.com/opengeospatial/ogcapi-processes/blob/master/openapi/schemas/processes-core/statusCode.yaml # http://docs.opengeospatial.org/is/14-065/14-065.html#17 # corresponding statuses are aligned vertically for 'COMPLIANT' groups StatusCompliant.OGC: frozenset([ + Status.CREATED, # Part 4: Job Management Status.ACCEPTED, Status.RUNNING, - Status.SUCCEEDED, # old (keep it because it matches existing ADES/EMS and other providers) + Status.SUCCEEDED, # new Status.FAILED, - Status.SUCCESSFUL, # new + Status.SUCCESSFUL, # old (keep it because it matches existing ADES/EMS and other providers) Status.DISMISSED # new ]), StatusCompliant.PYWPS: frozenset([ @@ -53,8 +62,7 @@ class Status(Constants): Status.STARTED, # running Status.SUCCEEDED, Status.FAILED, - Status.PAUSED, - Status.EXCEPTION + Status.PAUSED ]), StatusCompliant.OWSLIB: frozenset([ Status.ACCEPTED, @@ -63,31 +71,50 @@ class Status(Constants): Status.FAILED, Status.PAUSED ]), + StatusCompliant.OPENEO: frozenset([ + Status.CREATED, + Status.QUEUED, + Status.RUNNING, + Status.FINISHED, + Status.ERROR, + Status.CANCELED + ]), # utility categories StatusCategory.RUNNING: frozenset([ Status.ACCEPTED, Status.RUNNING, Status.STARTED, + Status.QUEUED, + Status.PAUSED + ]), + StatusCategory.PENDING: frozenset([ + Status.CREATED, + Status.ACCEPTED, + Status.QUEUED, Status.PAUSED ]), StatusCategory.FINISHED: frozenset([ Status.FAILED, Status.DISMISSED, + Status.CANCELED, Status.EXCEPTION, + Status.ERROR, Status.SUCCEEDED, - Status.SUCCESSFUL + Status.SUCCESSFUL, + Status.FINISHED ]), StatusCategory.FAILED: frozenset([ Status.FAILED, Status.DISMISSED, - Status.EXCEPTION + Status.EXCEPTION, + Status.ERROR ]), } # FIXME: see below detail in map_status about 'successful', partially compliant to OGC statuses # https://github.com/opengeospatial/ogcapi-processes/blob/ca8e90/core/openapi/schemas/statusCode.yaml JOB_STATUS_CODE_API = JOB_STATUS_CATEGORIES[StatusCompliant.OGC] - {Status.SUCCESSFUL} -JOB_STATUS_SEARCH_API = set(list(JOB_STATUS_CODE_API) + [StatusCategory.FINISHED.value.lower()]) +JOB_STATUS_SEARCH_API = set(list(JOB_STATUS_CODE_API) + [Status.FINISHED]) # id -> str STATUS_PYWPS_MAP = {s: _WPS_STATUS._fields[s].lower() for s in range(len(WPS_STATUS))} @@ -100,14 +127,19 @@ class Status(Constants): from weaver.typedefs import Literal, TypeAlias StatusType: Status = Literal[ + Status.CREATED, Status.ACCEPTED, Status.STARTED, + Status.QUEUED, Status.PAUSED, Status.SUCCEEDED, + Status.FINISHED, Status.FAILED, Status.RUNNING, Status.DISMISSED, + Status.CANCELED, Status.EXCEPTION, + Status.ERROR, Status.UNKNOWN ] AnyStatusType = Union[Status, StatusType, int] @@ -116,6 +148,7 @@ class Status(Constants): StatusCategory, Literal[ StatusCategory.RUNNING, + StatusCategory.PENDING, StatusCategory.FINISHED, StatusCategory.FAILED, ], @@ -131,7 +164,7 @@ class Status(Constants): ] -def map_status(wps_status, compliant=StatusCompliant.OGC): +def map_status(wps_status, compliant=StatusCompliant.OGC): # pylint: disable=R1260 # type: (AnyStatusType, StatusCompliant) -> StatusType """ Maps WPS execution statuses to between compatible values of different implementations. @@ -160,21 +193,48 @@ def map_status(wps_status, compliant=StatusCompliant.OGC): if job_status in JOB_STATUS_CATEGORIES[StatusCategory.RUNNING]: if job_status in [Status.STARTED, Status.PAUSED]: job_status = Status.RUNNING + elif job_status == Status.QUEUED: + job_status = Status.ACCEPTED + elif job_status in [Status.CANCELED, Status.DISMISSED]: + job_status = Status.DISMISSED elif job_status in JOB_STATUS_CATEGORIES[StatusCategory.FAILED]: - if job_status not in [Status.FAILED, Status.DISMISSED]: - job_status = Status.FAILED + job_status = Status.FAILED + elif job_status == Status.FINISHED: + job_status = Status.SUCCEEDED elif compliant == StatusCompliant.PYWPS: - if job_status == Status.RUNNING: + if job_status in [Status.RUNNING]: job_status = Status.STARTED - elif job_status == Status.DISMISSED: + elif job_status in [Status.DISMISSED, Status.CANCELED]: + job_status = Status.FAILED + elif job_status in JOB_STATUS_CATEGORIES[StatusCategory.FAILED]: job_status = Status.FAILED + elif job_status in JOB_STATUS_CATEGORIES[StatusCategory.PENDING]: + job_status = Status.PAUSED + elif job_status in JOB_STATUS_CATEGORIES[StatusCategory.FINISHED]: + job_status = Status.SUCCEEDED elif compliant == StatusCompliant.OWSLIB: - if job_status == Status.STARTED: + if job_status in JOB_STATUS_CATEGORIES[StatusCategory.PENDING]: + job_status = Status.PAUSED + elif job_status in JOB_STATUS_CATEGORIES[StatusCategory.RUNNING]: job_status = Status.RUNNING - elif job_status in JOB_STATUS_CATEGORIES[StatusCategory.FAILED] and job_status != Status.FAILED: + elif job_status in JOB_STATUS_CATEGORIES[StatusCategory.FAILED]: job_status = Status.FAILED + elif job_status in JOB_STATUS_CATEGORIES[StatusCategory.FINISHED]: + job_status = Status.SUCCEEDED + + elif compliant == StatusCompliant.OPENEO: + if job_status in JOB_STATUS_CATEGORIES[StatusCategory.PENDING]: + job_status = Status.QUEUED + elif job_status == Status.DISMISSED: + job_status = Status.CANCELED + elif job_status in JOB_STATUS_CATEGORIES[StatusCategory.RUNNING]: + job_status = Status.RUNNING + elif job_status in JOB_STATUS_CATEGORIES[StatusCategory.FAILED]: + job_status = Status.ERROR + elif job_status in JOB_STATUS_CATEGORIES[StatusCategory.FINISHED]: + job_status = Status.FINISHED # FIXME: new official status is 'successful', but this breaks everywhere (tests, local/remote execute, etc.) # https://github.com/opengeospatial/ogcapi-processes/blob/master/openapi/schemas/processes-core/statusCode.yaml diff --git a/weaver/store/base.py b/weaver/store/base.py index 6f02c95b8..4e432e387 100644 --- a/weaver/store/base.py +++ b/weaver/store/base.py @@ -12,9 +12,9 @@ from pywps import Process as ProcessWPS from weaver.datatype import Bill, Job, Process, Quote, Service, VaultFile - from weaver.execute import AnyExecuteResponse, AnyExecuteReturnPreference + from weaver.execute import AnyExecuteMode, AnyExecuteResponse, AnyExecuteReturnPreference from weaver.sort import AnySortType - from weaver.status import AnyStatusSearch + from weaver.status import AnyStatusSearch, AnyStatusType from weaver.typedefs import ( AnyProcessRef, AnyServiceRef, @@ -174,7 +174,8 @@ def save_job(self, outputs=None, # type: Optional[ExecutionOutputs] is_workflow=False, # type: bool is_local=False, # type: bool - execute_async=True, # type: bool + execute_mode=None, # type: Optional[AnyExecuteMode] + execute_wait=None, # type: Optional[int] execute_response=None, # type: Optional[AnyExecuteResponse] execute_return=None, # type: Optional[AnyExecuteReturnPreference] custom_tags=None, # type: Optional[List[str]] @@ -185,6 +186,7 @@ def save_job(self, accept_type=None, # type: Optional[str] accept_language=None, # type: Optional[str] created=None, # type: Optional[datetime.datetime] + status=None, # type: Optional[AnyStatusType] ): # type: (...) -> Job raise NotImplementedError diff --git a/weaver/store/mongodb.py b/weaver/store/mongodb.py index 28ade6a53..ad70d01a8 100644 --- a/weaver/store/mongodb.py +++ b/weaver/store/mongodb.py @@ -4,7 +4,7 @@ import copy import logging import uuid -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast import pymongo from pymongo.collation import Collation @@ -63,10 +63,10 @@ from pymongo.collection import Collection - from weaver.execute import AnyExecuteResponse, AnyExecuteReturnPreference + from weaver.execute import AnyExecuteMode, AnyExecuteResponse, AnyExecuteReturnPreference from weaver.processes.types import AnyProcessType from weaver.sort import AnySortType - from weaver.status import AnyStatusSearch + from weaver.status import AnyStatusSearch, AnyStatusType from weaver.store.base import DatetimeIntervalType, JobGroupCategory, JobSearchResult from weaver.typedefs import ( AnyProcess, @@ -790,7 +790,8 @@ def save_job(self, outputs=None, # type: Optional[ExecutionOutputs] is_workflow=False, # type: bool is_local=False, # type: bool - execute_async=True, # type: bool + execute_mode=None, # type: Optional[AnyExecuteMode] + execute_wait=None, # type: Optional[int] execute_response=None, # type: Optional[AnyExecuteResponse] execute_return=None, # type: Optional[AnyExecuteReturnPreference] custom_tags=None, # type: Optional[List[str]] @@ -801,6 +802,7 @@ def save_job(self, accept_type=None, # type: Optional[str] accept_language=None, # type: Optional[str] created=None, # type: Optional[datetime.datetime] + status=None, # type: Optional[AnyStatusType] ): # type: (...) -> Job """ Creates a new :class:`Job` and stores it in mongodb. @@ -812,13 +814,13 @@ def save_job(self, tags.append(ProcessType.WORKFLOW) else: tags.append(ProcessType.APPLICATION) - if execute_async: - tags.append(ExecuteMode.ASYNC) - else: - tags.append(ExecuteMode.SYNC) + if execute_mode is None: + execute_mode = ExecuteMode.AUTO + tags.append(execute_mode) if not access: access = Visibility.PRIVATE + status = map_status(Status.get(status, default=Status.ACCEPTED)) process = process.id if isinstance(process, Process) else process service = service.id if isinstance(service, Service) else service new_job = Job({ @@ -828,8 +830,9 @@ def save_job(self, "process": process, # process identifier (WPS request) "inputs": inputs, "outputs": outputs, - "status": map_status(Status.ACCEPTED), - "execute_async": execute_async, + "status": status, + "execution_mode": execute_mode, + "execution_wait": execute_wait, "execution_response": execute_response, "execution_return": execute_return, "is_workflow": is_workflow, @@ -1047,6 +1050,7 @@ def _find_jobs_grouped(self, pipeline, group_categories): items = found[0]["items"] # convert to Job object where applicable, since pipeline result contains (category, jobs, count) items = [{k: (v if k != "jobs" else [Job(j) for j in v]) for k, v in i.items()} for i in items] + items = cast("JobGroupCategory", items) if has_provider: for group_result in items: group_service = group_result["category"].pop("service", None) @@ -1147,13 +1151,14 @@ def _apply_status_filter(status): statuses = set() for _status in status: if _status in StatusCategory: - category_status = JOB_STATUS_CATEGORIES[StatusCategory[_status]] - statuses = statuses.union(category_status) + status_cat = StatusCategory.get(_status) + category_statuses = JOB_STATUS_CATEGORIES[status_cat] + statuses = statuses.union(category_statuses) else: statuses.add(_status) search_filters["status"] = {"$in": list(statuses)} # type: ignore elif status: - search_filters["status"] = status[0] + search_filters["status"] = str(status[0]) return search_filters @staticmethod diff --git a/weaver/typedefs.py b/weaver/typedefs.py index 8471456f5..76e146120 100644 --- a/weaver/typedefs.py +++ b/weaver/typedefs.py @@ -377,7 +377,11 @@ class CWL_SchemaName(Protocol): AnyHeadersCookieContainer = Union[AnyHeadersContainer, AnyCookiesContainer] AnyRequestType = Union[PyramidRequest, WerkzeugRequest, PreparedRequest, RequestsRequest, DummyRequest] AnyResponseType = Union[PyramidResponse, WebobResponse, RequestsResponse, TestResponse] - AnyViewResponse = Union[PyramidResponse, WebobResponse, HTTPException, JSON] + AnyResponseClass = Union[PyramidResponse, WebobResponse, HTTPException] + AnyViewResponse = Union[AnyResponseClass, JSON] + AnyViewCallableContextRequest = Callable[[Any, AnyRequestType], AnyViewResponse] + AnyViewCallableRequestOnly = Callable[[AnyRequestType], AnyViewResponse] + AnyViewCallable = Union[AnyViewCallableContextRequest, AnyViewCallableRequestOnly] RequestMethod = Literal[ "HEAD", "GET", "POST", "PUT", "PATCH", "DELETE", "head", "get", "post", "put", "patch", "delete", @@ -893,6 +897,14 @@ class CWL_SchemaName(Protocol): "schema": NotRequired[Union[str, OpenAPISchema]], "default": NotRequired[bool], }, total=False) + LiteralDataDomainDataType = TypedDict("LiteralDataDomainDataType", { + "name": Required[str] + }) + LiteralDataDomainType = TypedDict("LiteralDataDomainType", { + "dataType": Required[LiteralDataDomainDataType], + "valueDefinition": NotRequired[AnyValueType], + "defaultValue": NotRequired[AnyValueType], + }, total=False) ProcessInputOutputItem = TypedDict("ProcessInputOutputItem", { "id": str, "title": NotRequired[str], @@ -901,6 +913,7 @@ class CWL_SchemaName(Protocol): "metadata": NotRequired[List[Metadata]], "schema": NotRequired[OpenAPISchema], "formats": NotRequired[List[FormatMediaType]], + "literalDataDomains": NotRequired[List[LiteralDataDomainType]], "minOccurs": int, "maxOccurs": Union[int, Literal["unbounded"]], }, total=False) @@ -974,6 +987,8 @@ class CWL_SchemaName(Protocol): }, total=True) ProcessExecution = TypedDict("ProcessExecution", { + "process": NotRequired[str], + "status": NotRequired[Literal["create"]], "mode": NotRequired[AnyExecuteMode], "response": NotRequired[AnyExecuteResponse], "inputs": NotRequired[ExecutionInputs], diff --git a/weaver/utils.py b/weaver/utils.py index 3dd180df2..8c492040a 100644 --- a/weaver/utils.py +++ b/weaver/utils.py @@ -90,6 +90,7 @@ MutableMapping, NoReturn, Optional, + Sequence, Tuple, Type, TypeVar, @@ -1538,7 +1539,7 @@ def islambda(func): def get_path_kvp(path, sep=",", **params): - # type: (str, str, **AnyValueType) -> str + # type: (str, str, **Union[AnyValueType, Sequence[AnyValueType]]) -> str """ Generates the URL with Key-Value-Pairs (:term:`KVP`) query parameters. diff --git a/weaver/wps/service.py b/weaver/wps/service.py index a5ffe306e..c5b4dabec 100644 --- a/weaver/wps/service.py +++ b/weaver/wps/service.py @@ -21,7 +21,6 @@ from weaver.formats import ContentType, guess_target_format from weaver.owsexceptions import OWSNoApplicableCode from weaver.processes.convert import wps2json_job_payload -from weaver.processes.execution import submit_job_handler from weaver.processes.types import ProcessType from weaver.processes.utils import get_process from weaver.store.base import StoreProcesses @@ -197,6 +196,8 @@ def _submit_job(self, wps_request): Returns the status response as is if XML, or convert it to JSON, according to request ``Accept`` header. """ + from weaver.processes.execution import submit_job_handler # pylint: disable=C0415 # circular import error + req = wps_request.http_request # type: Union[PyramidRequest, WerkzeugRequest] pid = wps_request.identifier ctx = get_wps_output_context(req) # re-validate here in case submitted via WPS endpoint instead of REST-API @@ -263,15 +264,17 @@ def prepare_process_for_execution(self, identifier): def execute(self, identifier, wps_request, uuid): # type: (str, Union[WPSRequest, WorkerRequest], str) -> Union[WPSResponse, HTTPValid] """ - Handles the ``Execute`` KVP/XML request submitted on the WPS endpoint. + Handles the ``Execute`` :term:`KVP`/:term:`XML` request submitted on the :term:`WPS` endpoint. - Submit WPS request to corresponding WPS-REST endpoint and convert back for requested ``Accept`` content-type. + Submit :term:`WPS` request to corresponding :term:`WPS-REST` endpoint and convert back for + requested ``Accept`` content-type. - Overrides the original execute operation, that will instead be handled by :meth:`execute_job` following - callback from Celery Worker, which handles process job creation and monitoring. + Overrides the original execute operation, that will instead be handled by :meth:`execute_job` + following callback from :mod:`celery` worker, which handles :term:`Job` creation and monitoring. - If ``Accept`` is JSON, the result is directly returned from :meth:`_submit_job`. - If ``Accept`` is XML or undefined, :class:`WorkerExecuteResponse` converts the received JSON with XML template. + If ``Accept`` is :term:`JSON`, the result is directly returned from :meth:`_submit_job`. + If ``Accept`` is :term:`XML` or undefined, :class:`WorkerExecuteResponse` converts the + received :term:`JSON` with :term:`XML` template. """ result = self._submit_job(wps_request) if not isinstance(result, dict): diff --git a/weaver/wps_restapi/api.py b/weaver/wps_restapi/api.py index c95bf5c66..adee8b822 100644 --- a/weaver/wps_restapi/api.py +++ b/weaver/wps_restapi/api.py @@ -97,6 +97,7 @@ def get_conformance(category, settings): ogcapi_proc_core = "http://www.opengis.net/spec/ogcapi-processes-1/1.0" ogcapi_proc_part2 = "http://www.opengis.net/spec/ogcapi-processes-2/1.0" ogcapi_proc_part3 = "http://www.opengis.net/spec/ogcapi-processes-3/0.0" + ogcapi_proc_part4 = "http://www.opengis.net/spec/ogcapi-processes-4/1.0" ogcapi_proc_apppkg = "http://www.opengis.net/spec/eoap-bp/1.0" # FIXME: https://github.com/crim-ca/weaver/issues/412 # ogcapi_proc_part3 = "http://www.opengis.net/spec/ogcapi-processes-3/1.0" @@ -366,12 +367,18 @@ def get_conformance(category, settings): f"{ogcapi_proc_core}/conf/ogc-process-description", f"{ogcapi_proc_core}/req/json", f"{ogcapi_proc_core}/req/json/definition", + f"{ogcapi_proc_core}/req/job-list/datetime-definition", + f"{ogcapi_proc_core}/req/job-list/datetime-response", + f"{ogcapi_proc_core}/req/job-list/duration-definition", + f"{ogcapi_proc_core}/req/job-list/duration-response", f"{ogcapi_proc_core}/req/job-list/links", f"{ogcapi_proc_core}/req/job-list/jl-limit-definition", f"{ogcapi_proc_core}/req/job-list/job-list-op", f"{ogcapi_proc_core}/req/job-list/processID-definition", f"{ogcapi_proc_core}/req/job-list/processID-mandatory", f"{ogcapi_proc_core}/req/job-list/processid-response", + f"{ogcapi_proc_core}/req/job-list/status-definition", + f"{ogcapi_proc_core}/req/job-list/status-response", f"{ogcapi_proc_core}/req/job-list/type-definition", f"{ogcapi_proc_core}/req/job-list/type-response", # FIXME: KVP exec (https://github.com/crim-ca/weaver/issues/607, https://github.com/crim-ca/weaver/issues/445) @@ -519,6 +526,35 @@ def get_conformance(category, settings): # FIXME: support openEO processes (https://github.com/crim-ca/weaver/issues/564) # f"{ogcapi_proc_part3}/conf/openeo-workflows", # f"{ogcapi_proc_part3}/req/openeo-workflows", + f"{ogcapi_proc_part4}/conf/jm/create/post-op", + f"{ogcapi_proc_part4}/per/job-management/additional-status-codes", # see 'weaver.status.map_status' + f"{ogcapi_proc_part4}/per/job-management/create-body", # Weaver has XML for WPS + f"{ogcapi_proc_part4}/per/job-management/create-content-schema", + f"{ogcapi_proc_part4}/per/job-management/update-body", + f"{ogcapi_proc_part4}/per/job-management/update-content-schema", + # FIXME: support part 3: Nested Workflow Execution request (https://github.com/crim-ca/weaver/issues/412) + # f"{ogcapi_proc_part4}/rec/job-management/create-body-ogcapi-processes", + # f"{ogcapi_proc_part4}/rec/job-management/update-body-ogcapi-processes", + # FIXME: support openEO processes (https://github.com/crim-ca/weaver/issues/564) + # f"{ogcapi_proc_part4}/rec/job-management/create-body-openeo", + # f"{ogcapi_proc_part4}/rec/job-management/update-body-openeo", + f"{ogcapi_proc_part4}/req/job-management/create-post-op", + f"{ogcapi_proc_part4}/req/job-management/create-content-type", + f"{ogcapi_proc_part4}/req/job-management/create-response-body", + f"{ogcapi_proc_part4}/req/job-management/create-response-jobid", + f"{ogcapi_proc_part4}/req/job-management/create-response-success", + # f"{ogcapi_proc_part4}/req/job-management/create-unsupported-schema", + f"{ogcapi_proc_part4}/req/job-management/create-unsupported-media-type", + f"{ogcapi_proc_part4}/req/job-management/definition-get-op", + f"{ogcapi_proc_part4}/req/job-management/definition-response-body", + f"{ogcapi_proc_part4}/req/job-management/definition-response-success", + f"{ogcapi_proc_part4}/req/job-management/start-post-op", + f"{ogcapi_proc_part4}/req/job-management/start-response", + f"{ogcapi_proc_part4}/req/job-management/update-body", + f"{ogcapi_proc_part4}/req/job-management/update-content-type", + f"{ogcapi_proc_part4}/req/job-management/update-patch-op", + f"{ogcapi_proc_part4}/req/job-management/update-response", + f"{ogcapi_proc_part4}/req/job-management/update-response-locked", # FIXME: employ 'weaver.wps_restapi.quotation.utils.check_quotation_supported' to add below conditionally # FIXME: https://github.com/crim-ca/weaver/issues/156 (billing/quotation) # https://github.com/opengeospatial/ogcapi-processes/tree/master/extensions/billing @@ -1039,9 +1075,14 @@ def format_response_details(response, request): http_headers = get_header("Content-Type", http_response.headers) or [] req_headers = get_header("Accept", request.headers) or [] if any([ContentType.APP_JSON in http_headers, ContentType.APP_JSON in req_headers]): + req_detail = get_request_info(request) + # return the response instead of generate less detailed one if it was already formed with JSON error details + # this can happen when a specific code like 404 triggers a pyramid lookup against other route/view handlers + if isinstance(response, HTTPException) and isinstance(req_detail, dict): + return response body = OWSException.json_formatter(http_response.status, response.message or "", http_response.title, request.environ) - body["detail"] = get_request_info(request) + body["detail"] = req_detail http_response._json = body if http_response.status_code != response.status_code: raise http_response # re-raise if code was fixed diff --git a/weaver/wps_restapi/colander_extras.py b/weaver/wps_restapi/colander_extras.py index 2c5f41b63..2f85b2aac 100644 --- a/weaver/wps_restapi/colander_extras.py +++ b/weaver/wps_restapi/colander_extras.py @@ -598,7 +598,7 @@ def serialize(self, node, appstruct): # noqa raise colander.Invalid( node, colander._( - "${val} cannot be serialized: ${err}", + "${val} cannot be processed: ${err}", mapping={"val": appstruct, "err": "Not 'null'."}, ), ) @@ -788,7 +788,7 @@ class SchemaB(MappingSchema): SchemaB().deserialize({"s1": None, "s2": {"field": "ok"}}) # results: {'s2': {'field': 'ok'}} - .. seealso: + .. seealso:: - https://github.com/Pylons/colander/issues/276 - https://github.com/Pylons/colander/issues/299 @@ -1446,6 +1446,11 @@ def _deserialize_impl(self, cstruct): # pylint: disable=W0222,signature-differs """ Converts the data using validation against the :term:`JSON` schema definition. """ + # don't inject the schema meta/id if the mapping is empty + # this is to avoid creating a non-empty mapping, which often as a "special" meaning + # furthermore, when the mapping is empty, there is no data to ensuring this schema is actually applied + if not cstruct: + return cstruct # meta-schema always disabled in this context since irrelevant # refer to the "id" of the parent schema representing this data using "$schema" # this is not "official" JSON requirement, but very common in practice diff --git a/weaver/wps_restapi/examples/job_status_created.json b/weaver/wps_restapi/examples/job_status_created.json new file mode 100644 index 000000000..bec621212 --- /dev/null +++ b/weaver/wps_restapi/examples/job_status_created.json @@ -0,0 +1,7 @@ +{ + "description": "Job successfully submitted for creation. Waiting on trigger request to being execution.", + "jobID": "797c0c5e-9bc2-4bf3-ab73-5f3df32044a8", + "processID": "Echo", + "status": "created", + "location": "http://schema-example.com/processes/Echo/jobs/797c0c5e-9bc2-4bf3-ab73-5f3df32044a8" +} diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index b2921ca9c..e5a17e953 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -3,30 +3,56 @@ from box import Box from celery.utils.log import get_task_logger from colander import Invalid -from pyramid.httpexceptions import HTTPBadRequest, HTTPOk, HTTPPermanentRedirect, HTTPUnprocessableEntity +from pyramid.httpexceptions import ( + HTTPBadRequest, + HTTPNoContent, + HTTPOk, + HTTPPermanentRedirect, + HTTPUnprocessableEntity, + HTTPUnsupportedMediaType +) +from weaver import xml_util from weaver.database import get_db from weaver.datatype import Job from weaver.exceptions import JobNotFound, JobStatisticsNotFound, log_unhandled_exceptions -from weaver.formats import ContentType, OutputFormat, add_content_type_charset, guess_target_format, repr_json +from weaver.execute import parse_prefer_header_execute_mode, rebuild_prefer_header +from weaver.formats import ( + ContentType, + OutputFormat, + add_content_type_charset, + clean_media_type_format, + guess_target_format, + repr_json +) +from weaver.processes.constants import JobInputsOutputsSchema, JobStatusSchema from weaver.processes.convert import convert_input_values_schema, convert_output_params_schema +from weaver.processes.execution import ( + submit_job, + submit_job_dispatch_task, + submit_job_dispatch_wps, + update_job_parameters +) from weaver.processes.utils import get_process from weaver.processes.wps_package import mask_process_inputs -from weaver.status import JOB_STATUS_CATEGORIES, Status, StatusCategory +from weaver.status import JOB_STATUS_CATEGORIES, Status, StatusCategory, StatusCompliant, map_status from weaver.store.base import StoreJobs -from weaver.utils import get_settings +from weaver.utils import get_header, get_settings, make_link_header from weaver.wps_restapi import swagger_definitions as sd from weaver.wps_restapi.jobs.utils import ( dismiss_job_task, get_job, + get_job_io_schema_query, get_job_list_links, get_job_results_response, + get_job_status_schema, get_results, - get_schema_query, - raise_job_bad_status, + raise_job_bad_status_locked, + raise_job_bad_status_success, raise_job_dismissed, validate_service_process ) +from weaver.wps_restapi.providers.utils import get_service from weaver.wps_restapi.swagger_definitions import datetime_interval_parser if TYPE_CHECKING: @@ -179,24 +205,145 @@ def _job_list(_jobs): # type: (Iterable[Job]) -> List[JSON] return Box(body) +@sd.jobs_service.post( + tags=[sd.TAG_EXECUTE, sd.TAG_JOBS], + content_type=list(ContentType.ANY_XML), + schema=sd.PostJobsEndpointXML(), + accept=ContentType.APP_JSON, + renderer=OutputFormat.JSON, + response_schemas=sd.post_jobs_responses, +) +@sd.jobs_service.post( + tags=[sd.TAG_EXECUTE, sd.TAG_JOBS, sd.TAG_PROCESSES], + content_type=ContentType.APP_JSON, + schema=sd.PostJobsEndpointJSON(), + accept=ContentType.APP_JSON, + renderer=OutputFormat.JSON, + response_schemas=sd.post_jobs_responses, +) +def create_job(request): + # type: (PyramidRequest) -> AnyViewResponse + """ + Create a new processing job with advanced management and execution capabilities. + """ + proc_key = "process" + proc_url = None + proc_id = None + prov_id = None + try: + ctype = get_header("Content-Type", request.headers, default=ContentType.APP_JSON) + ctype = clean_media_type_format(ctype, strip_parameters=True) + if ctype == ContentType.APP_JSON and "process" in request.json_body: + proc_url = request.json_body["process"] + proc_url = sd.ProcessURL().deserialize(proc_url) + prov_url, proc_id = proc_url.rsplit("/processes/", 1) + prov_parts = prov_url.rsplit("/providers/", 1) + prov_id = prov_parts[-1] if len(prov_parts) > 1 else None + elif ctype in ContentType.ANY_XML: + proc_key = "ows:Identifier" + body_xml = xml_util.fromstring(request.text) + proc_id = body_xml.xpath(proc_key, namespaces=body_xml.getroottree().nsmap)[0].text + except Exception as exc: + raise HTTPBadRequest(json={ + "title": "NoSuchProcess", + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process", + "detail": "Process URL or identifier reference could not be parsed.", + "status": HTTPBadRequest.code, + "cause": {"in": "body", proc_key: repr_json(proc_url, force_string=False)} + }) from exc + + if ctype in ContentType.ANY_XML: + process = get_process(process_id=proc_id) + return submit_job_dispatch_wps(request, process) + + if prov_id: + ref = get_service(request, provider_id=prov_id) + else: + ref = get_process(process_id=proc_id) + proc_id = None # ensure ref is used, process ID needed only for provider + return submit_job(request, ref, process_id=proc_id, tags=["wps-rest", "ogc-api"]) + + +@sd.jobs_service.post() +def create_job_unsupported_media_type(request): + # type: (PyramidRequest) -> AnyViewResponse + """ + Handle the case where no ``content_type`` was matched for decorated service handlers on :func:`create_job`. + + This operation must be defined with a separate service decorator allowing "any" ``content_type`` because + match by ``content_type`` is performed prior to invoking the applicable decorated view function. + Therefore, using a custom ``error_handler`` on the decorators of :func:`create_job` would never be invoked + since their preconditions would never be encountered. Decorated views that provide a ``content_type`` explicitly + are prioritized. Therefore, this will match any fallback ``content_type`` not already defined by another decorator. + + .. warning:: + It is very important that this is defined after :func:`create_job` such that its docstring is employed for + rendering the :term:`OpenAPI` definition instead of this docstring. + """ + ctype = get_header("Content-Type", request.headers) + return HTTPUnsupportedMediaType( + json={ + "title": "Unsupported Media Type", + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-4/1.0/unsupported-media-type", + "detail": "Process URL or identifier reference missing or invalid.", + "status": HTTPUnsupportedMediaType.code, + "cause": {"in": "headers", "name": "Content-Type", "value": ctype}, + } + ) + + +@sd.process_results_service.post( + tags=[sd.TAG_JOBS, sd.TAG_EXECUTE, sd.TAG_RESULTS, sd.TAG_PROCESSES], + schema=sd.ProcessJobResultsTriggerExecutionEndpoint(), + accept=ContentType.APP_JSON, + renderer=OutputFormat.JSON, + response_schemas=sd.post_job_results_responses, +) +@sd.job_results_service.post( + tags=[sd.TAG_JOBS, sd.TAG_EXECUTE, sd.TAG_RESULTS], + schema=sd.JobResultsTriggerExecutionEndpoint(), + accept=ContentType.APP_JSON, + renderer=OutputFormat.JSON, + response_schemas=sd.post_job_results_responses, +) +def trigger_job_execution(request): + # type: (PyramidRequest) -> AnyResponseType + """ + Trigger the execution of a previously created job. + """ + job = get_job(request) + raise_job_dismissed(job, request) + raise_job_bad_status_locked(job, request) + return submit_job_dispatch_task(job, container=request, force_submit=True) + + @sd.provider_job_service.get( tags=[sd.TAG_JOBS, sd.TAG_STATUS, sd.TAG_PROVIDERS], - schema=sd.ProviderJobEndpoint(), - accept=ContentType.APP_JSON, + schema=sd.GetProviderJobEndpoint(), + accept=[ContentType.APP_JSON] + [ + f"{ContentType.APP_JSON}; profile={profile}" + for profile in JobStatusSchema.values() + ], renderer=OutputFormat.JSON, response_schemas=sd.get_prov_single_job_status_responses, ) @sd.process_job_service.get( tags=[sd.TAG_PROCESSES, sd.TAG_JOBS, sd.TAG_STATUS], schema=sd.GetProcessJobEndpoint(), - accept=ContentType.APP_JSON, + accept=[ContentType.APP_JSON] + [ + f"{ContentType.APP_JSON}; profile={profile}" + for profile in JobStatusSchema.values() + ], renderer=OutputFormat.JSON, response_schemas=sd.get_single_job_status_responses, ) @sd.job_service.get( tags=[sd.TAG_JOBS, sd.TAG_STATUS], - schema=sd.JobEndpoint(), - accept=ContentType.APP_JSON, + schema=sd.GetJobEndpoint(), + accept=[ContentType.APP_JSON] + [ + f"{ContentType.APP_JSON}; profile={profile}" + for profile in JobStatusSchema.values() + ], renderer=OutputFormat.JSON, response_schemas=sd.get_single_job_status_responses, ) @@ -207,13 +354,51 @@ def get_job_status(request): Retrieve the status of a job. """ job = get_job(request) - job_status = job.json(request) - return HTTPOk(json=job_status) + job_body = job.json(request) + schema, headers = get_job_status_schema(request) + if schema == JobStatusSchema.OPENEO: + job_body["status"] = map_status(job_body["status"], StatusCompliant.OPENEO) + return HTTPOk(json=job_body, headers=headers) + + +@sd.provider_job_service.patch( + tags=[sd.TAG_JOBS, sd.TAG_PROVIDERS], + schema=sd.PatchProviderJobEndpoint(), + accept=ContentType.APP_JSON, + renderer=OutputFormat.JSON, + response_schemas=sd.patch_provider_job_responses, +) +@sd.process_job_service.patch( + tags=[sd.TAG_JOBS, sd.TAG_PROCESSES], + schema=sd.PatchProcessJobEndpoint(), + accept=ContentType.APP_JSON, + renderer=OutputFormat.JSON, + response_schemas=sd.patch_process_job_responses, +) +@sd.job_service.patch( + tags=[sd.TAG_JOBS], + schema=sd.PatchJobEndpoint(), + accept=ContentType.APP_JSON, + renderer=OutputFormat.JSON, + response_schemas=sd.patch_job_responses, +) +def update_pending_job(request): + # type: (PyramidRequest) -> AnyResponseType + """ + Update a previously created job still pending execution. + """ + job = get_job(request) + raise_job_dismissed(job, request) + raise_job_bad_status_locked(job, request) + update_job_parameters(job, request) + links = job.links(request, self_link="status") + headers = [("Link", make_link_header(link)) for link in links] + return HTTPNoContent(headers=headers) @sd.provider_job_service.delete( tags=[sd.TAG_JOBS, sd.TAG_DISMISS, sd.TAG_PROVIDERS], - schema=sd.ProviderJobEndpoint(), + schema=sd.DeleteProviderJobEndpoint(), accept=ContentType.APP_JSON, renderer=OutputFormat.JSON, response_schemas=sd.delete_prov_job_responses, @@ -227,7 +412,7 @@ def get_job_status(request): ) @sd.job_service.delete( tags=[sd.TAG_JOBS, sd.TAG_DISMISS], - schema=sd.JobEndpoint(), + schema=sd.DeleteJobEndpoint(), accept=ContentType.APP_JSON, renderer=OutputFormat.JSON, response_schemas=sd.delete_job_responses, @@ -337,17 +522,30 @@ def get_job_inputs(request): Retrieve the inputs values and outputs definitions of a job. """ job = get_job(request) - schema = get_schema_query(request.params.get("schema"), strict=False) + schema = get_job_io_schema_query(request.params.get("schema"), strict=False, default=JobInputsOutputsSchema.OGC) job_inputs = job.inputs job_outputs = job.outputs if job.is_local: process = get_process(job.process, request=request) job_inputs = mask_process_inputs(process.package, job_inputs) - if schema: - job_inputs = convert_input_values_schema(job_inputs, schema) - job_outputs = convert_output_params_schema(job_outputs, schema) - body = {"inputs": job_inputs, "outputs": job_outputs} - body.update({"links": job.links(request, self_link="inputs")}) + job_inputs = convert_input_values_schema(job_inputs, schema) + job_outputs = convert_output_params_schema(job_outputs, schema) + job_prefer = rebuild_prefer_header(job) + job_mode, _, _ = parse_prefer_header_execute_mode({"Prefer": job_prefer}, return_auto=True) + job_headers = { + "Accept": job.accept_type, + "Accept-Language": job.accept_language, + "Prefer": job_prefer, + "X-WPS-Output-Context": job.context, + } + body = { + "mode": job_mode, + "response": job.execution_response, + "inputs": job_inputs, + "outputs": job_outputs, + "headers": job_headers, + "links": job.links(request, self_link="inputs"), + } body = sd.JobInputsBody().deserialize(body) return HTTPOk(json=body) @@ -381,8 +579,8 @@ def get_job_outputs(request): """ job = get_job(request) raise_job_dismissed(job, request) - raise_job_bad_status(job, request) - schema = get_schema_query(request.params.get("schema")) + raise_job_bad_status_success(job, request) + schema = get_job_io_schema_query(request.params.get("schema"), default=JobInputsOutputsSchema.OGC) results, _ = get_results(job, request, schema=schema, link_references=False) outputs = {"outputs": results} outputs.update({"links": job.links(request, self_link="outputs")}) diff --git a/weaver/wps_restapi/jobs/utils.py b/weaver/wps_restapi/jobs/utils.py index 7a42e0e29..17aa3035d 100644 --- a/weaver/wps_restapi/jobs/utils.py +++ b/weaver/wps_restapi/jobs/utils.py @@ -3,7 +3,7 @@ import os import shutil from copy import deepcopy -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, cast, overload import colander from celery.utils.log import get_task_logger @@ -12,6 +12,7 @@ HTTPCreated, HTTPForbidden, HTTPInternalServerError, + HTTPLocked, HTTPNoContent, HTTPNotFound, HTTPOk @@ -39,9 +40,9 @@ parse_prefer_header_return, update_preference_applied_return_header ) -from weaver.formats import ContentEncoding, ContentType, get_format, repr_json +from weaver.formats import ContentEncoding, ContentType, clean_media_type_format, get_format, repr_json from weaver.owsexceptions import OWSNoApplicableCode, OWSNotFound -from weaver.processes.constants import JobInputsOutputsSchema +from weaver.processes.constants import JobInputsOutputsSchema, JobStatusSchema from weaver.processes.convert import any2wps_literal_datatype, convert_output_params_schema, get_field from weaver.status import JOB_STATUS_CATEGORIES, Status, StatusCategory, map_status from weaver.store.base import StoreJobs, StoreProcesses, StoreServices @@ -53,12 +54,14 @@ get_header, get_href_headers, get_path_kvp, + get_request_args, get_sane_name, get_secure_path, get_settings, get_weaver_url, is_uuid, - make_link_header + make_link_header, + parse_kvp ) from weaver.visibility import Visibility from weaver.wps.utils import get_wps_output_dir, get_wps_output_url, map_wps_output_location @@ -67,15 +70,16 @@ from weaver.wps_restapi.providers.utils import forbid_local_only if TYPE_CHECKING: - from typing import Any, Dict, List, Optional, Sequence, Tuple, Union + from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, Union from weaver.execute import AnyExecuteResponse, AnyExecuteReturnPreference, AnyExecuteTransmissionMode from weaver.formats import AnyContentEncoding - from weaver.processes.constants import JobInputsOutputsSchemaType + from weaver.processes.constants import JobInputsOutputsSchemaType, JobStatusSchemaType from weaver.typedefs import ( AnyDataStream, AnyHeadersContainer, AnyRequestType, + AnyResponseClass, AnyResponseType, AnySettingsContainer, AnyValueType, @@ -278,10 +282,22 @@ def get_job_list_links(job_total, filters, request): return links -def get_schema_query(schema, strict=True): - # type: (Optional[JobInputsOutputsSchemaType], bool) -> Optional[JobInputsOutputsSchemaType] +@overload +def get_job_io_schema_query( + schema, # type: Optional[str] + strict=True, # type: bool + default=None, # type: JobInputsOutputsSchemaType +): # type: (...) -> JobInputsOutputsSchemaType + ... + + +def get_job_io_schema_query( + schema, # type: Optional[str] + strict=True, # type: bool + default=None, # type: Optional[JobInputsOutputsSchemaType] +): # type: (...) -> Optional[JobInputsOutputsSchemaType] if not schema: - return None + return default # unescape query (eg: "OGC+strict" becomes "OGC string" from URL parsing) schema_checked = cast( "JobInputsOutputsSchemaType", @@ -300,6 +316,49 @@ def get_schema_query(schema, strict=True): return schema_checked +def get_job_status_schema(request): + # type: (AnyRequestType) -> Tuple[JobStatusSchemaType, HeadersType] + """ + Identifies if a :term:`Job` status response schema applies for the request. + """ + + def make_headers(resolved_schema): + # type: (JobStatusSchemaType) -> HeadersType + content_type = clean_media_type_format(content_accept, strip_parameters=True) + content_profile = f"{content_type}; profile={resolved_schema}" + content_headers = {"Content-Type": content_profile} + if resolved_schema == JobStatusSchema.OGC: + content_headers["Content-Schema"] = sd.OGC_API_SCHEMA_JOB_STATUS_URL + elif resolved_schema == JobStatusSchema.OPENEO: + content_headers["Content-Schema"] = sd.OPENEO_API_SCHEMA_JOB_STATUS_URL + return content_headers + + content_accept = request.accept.header_value or ContentType.APP_JSON + if content_accept == ContentType.ANY: + content_accept = ContentType.APP_JSON + + params = get_request_args(request) + schema = JobStatusSchema.get(params.get("profile") or params.get("schema")) + if schema: + headers = make_headers(schema) + return schema, headers + ctype = get_header("Accept", request.headers) + if not ctype: + return JobStatusSchema.OGC, {} + params = parse_kvp(ctype) + profile = params.get("profile") + if not profile: + schema = JobStatusSchema.OGC + headers = make_headers(schema) + return schema, headers + schema = cast( + "JobStatusSchemaType", + JobStatusSchema.get(profile[0], default=JobStatusSchema.OGC) + ) + headers = make_headers(schema) + return schema, headers + + def make_result_link( job, # type: Job result, # type: ExecutionResultValue @@ -567,7 +626,7 @@ def get_job_results_response( :param results_contents: Body contents that override originally submitted job parameters when requesting results. """ raise_job_dismissed(job, container) - raise_job_bad_status(job, container) + raise_job_bad_status_success(job, container) settings = get_settings(container) results, _ = get_results( @@ -1049,8 +1108,12 @@ def add_result_parts(result_parts): return resp -def get_job_submission_response(body, headers, error=False): - # type: (JSON, AnyHeadersContainer, bool) -> Union[HTTPOk, HTTPCreated, HTTPBadRequest] +def get_job_submission_response( + body, # type: JSON + headers, # type: AnyHeadersContainer + error=False, # type: bool + response_class=None, # type: Optional[Type[AnyResponseClass]] +): # type: (...) -> Union[AnyResponseClass, HTTPBadRequest] """ Generates the response contents returned by :term:`Job` submission process. @@ -1079,16 +1142,26 @@ def get_job_submission_response(body, headers, error=False): http_class = HTTPBadRequest http_desc = sd.FailedSyncJobResponse.description else: - http_class = HTTPOk + http_class = response_class or HTTPOk http_desc = sd.CompletedJobResponse.description body = sd.CompletedJobStatusSchema().deserialize(body) body["description"] = http_desc return http_class(json=body, headerlist=headers) - body["description"] = sd.CreatedLaunchJobResponse.description + if status == Status.CREATED: + body["description"] = ( + "Job successfully submitted for creation. " + "Waiting on trigger request to being execution." + ) + else: + body["description"] = ( + "Job successfully submitted to processing queue. " + "Execution should begin when resources are available." + ) body = sd.CreatedJobStatusSchema().deserialize(body) - return HTTPCreated(json=body, headerlist=headers) + http_class = response_class or HTTPCreated + return http_class(json=body, headerlist=headers) def validate_service_process(request): @@ -1169,13 +1242,42 @@ def validate_service_process(request): return service_name, process_name -def raise_job_bad_status(job, container=None): +def raise_job_bad_status_locked(job, container=None): + # type: (Job, Optional[AnySettingsContainer]) -> None + """ + Raise the appropriate message for :term:`Job` unable to be modified. + """ + if job.status != Status.CREATED: + links = job.links(container=container) + headers = [("Link", make_link_header(link)) for link in links] + job_reason = "" + if job.status in JOB_STATUS_CATEGORIES[StatusCategory.FINISHED]: + job_reason = " It has already finished execution." + elif job.status in JOB_STATUS_CATEGORIES[StatusCategory.PENDING]: + job_reason = " It is already queued for execution." + elif job.status in JOB_STATUS_CATEGORIES[StatusCategory.RUNNING]: + job_reason = " It is already executing." + raise HTTPLocked( + headers=headers, + json={ + "title": "Job Locked for Execution", + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-4/1.0/locked", + "detail": f"Job cannot be modified.{job_reason}", + "status": HTTPLocked.code, + "cause": {"status": job.status}, + "links": links + } + ) + + +def raise_job_bad_status_success(job, container=None): # type: (Job, Optional[AnySettingsContainer]) -> None """ Raise the appropriate message for :term:`Job` not ready or unable to retrieve output results due to status. """ if job.status != Status.SUCCEEDED: links = job.links(container=container) + headers = [("Link", make_link_header(link)) for link in links] if job.status == Status.FAILED: err_code = None err_info = None @@ -1203,26 +1305,32 @@ def raise_job_bad_status(job, container=None): err_code = OWSNoApplicableCode.code err_info = "unknown" # /req/core/job-results-failed - raise HTTPBadRequest(json={ - "title": "JobResultsFailed", - "type": err_code, - "detail": "Job results not available because execution failed.", - "status": HTTPBadRequest.code, - "cause": err_info, - "links": links - }) + raise HTTPBadRequest( + headers=headers, + json={ + "title": "JobResultsFailed", + "type": err_code, + "detail": "Job results not available because execution failed.", + "status": HTTPBadRequest.code, + "cause": err_info, + "links": links + } + ) # /req/core/job-results-exception/results-not-ready # must use OWS instead of HTTP class to preserve provided JSON body # otherwise, pyramid considers it as not found view/path and rewrites contents in append slash handler - raise OWSNotFound(json={ - "title": "JobResultsNotReady", - "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/result-not-ready", - "detail": "Job is not ready to obtain results.", - "status": HTTPNotFound.code, - "cause": {"status": job.status}, - "links": links - }) + raise OWSNotFound( + headers=headers, + json={ + "title": "JobResultsNotReady", + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/result-not-ready", + "detail": "Job is not ready to obtain results.", + "status": HTTPNotFound.code, + "cause": {"status": job.status}, + "links": links + } + ) def raise_job_dismissed(job, container=None): @@ -1235,7 +1343,9 @@ def raise_job_dismissed(job, container=None): settings = get_settings(container) job_links = job.links(settings) job_links = [link for link in job_links if link["rel"] in ["status", "alternate", "collection", "up"]] + headers = [("Link", make_link_header(link)) for link in job_links] raise JobGone( + headers=headers, json={ "title": "JobDismissed", "type": "JobDismissed", diff --git a/weaver/wps_restapi/patches.py b/weaver/wps_restapi/patches.py index ed0df1a3c..372687f40 100644 --- a/weaver/wps_restapi/patches.py +++ b/weaver/wps_restapi/patches.py @@ -4,13 +4,15 @@ import contextlib from typing import TYPE_CHECKING -from cornice import Service as ServiceAutoGetHead +from cornice import Service as CorniceService from pyramid.config import Configurator as PyramidConfigurator from pyramid.predicates import RequestMethodPredicate from pyramid.util import as_sorted_tuple if TYPE_CHECKING: - from typing import Any, Tuple, Union + from typing import Any, Callable, Optional, Sequence, Tuple, Union + + from weaver.typedefs import AnyViewCallable, RequestMethod class Configurator(PyramidConfigurator): @@ -68,7 +70,34 @@ def append(self, __object): super(NoAutoHeadList, self).append(__object) -class ServiceOnlyExplicitGetHead(ServiceAutoGetHead): +class ServiceAutoAcceptDecorator(CorniceService): + """ + Extends the view :meth:`decorator` to allow multiple ``accept`` headers provided all at once. + + The base :class:`CorniceService` only allows a single ``accept`` header value, which forces repeating the entire + parameters over multiple separate decorator calls. + """ + + def decorator(self, method, accept=None, **kwargs): + # type: (RequestMethod, Optional[str, Sequence[str]], Any) -> Callable[[AnyViewCallable], AnyViewCallable] + if not accept: + return super().decorator(method, **kwargs) # don't inject 'accept=None', causes cornice-swagger error + if isinstance(accept, str): + return super().decorator(method, accept=accept, **kwargs) + if not hasattr(accept, "__iter__") or not all(isinstance(header, str) for header in accept): # type: ignore + raise ValueError("Service decorator parameter 'accept' must be a single string or a sequence of strings.") + + def wrapper(view): + # type: (AnyViewCallable) -> AnyViewCallable + for header in accept: + wrap_view = CorniceService.decorator(self, method, accept=header, **kwargs) + wrap_view(view) + return view + + return wrapper + + +class ServiceOnlyExplicitGetHead(CorniceService): """ Service that disallow the auto-insertion of HTTP HEAD method view when HTTP GET view is defined. @@ -99,6 +128,12 @@ def add_view(self, method, view, **kwargs): super(ServiceOnlyExplicitGetHead, self).add_view(method, view, **kwargs) +class WeaverService(ServiceAutoAcceptDecorator, ServiceOnlyExplicitGetHead): + """ + Service that combines all respective capabilities required by :mod:`weaver`. + """ + + class RequestMethodPredicateNoGetHead(RequestMethodPredicate): # pylint: disable=W0231,super-init-not-called # whole point of this init is to bypass original behavior diff --git a/weaver/wps_restapi/processes/processes.py b/weaver/wps_restapi/processes/processes.py index 2ccf2f550..6f834d62b 100644 --- a/weaver/wps_restapi/processes/processes.py +++ b/weaver/wps_restapi/processes/processes.py @@ -14,7 +14,6 @@ ) from pyramid.response import Response from pyramid.settings import asbool -from werkzeug.wrappers.request import Request as WerkzeugRequest from weaver.database import get_db from weaver.exceptions import ProcessNotFound, ServiceException, log_unhandled_exceptions @@ -28,21 +27,12 @@ ) from weaver.processes import opensearch from weaver.processes.constants import ProcessSchema -from weaver.processes.execution import submit_job +from weaver.processes.execution import submit_job, submit_job_dispatch_wps from weaver.processes.utils import deploy_process_from_payload, get_process, update_process_metadata from weaver.status import Status from weaver.store.base import StoreJobs, StoreProcesses -from weaver.utils import ( - clean_json_text_body, - extend_instance, - fully_qualified_name, - get_any_id, - get_header, - get_path_kvp -) +from weaver.utils import clean_json_text_body, fully_qualified_name, get_any_id, get_header from weaver.visibility import Visibility -from weaver.wps.service import get_pywps_service -from weaver.wps.utils import get_wps_path from weaver.wps_restapi import swagger_definitions as sd from weaver.wps_restapi.processes.utils import get_process_list_links, get_processes_filtered_by_valid_schemas from weaver.wps_restapi.providers.utils import get_provider_services @@ -489,20 +479,11 @@ def submit_local_job(request): Execution location and method is according to deployed Application Package. """ process = get_process(request=request) - ctype = clean_media_type_format(get_header("content-type", request.headers, default=None), strip_parameters=True) + ctype = get_header("Content-Type", request.headers, default=None) + ctype = clean_media_type_format(ctype, strip_parameters=True) if ctype in ContentType.ANY_XML: - # Send the XML request to the WPS endpoint which knows how to parse it properly. - # Execution will end up in the same 'submit_job_handler' function as other branch for JSON. - service = get_pywps_service() - wps_params = {"version": "1.0.0", "request": "Execute", "service": "WPS", "identifier": process.id} - request.path_info = get_wps_path(request) - request.query_string = get_path_kvp("", **wps_params)[1:] - location = request.application_url + request.path_info + request.query_string - LOGGER.warning("Route redirection [%s] -> [%s] for WPS-XML support.", request.url, location) - http_request = extend_instance(request, WerkzeugRequest) - http_request.shallow = False - return service.call(http_request) - return submit_job(request, process, tags=["wps-rest"]) + return submit_job_dispatch_wps(request, process) + return submit_job(request, process, tags=["wps-rest", "ogc-api"]) def includeme(config): diff --git a/weaver/wps_restapi/providers/providers.py b/weaver/wps_restapi/providers/providers.py index 9aa752ead..57cbacdb0 100644 --- a/weaver/wps_restapi/providers/providers.py +++ b/weaver/wps_restapi/providers/providers.py @@ -17,8 +17,9 @@ from weaver.exceptions import ServiceNotFound, ServiceParsingError, log_unhandled_exceptions from weaver.formats import ContentType, OutputFormat from weaver.owsexceptions import OWSMissingParameterValue, OWSNotImplemented +from weaver.processes.execution import submit_job from weaver.store.base import StoreServices -from weaver.utils import get_any_id, get_settings +from weaver.utils import get_any_id from weaver.wps.utils import get_wps_client from weaver.wps_restapi import swagger_definitions as sd from weaver.wps_restapi.processes.utils import get_process_list_links @@ -141,7 +142,8 @@ def remove_provider(request): """ Remove an existing service provider. """ - service, store = get_service(request) + store = get_db(request).get_store(StoreServices) + service = get_service(request) try: store.delete_service(service.name) @@ -165,7 +167,7 @@ def get_provider(request): """ Get a provider definition (GetCapabilities). """ - service, _ = get_service(request) + service = get_service(request) data = get_schema_ref(sd.ProviderSummarySchema, request, ref_name=False) info = service.summary(request) data.update(info) @@ -208,14 +210,12 @@ def describe_provider_process(request): Note: this processes won't be stored to the local process storage. """ - provider_id = request.matchdict.get("provider_id") - process_id = request.matchdict.get("process_id") - store = get_db(request).get_store(StoreServices) - service = store.fetch_by_name(provider_id) + service = get_service(request) # FIXME: support other providers (https://github.com/crim-ca/weaver/issues/130) wps = get_wps_client(service.url, request) - process = wps.describeprocess(process_id) - return Process.convert(process, service, get_settings(request)) + proc_id = request.matchdict.get("process_id") + process = wps.describeprocess(proc_id) + return Process.convert(process, service, container=request) @sd.provider_process_service.get( @@ -278,11 +278,7 @@ def submit_provider_job(request): """ Execute a remote provider process. """ - from weaver.processes.execution import submit_job # isort:skip # noqa: E402 # pylint: disable=C0413 - - store = get_db(request).get_store(StoreServices) - provider_id = request.matchdict.get("provider_id") - service = store.fetch_by_name(provider_id) + service = get_service(request) return submit_job(request, service, tags=["wps-rest"]) diff --git a/weaver/wps_restapi/providers/utils.py b/weaver/wps_restapi/providers/utils.py index 41e7f6d82..feee53a02 100644 --- a/weaver/wps_restapi/providers/utils.py +++ b/weaver/wps_restapi/providers/utils.py @@ -11,7 +11,7 @@ from weaver.utils import get_settings if TYPE_CHECKING: - from typing import Any, Callable, List, Tuple + from typing import Any, Callable, List, Optional from weaver.datatype import Service from weaver.typedefs import AnyRequestType, AnySettingsContainer @@ -68,15 +68,15 @@ def forbid_local(container): return forbid_local -def get_service(request): - # type: (AnyRequestType) -> Tuple[Service, StoreServices] +def get_service(request, provider_id=None): + # type: (AnyRequestType, Optional[str]) -> Service """ Get the request service using provider_id from the service store. """ store = get_db(request).get_store(StoreServices) - provider_id = request.matchdict.get("provider_id") + prov_id = provider_id or request.matchdict.get("provider_id") try: - service = store.fetch_by_name(provider_id) + service = store.fetch_by_name(prov_id) except ServiceNotFound: - raise HTTPNotFound(f"Provider {provider_id} cannot be found.") - return service, store + raise HTTPNotFound(f"Provider {prov_id} cannot be found.") + return service diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index 1dcb2f655..e4975abff 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -102,6 +102,7 @@ PACKAGE_TYPE_POSSIBLE_VALUES, WPS_LITERAL_DATA_TYPES, JobInputsOutputsSchema, + JobStatusSchema, ProcessSchema ) from weaver.quotation.status import QuoteStatus @@ -138,7 +139,7 @@ XMLObject ) from weaver.wps_restapi.constants import ConformanceCategory -from weaver.wps_restapi.patches import ServiceOnlyExplicitGetHead as Service # warning: don't use 'cornice.Service' +from weaver.wps_restapi.patches import WeaverService as Service # warning: don't use 'cornice.Service' if TYPE_CHECKING: from typing import Any, Dict, Type, Union @@ -221,6 +222,11 @@ OGC_API_BBOX_FORMAT = "ogc-bbox" # equal CRS:84 and EPSG:4326, equivalent to WGS84 with swapped lat-lon order OGC_API_BBOX_EPSG = "EPSG:4326" +OGC_API_SCHEMA_JOB_STATUS_URL = f"{OGC_API_PROC_PART1_SCHEMAS}/statusInfo.yaml" + +OPENEO_API_SCHEMA_URL = "https://openeo.org/documentation/1.0/developers/api/openapi.yaml" +OPENEO_API_SCHEMA_JOB_STATUS_URL = f"{OPENEO_API_SCHEMA_URL}#/components/schemas/batch_job" + WEAVER_SCHEMA_VERSION = "master" WEAVER_SCHEMA_URL = f"https://raw.githubusercontent.com/crim-ca/weaver/{WEAVER_SCHEMA_VERSION}/weaver/schemas" @@ -696,6 +702,18 @@ class ResponseContentTypeHeader(ContentTypeHeader): ]) +class PreferHeader(ExtendedSchemaNode): + summary = "Header that describes job execution parameters." + description = ( + "Header that describes the desired execution mode of the process job and desired results. " + "Parameter 'return' indicates the structure and contents how results should be returned. " + "Parameter 'wait' and 'respond-async' indicate the execution mode of the process job. " + f"For more details, see {DOC_URL}/processes.html#execution-mode and {DOC_URL}/processes.html#execution-results." + ) + name = "Prefer" + schema_type = String + + class RequestHeaders(ExtendedMappingSchema): """ Headers that can indicate how to adjust the behavior and/or result to be provided in the response. @@ -705,7 +723,7 @@ class RequestHeaders(ExtendedMappingSchema): content_type = RequestContentTypeHeader() -class ResponseHeaders(ResponseContentTypeHeader): +class ResponseHeaders(ExtendedMappingSchema): """ Headers describing resulting response. """ @@ -2102,6 +2120,11 @@ class JobExecuteModeEnum(ExtendedSchemaNode): # no default to enforce required input as per OGC-API schemas # default = EXECUTE_MODE_AUTO example = ExecuteMode.ASYNC + description = ( + "Desired execution mode specified directly. This is intended for backward compatibility support. " + "To obtain more control over execution mode selection, employ the official Prefer header instead " + f"(see for more details: {DOC_URL}/processes.html#execution-mode)." + ) validator = OneOf(ExecuteMode.values()) @@ -2122,6 +2145,10 @@ class JobResponseOptionsEnum(ExtendedSchemaNode): # no default to enforce required input as per OGC-API schemas # default = ExecuteResponse.DOCUMENT example = ExecuteResponse.DOCUMENT + description = ( + "Indicates the desired representation format of the response. " + f"(see for more details: {DOC_URL}/processes.html#execution-body)." + ) validator = OneOf(ExecuteResponse.values()) @@ -2144,6 +2171,12 @@ class JobStatusEnum(ExtendedSchemaNode): validator = OneOf(JOB_STATUS_CODE_API) +class JobStatusCreate(ExtendedSchemaNode): + schema_type = String + title = "JobStatus" + validator = OneOf(["create"]) + + class JobStatusSearchEnum(ExtendedSchemaNode): schema_type = String title = "JobStatusSearch" @@ -2161,6 +2194,12 @@ class JobTypeEnum(ExtendedSchemaNode): validator = OneOf(["process", "provider", "service"]) +class JobTitle(ExtendedSchemaNode): + schema_type = String + description = "Title assigned to the job for user-readable identification." + validator = Length(min=1) + + class JobSortEnum(ExtendedSchemaNode): schema_type = String title = "JobSortingMethod" @@ -2212,13 +2251,16 @@ class JobExecuteSubscribers(ExtendedMappingSchema): success_uri = URL( name="successUri", description="Location where to POST the job results on successful completion.", + # allow omitting against official schema to support partial use/update + # (see https://github.com/opengeospatial/ogcapi-processes/issues/460) + missing=drop, ) - failure_uri = URL( + failed_uri = URL( name="failedUri", description="Location where to POST the job status if it fails execution.", missing=drop, ) - started_uri = URL( + in_progress_uri = URL( name="inProgressUri", description="Location where to POST the job status once it starts execution.", missing=drop, @@ -2229,12 +2271,12 @@ class JobExecuteSubscribers(ExtendedMappingSchema): description="Email recipient to send a notification on successful job completion.", missing=drop, ) - failure_email = Email( + failed_email = Email( name="failedEmail", description="Email recipient to send a notification on failed job completion.", missing=drop, ) - started_email = Email( + in_progress_email = Email( name="inProgressEmail", description="Email recipient to send a notification of job status once it starts execution.", missing=drop, @@ -2551,9 +2593,13 @@ class OWSIdentifier(ExtendedSchemaNode, OWSNamespace): name = "Identifier" -class OWSIdentifierList(ExtendedSequenceSchema, OWSNamespace): +class OWSProcessIdentifier(ProcessIdentifier, OWSNamespace): + pass + + +class OWSProcessIdentifierList(ExtendedSequenceSchema, OWSNamespace): name = "Identifiers" - item = OWSIdentifier() + item = OWSProcessIdentifier() class OWSTitle(ExtendedSchemaNode, OWSNamespace): @@ -2586,7 +2632,7 @@ class WPSDescribeProcessPost(WPSOperationPost, WPSNamespace): _schema = f"{OGC_WPS_1_SCHEMAS}/wpsDescribeProcess_request.xsd" name = "DescribeProcess" title = "DescribeProcess" - identifier = OWSIdentifierList( + identifier = OWSProcessIdentifierList( description="Single or comma-separated list of process identifier to describe.", example="example" ) @@ -2603,7 +2649,7 @@ class WPSExecutePost(WPSOperationPost, WPSNamespace): _schema = f"{OGC_WPS_1_SCHEMAS}/wpsExecute_request.xsd" name = "Execute" title = "Execute" - identifier = OWSIdentifier(description="Identifier of the process to execute with data inputs.") + identifier = OWSProcessIdentifier(description="Identifier of the process to execute with data inputs.") dataInputs = WPSExecuteDataInputs(description="Data inputs to be provided for process execution.") @@ -2777,7 +2823,7 @@ class ProcessVersion(ExtendedSchemaNode, WPSNamespace): class OWSProcessSummary(ExtendedMappingSchema, WPSNamespace): version = ProcessVersion(name="processVersion", default="None", example="1.2", description="Version of the corresponding process summary.") - identifier = OWSIdentifier(example="example", description="Identifier to refer to the process.") + identifier = OWSProcessIdentifier(example="example", description="Identifier to refer to the process.") _title = OWSTitle(example="Example Process", description="Title of the process.") abstract = OWSAbstract(example="Process for example schema.", description="Detail about the process.") @@ -3015,7 +3061,7 @@ class WPSStatus(ExtendedMappingSchema, WPSNamespace): class WPSProcessSummary(ExtendedMappingSchema, WPSNamespace): name = "Process" title = "Process" - identifier = OWSIdentifier() + identifier = OWSProcessIdentifier() _title = OWSTitle() abstract = OWSAbstract(missing=drop) @@ -3227,12 +3273,29 @@ class ProcessVisibilityPutEndpoint(LocalProcessPath): body = VisibilitySchema() -class ProviderJobEndpoint(ProviderProcessPath, JobPath): +class JobStatusQueryProfileSchema(ExtendedSchemaNode): + summary = "Job status schema representation." + description = "Selects the schema employed for representation of returned job status response." + schema_type = String + title = "JobStatusQuerySchema" + example = JobStatusSchema.OGC + default = JobStatusSchema.OGC + validator = OneOfCaseInsensitive(JobStatusSchema.values()) + + +class GetJobQuery(ExtendedMappingSchema): + schema = JobStatusQueryProfileSchema(missing=drop) + profile = JobStatusQueryProfileSchema(missing=drop) + + +class GetProviderJobEndpoint(ProviderProcessPath, JobPath): header = RequestHeaders() + querystring = GetJobQuery() -class JobEndpoint(JobPath): +class GetJobEndpoint(JobPath): header = RequestHeaders() + querystring = GetJobQuery() class ProcessInputsEndpoint(LocalProcessPath, JobPath): @@ -3248,7 +3311,7 @@ class JobInputsOutputsQuery(ExtendedMappingSchema): String(), title="JobInputsOutputsQuerySchema", example=JobInputsOutputsSchema.OGC, - default=JobInputsOutputsSchema.OLD, + default=JobInputsOutputsSchema.OGC, validator=OneOfCaseInsensitive(JobInputsOutputsSchema.values()), summary="Selects the schema employed for representation of submitted job inputs and outputs.", description=( @@ -3271,7 +3334,7 @@ class JobResultsQuery(FormatQuery): String(), title="JobOutputResultsSchema", example=JobInputsOutputsSchema.OGC, - default=JobInputsOutputsSchema.OLD, + default=JobInputsOutputsSchema.OGC, validator=OneOfCaseInsensitive(JobInputsOutputsSchema.values()), summary="Selects the schema employed for representation of job outputs.", description=( @@ -3329,10 +3392,19 @@ class ProviderResultsEndpoint(ProviderProcessPath, JobPath): header = RequestHeaders() -class JobResultsEndpoint(ProviderProcessPath, JobPath): +class JobResultsEndpoint(JobPath): header = RequestHeaders() +class JobResultsTriggerExecutionEndpoint(JobResultsEndpoint): + header = RequestHeaders() + body = NoContent() + + +class ProcessJobResultsTriggerExecutionEndpoint(JobResultsTriggerExecutionEndpoint, LocalProcessPath): + pass + + class ProviderExceptionsEndpoint(ProviderProcessPath, JobPath): header = RequestHeaders() @@ -3674,13 +3746,14 @@ def deserialize(self, cstruct): class JobStatusInfo(ExtendedMappingSchema): - _schema = f"{OGC_API_PROC_PART1_SCHEMAS}/statusInfo.yaml" + _schema = OGC_API_SCHEMA_JOB_STATUS_URL jobID = JobID() processID = ProcessIdentifierTag(missing=None, default=None, description="Process identifier corresponding to the job execution.") providerID = ProcessIdentifier(missing=None, default=None, description="Provider identifier corresponding to the job execution.") type = JobTypeEnum(description="Type of the element associated to the creation of this job.") + title = JobTitle(missing=drop) status = JobStatusEnum(description="Last updated status.") message = ExtendedSchemaNode(String(), missing=drop, description="Information about the last status update.") created = ExtendedSchemaNode(DateTime(), missing=drop, default=None, @@ -4167,24 +4240,29 @@ class Execute(ExecuteInputOutputs): "value": EXAMPLES["job_execute.json"], }, } + process = ProcessURL( + missing=drop, + description=( + "Process reference to be executed. " + "This parameter is required if the process cannot be inferred from the request endpoint." + ), + ) + title = JobTitle(missing=drop) + status = JobStatusCreate( + description=( + "Status to request creation of the job without submitting it to processing queue " + "and leave it pending until triggered by another results request to start it " + "(see *OGC API - Processes* - Part 4: Job Management)." + ), + missing=drop, + ) mode = JobExecuteModeEnum( missing=drop, default=ExecuteMode.AUTO, deprecated=True, - description=( - "Desired execution mode specified directly. This is intended for backward compatibility support. " - "To obtain more control over execution mode selection, employ the official Prefer header instead " - f"(see for more details: {DOC_URL}/processes.html#execution-mode)." - ), - validator=OneOf(ExecuteMode.values()) ) response = JobResponseOptionsEnum( missing=drop, # no default to ensure 'Prefer' header vs 'response' body resolution order can be performed - description=( - "Indicates the desired representation format of the response. " - f"(see for more details: {DOC_URL}/processes.html#execution-body)." - ), - validator=OneOf(ExecuteResponse.values()) ) notification_email = Email( missing=drop, @@ -5996,7 +6074,37 @@ class ResultsBody(OneOfKeywordSchema): ] +class WpsOutputContextHeader(ExtendedSchemaNode): + # ok to use 'name' in this case because target 'key' in the mapping must + # be that specific value but cannot have a field named with this format + name = "X-WPS-Output-Context" + description = ( + "Contextual location where to store WPS output results from job execution. ", + "When provided, value must be a directory or sub-directories slug. ", + "Resulting contextual location will be relative to server WPS outputs when no context is provided.", + ) + schema_type = String + missing = drop + example = "my-directory/sub-project" + default = None + + +class JobExecuteHeaders(ExtendedMappingSchema): + description = "Indicates the relevant headers that were supplied for job execution or a null value if omitted." + accept = AcceptHeader(missing=None) + accept_language = AcceptLanguageHeader(missing=None) + content_type = RequestContentTypeHeader(missing=None, default=None) + prefer = PreferHeader(missing=None) + x_wps_output_context = WpsOutputContextHeader(missing=None) + + class JobInputsBody(ExecuteInputOutputs): + # note: + # following definitions do not employ 'missing=drop' to explicitly indicate the fields + # this makes it easier to consider everything that could be implied when executing the job + mode = JobExecuteModeEnum(default=ExecuteMode.AUTO) + response = JobResponseOptionsEnum(default=None) + headers = JobExecuteHeaders(missing={}, default={}) links = LinkList(missing=drop) @@ -6465,23 +6573,9 @@ class PutProcessEndpoint(LocalProcessPath): body = PutProcessBodySchema() -class WpsOutputContextHeader(ExtendedSchemaNode): - # ok to use 'name' in this case because target 'key' in the mapping must - # be that specific value but cannot have a field named with this format - name = "X-WPS-Output-Context" - description = ( - "Contextual location where to store WPS output results from job execution. ", - "When provided, value must be a directory or sub-directories slug. ", - "Resulting contextual location will be relative to server WPS outputs when no context is provided.", - ) - schema_type = String - missing = drop - example = "my-directory/sub-project" - default = None - - class ExecuteHeadersBase(RequestHeaders): description = "Request headers supported for job execution." + prefer = PreferHeader(missing=drop) x_wps_output_context = WpsOutputContextHeader() @@ -6499,13 +6593,17 @@ class ExecuteHeadersXML(ExecuteHeadersBase): ) -class PostProcessJobsEndpointJSON(LocalProcessPath): +class PostJobsEndpointJSON(ExtendedMappingSchema): header = ExecuteHeadersJSON() querystring = LocalProcessQuery() body = Execute() -class PostProcessJobsEndpointXML(LocalProcessPath): +class PostProcessJobsEndpointJSON(PostJobsEndpointJSON, LocalProcessPath): + pass + + +class PostJobsEndpointXML(ExtendedMappingSchema): header = ExecuteHeadersXML() querystring = LocalProcessQuery() body = WPSExecutePost( @@ -6522,6 +6620,56 @@ class PostProcessJobsEndpointXML(LocalProcessPath): ) +class PostProcessJobsEndpointXML(PostJobsEndpointXML, LocalProcessPath): + pass + + +class JobTitleNullable(OneOfKeywordSchema): + description = "Job title to update, or unset if 'null'." + _one_of = [ + JobTitle(), + ExtendedSchemaNode(NoneType(), name="null"), # allow explicit 'title: null' to unset a predefined title + ] + + +class PatchJobBodySchema(Execute): + description = "Execution request contents to be updated." + # 'missing=null' ensures that, if a field is provided with an "empty" definition (JSON null, no-field dict, etc.), + # contents are passed down as is rather than dropping them (what 'missing=drop' would do due to DropableSchemaNode) + # this is to allow "unsetting" any values that could have been defined during job creation or previous updates + title = JobTitleNullable(missing=null) + subscribers = JobExecuteSubscribers(missing=null) + # all parameters that are not 'missing=drop' in original 'Execute' definition must be added to allow partial update + inputs = ExecuteInputValues(missing=drop, description="Input values or references to be updated.") + outputs = ExecuteOutputSpec(missing=drop, description="Output format and transmission mode to be updated.") + + +class PatchJobEndpoint(JobPath): + summary = "Execution request parameters to be updated." + description = ( + "Execution request parameters to be updated. " + "If parameters are omitted, they will be left unmodified. " + "If provided, parameters will override existing definitions integrally. " + "Therefore, if only a partial update of certain nested elements in a mapping or list is desired, " + "all elements under the corresponding parameters must be resubmitted entirely with the applied changes. " + "In the case of certain parameters, equivalent definitions can cause conflicting definitions between " + "headers and contents " + f"(see for more details: {DOC_URL}/processes.html#execution-body and {DOC_URL}/processes.html#execution-mode). " + "To verify the resulting parameterization of any pending job, consider using the `GET /jobs/{jobId}/inputs`." + ) + header = JobExecuteHeaders() + querystring = LocalProcessQuery() + body = PatchJobBodySchema() + + +class PatchProcessJobEndpoint(JobPath, ProcessEndpoint): + body = PatchJobBodySchema() + + +class PatchProviderJobEndpoint(PatchProcessJobEndpoint): + header = RequestHeaders() + + class PagingQueries(ExtendedMappingSchema): page = ExtendedSchemaNode(Integer(allow_string=True), missing=0, default=0, validator=Range(min=0)) limit = ExtendedSchemaNode(Integer(allow_string=True), missing=10, default=10, validator=Range(min=1, max=1000), @@ -6610,14 +6758,26 @@ class DeleteProviderJobsEndpoint(DeleteJobsEndpoint, ProviderProcessPath): pass +class GetProcessJobQuery(LocalProcessQuery, GetJobQuery): + pass + + class GetProcessJobEndpoint(LocalProcessPath): + header = RequestHeaders() + querystring = GetProcessJobQuery() + + +class DeleteJobEndpoint(JobPath): header = RequestHeaders() querystring = LocalProcessQuery() class DeleteProcessJobEndpoint(LocalProcessPath): header = RequestHeaders() - querystring = LocalProcessQuery() + + +class DeleteProviderJobEndpoint(DeleteProcessJobEndpoint, ProviderProcessPath): + pass class BillsEndpoint(ExtendedMappingSchema): @@ -6789,7 +6949,7 @@ def __new__(cls, *, name, description, **kwargs): # pylint: disable=W0221 "New schema name must be provided to avoid invalid mixed use of $ref pointers. " f"Name '{name}' is invalid." ) - obj = super().__new__(cls) + obj = super().__new__(cls) # type: ExtendedSchemaNode obj.__init__(name=name, description=description) obj.__class__.__name__ = name obj.children = [ @@ -6805,11 +6965,6 @@ def __deepcopy__(self, *args, **kwargs): return GenericHTMLResponse(name=self.name, description=self.description, children=self.children) -class ErrorDetail(ExtendedMappingSchema): - code = ExtendedSchemaNode(Integer(), description="HTTP status code.", example=400) - status = ExtendedSchemaNode(String(), description="HTTP status detail.", example="400 Bad Request") - - class OWSErrorCode(ExtendedSchemaNode): schema_type = String example = "InvalidParameterValue" @@ -6828,6 +6983,18 @@ class OWSExceptionResponse(ExtendedMappingSchema): description="Specific description of the error.") +class ErrorDetail(ExtendedMappingSchema): + code = ExtendedSchemaNode(Integer(), description="HTTP status code.", example=400) + status = ExtendedSchemaNode(String(), description="HTTP status detail.", example="400 Bad Request") + + +class ErrorSource(OneOfKeywordSchema): + _one_of = [ + ExtendedSchemaNode(String(), description="Error name or description."), + ErrorDetail(description="Detailed error representation.") + ] + + class ErrorCause(OneOfKeywordSchema): _one_of = [ ExtendedSchemaNode(String(), description="Error message from exception or cause of failure."), @@ -6844,7 +7011,7 @@ class ErrorJsonResponseBodySchema(ExtendedMappingSchema): status = ExtendedSchemaNode(Integer(), description="Error status code.", example=500, missing=drop) cause = ErrorCause(missing=drop) value = ErrorCause(missing=drop) - error = ErrorDetail(missing=drop) + error = ErrorSource(missing=drop) instance = URI(missing=drop) exception = OWSExceptionResponse(missing=drop) @@ -6866,7 +7033,7 @@ class ConflictRequestResponseSchema(ServerErrorBaseResponseSchema): class UnprocessableEntityResponseSchema(ServerErrorBaseResponseSchema): - description = "Wrong format of given parameters." + description = "Wrong format or schema of given parameters." header = ResponseHeaders() body = ErrorJsonResponseBodySchema() @@ -7184,11 +7351,18 @@ class CreatedJobLocationHeader(ResponseHeaders): class CreatedLaunchJobResponse(ExtendedMappingSchema): - description = "Job successfully submitted to processing queue. Execution should begin when resources are available." + description = ( + "Job successfully submitted. " + "Execution should begin when resources are available or when triggered, according to requested execution mode." + ) examples = { "JobAccepted": { - "summary": "Job accepted for execution.", + "summary": "Job accepted for execution asynchronously.", "value": EXAMPLES["job_status_accepted.json"] + }, + "JobCreated": { + "summary": "Job created for later execution by trigger.", + "value": EXAMPLES["job_status_created.json"] } } header = CreatedJobLocationHeader() @@ -7245,6 +7419,12 @@ class OkDismissJobResponse(ExtendedMappingSchema): body = DismissedJobSchema() +class NoContentJobUpdatedResponse(ExtendedMappingSchema): + description = "Job detail updated with provided parameters." + header = ResponseHeaders() + body = NoContent() + + class OkGetJobStatusResponse(ExtendedMappingSchema): _schema = f"{OGC_API_PROC_PART1_RESPONSES}/Status.yaml" header = ResponseHeaders() @@ -7312,6 +7492,7 @@ class NoContentJobResultsHeaders(NoContent): class NoContentJobResultsResponse(ExtendedMappingSchema): + description = "Job completed execution synchronously with results returned in Link headers." header = NoContentJobResultsHeaders() body = NoContent(default="") @@ -7750,25 +7931,35 @@ class GoneVaultFileDownloadResponse(ExtendedMappingSchema): "501": NotImplementedPostProviderResponse(), } post_provider_process_job_responses = { - "200": CompletedJobResponse(description="success"), - "201": CreatedLaunchJobResponse(description="success"), - "204": NoContentJobResultsResponse(description="success"), + "200": CompletedJobResponse(), + "201": CreatedLaunchJobResponse(), + "204": NoContentJobResultsResponse(), "400": InvalidJobParametersResponse(), "403": ForbiddenProviderAccessResponseSchema(), "405": MethodNotAllowedErrorResponseSchema(), "406": NotAcceptableErrorResponseSchema(), + "415": UnsupportedMediaTypeResponseSchema(), + "422": UnprocessableEntityResponseSchema(), "500": InternalServerErrorResponseSchema(), } post_process_jobs_responses = { - "200": CompletedJobResponse(description="success"), - "201": CreatedLaunchJobResponse(description="success"), - "204": NoContentJobResultsResponse(description="success"), + "200": CompletedJobResponse(), + "201": CreatedLaunchJobResponse(), + "204": NoContentJobResultsResponse(), "400": InvalidJobParametersResponse(), "403": ForbiddenProviderAccessResponseSchema(), "405": MethodNotAllowedErrorResponseSchema(), "406": NotAcceptableErrorResponseSchema(), + "415": UnsupportedMediaTypeResponseSchema(), + "422": UnprocessableEntityResponseSchema(), "500": InternalServerErrorResponseSchema(), } +post_jobs_responses = copy(post_process_jobs_responses) +post_job_results_responses = copy(post_process_jobs_responses) +post_job_results_responses.pop("201") # job already created, therefore invalid +post_job_results_responses.update({ + "202": CreatedLaunchJobResponse(), # alternate to '201' for async case since job already exists +}) get_all_jobs_responses = { "200": OkGetQueriedJobsResponse(description="success", examples={ "JobListing": { @@ -7814,6 +8005,17 @@ class GoneVaultFileDownloadResponse(ExtendedMappingSchema): get_prov_single_job_status_responses.update({ "403": ForbiddenProviderLocalResponseSchema(), }) +patch_job_responses = { + "204": NoContentJobUpdatedResponse(), + "404": NotFoundJobResponseSchema(), + "405": MethodNotAllowedErrorResponseSchema(), + "406": NotAcceptableErrorResponseSchema(), + "415": UnsupportedMediaTypeResponseSchema(), + "422": UnprocessableEntityResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} +patch_process_job_responses = copy(patch_job_responses) +patch_provider_job_responses = copy(patch_job_responses) delete_job_responses = { "200": OkDismissJobResponse(description="success", examples={ "JobDismissedSuccess": { diff --git a/weaver/wps_restapi/utils.py b/weaver/wps_restapi/utils.py index 8cc695951..883ea65b7 100644 --- a/weaver/wps_restapi/utils.py +++ b/weaver/wps_restapi/utils.py @@ -6,38 +6,18 @@ from typing import TYPE_CHECKING import colander -import yaml from box import Box from pyramid.events import BeforeRender, subscriber -from pyramid.httpexceptions import ( - HTTPBadRequest, - HTTPInternalServerError, - HTTPSuccessful, - HTTPUnprocessableEntity, - HTTPUnsupportedMediaType, - status_map -) +from pyramid.httpexceptions import HTTPBadRequest, HTTPSuccessful, status_map from weaver import __meta__ -from weaver.formats import repr_json from weaver.utils import get_header, get_settings, get_weaver_url from weaver.wps_restapi import swagger_definitions as sd if TYPE_CHECKING: - from typing import Any, Dict, Optional, Union + from typing import Any, Dict, Optional - from weaver.formats import ContentType - from weaver.typedefs import ( - AnyCallableWrapped, - AnyRequestType, - AnySettingsContainer, - CWL, - HeadersType, - JSON, - Params, - Return, - SettingsType - ) + from weaver.typedefs import AnyCallableWrapped, AnySettingsContainer, HeadersType, Params, Return, SettingsType LOGGER = logging.getLogger(__name__) @@ -187,64 +167,6 @@ def wrapped(*args, **kwargs): return decorator -def parse_content(request=None, # type: Optional[AnyRequestType] - content=None, # type: Optional[Union[JSON, str]] - content_schema=None, # type: Optional[colander.SchemaNode] - content_type=sd.RequestContentTypeHeader.default, # type: Optional[ContentType] - content_type_schema=sd.RequestContentTypeHeader, # type: Optional[colander.SchemaNode] - ): # type: (...) -> Union[JSON, CWL] - """ - Load the request content with validation of expected content type and their schema. - """ - if request is None and content is None: # pragma: no cover # safeguard for early detect invalid implementation - raise HTTPInternalServerError(json={ - "title": "Internal Server Error", - "type": "InternalServerError", - "detail": "Cannot parse undefined contents.", - "status": HTTPInternalServerError.code, - "cause": "Request content and content argument are undefined.", - }) - try: - if request is not None: - content = request.text - content_type = request.content_type - if content_type is not None and content_type_schema is not None: - content_type = content_type_schema().deserialize(content_type) - if isinstance(content, str): - content = yaml.safe_load(content) - if not isinstance(content, dict): - raise TypeError("Not a valid JSON body for process deployment.") - except colander.Invalid as exc: - raise HTTPUnsupportedMediaType(json={ - "title": "Unsupported Media Type", - "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-2/1.0/unsupported-media-type", - "detail": str(exc), - "status": HTTPUnsupportedMediaType.code, - "cause": {"Content-Type": None if content_type is None else str(content_type)}, - }) - except Exception as exc: - raise HTTPBadRequest(json={ - "title": "Bad Request", - "type": "BadRequest", - "detail": "Unable to parse contents.", - "status": HTTPBadRequest.code, - "cause": str(exc), - }) - try: - if content_schema is not None: - content = content_schema().deserialize(content) - except colander.Invalid as exc: - raise HTTPUnprocessableEntity(json={ - "type": "InvalidParameterValue", - "title": "Failed schema validation.", - "status": HTTPUnprocessableEntity.code, - "error": colander.Invalid.__name__, - "cause": exc.msg, - "value": repr_json(exc.value, force_string=False), - }) - return content - - @subscriber(BeforeRender) def add_renderer_context(event): # type: (BeforeRender) -> None