From 9f40a129d3deeb4143b64bd7fd32247cf64998e6 Mon Sep 17 00:00:00 2001 From: Jeremy McCormick Date: Fri, 25 Oct 2024 17:13:21 -0500 Subject: [PATCH 1/7] Delete schemas symlink --- python/lsst/sdm_schemas/schemas | 1 - 1 file changed, 1 deletion(-) delete mode 120000 python/lsst/sdm_schemas/schemas diff --git a/python/lsst/sdm_schemas/schemas b/python/lsst/sdm_schemas/schemas deleted file mode 120000 index d9f31248..00000000 --- a/python/lsst/sdm_schemas/schemas +++ /dev/null @@ -1 +0,0 @@ -../../../yml/ \ No newline at end of file From 593eac140f0c9c9ce7cc908ee00522629e13927f Mon Sep 17 00:00:00 2001 From: Jeremy McCormick Date: Fri, 25 Oct 2024 17:14:55 -0500 Subject: [PATCH 2/7] Move schemas into Python source tree --- {yml => python/lsst/sdm_schemas/schemas}/README.md | 0 {yml => python/lsst/sdm_schemas/schemas}/apdb.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/archive/README.md | 0 .../lsst/sdm_schemas/schemas}/archive/baselineSchema.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/archive/dp01_dc2.yaml | 0 .../lsst/sdm_schemas/schemas}/archive/sdss_stripe82_01.yml | 0 {yml => python/lsst/sdm_schemas/schemas}/archive/wise_00.yml | 0 {yml => python/lsst/sdm_schemas/schemas}/archive/wise_01.yml | 0 {yml => python/lsst/sdm_schemas/schemas}/cdb_latiss.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/cdb_lsstcomcam.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/cdb_lsstcomcamsim.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/cdb_startrackerfast.yaml | 0 .../lsst/sdm_schemas/schemas}/cdb_startrackernarrow.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/cdb_startrackerwide.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/dp02_dc2.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/dp02_obscore.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/dp03_10yr.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/dp03_1yr.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/hsc.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/imsim.yaml | 0 .../lsst/sdm_schemas/schemas}/ivoa/ObsCore-v1.1-descriptions.csv | 0 .../lsst/sdm_schemas/schemas}/ivoa/ObsCore-v1.1-mandatory.csv | 0 .../lsst/sdm_schemas/schemas}/ivoa/ObsCore-v1.1-optional.csv | 0 {yml => python/lsst/sdm_schemas/schemas}/ivoa/README.md | 0 .../lsst/sdm_schemas/schemas}/ivoa/make_obscore_nominal.py | 0 .../lsst/sdm_schemas/schemas}/ivoa/obscore_nominal.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/obsloctap.yaml | 0 {yml => python/lsst/sdm_schemas/schemas}/oga_live_obscore.yaml | 0 28 files changed, 0 insertions(+), 0 deletions(-) rename {yml => python/lsst/sdm_schemas/schemas}/README.md (100%) rename {yml => python/lsst/sdm_schemas/schemas}/apdb.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/archive/README.md (100%) rename {yml => python/lsst/sdm_schemas/schemas}/archive/baselineSchema.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/archive/dp01_dc2.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/archive/sdss_stripe82_01.yml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/archive/wise_00.yml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/archive/wise_01.yml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/cdb_latiss.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/cdb_lsstcomcam.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/cdb_lsstcomcamsim.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/cdb_startrackerfast.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/cdb_startrackernarrow.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/cdb_startrackerwide.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/dp02_dc2.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/dp02_obscore.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/dp03_10yr.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/dp03_1yr.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/hsc.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/imsim.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/ivoa/ObsCore-v1.1-descriptions.csv (100%) rename {yml => python/lsst/sdm_schemas/schemas}/ivoa/ObsCore-v1.1-mandatory.csv (100%) rename {yml => python/lsst/sdm_schemas/schemas}/ivoa/ObsCore-v1.1-optional.csv (100%) rename {yml => python/lsst/sdm_schemas/schemas}/ivoa/README.md (100%) rename {yml => python/lsst/sdm_schemas/schemas}/ivoa/make_obscore_nominal.py (100%) rename {yml => python/lsst/sdm_schemas/schemas}/ivoa/obscore_nominal.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/obsloctap.yaml (100%) rename {yml => python/lsst/sdm_schemas/schemas}/oga_live_obscore.yaml (100%) diff --git a/yml/README.md b/python/lsst/sdm_schemas/schemas/README.md similarity index 100% rename from yml/README.md rename to python/lsst/sdm_schemas/schemas/README.md diff --git a/yml/apdb.yaml b/python/lsst/sdm_schemas/schemas/apdb.yaml similarity index 100% rename from yml/apdb.yaml rename to python/lsst/sdm_schemas/schemas/apdb.yaml diff --git a/yml/archive/README.md b/python/lsst/sdm_schemas/schemas/archive/README.md similarity index 100% rename from yml/archive/README.md rename to python/lsst/sdm_schemas/schemas/archive/README.md diff --git a/yml/archive/baselineSchema.yaml b/python/lsst/sdm_schemas/schemas/archive/baselineSchema.yaml similarity index 100% rename from yml/archive/baselineSchema.yaml rename to python/lsst/sdm_schemas/schemas/archive/baselineSchema.yaml diff --git a/yml/archive/dp01_dc2.yaml b/python/lsst/sdm_schemas/schemas/archive/dp01_dc2.yaml similarity index 100% rename from yml/archive/dp01_dc2.yaml rename to python/lsst/sdm_schemas/schemas/archive/dp01_dc2.yaml diff --git a/yml/archive/sdss_stripe82_01.yml b/python/lsst/sdm_schemas/schemas/archive/sdss_stripe82_01.yml similarity index 100% rename from yml/archive/sdss_stripe82_01.yml rename to python/lsst/sdm_schemas/schemas/archive/sdss_stripe82_01.yml diff --git a/yml/archive/wise_00.yml b/python/lsst/sdm_schemas/schemas/archive/wise_00.yml similarity index 100% rename from yml/archive/wise_00.yml rename to python/lsst/sdm_schemas/schemas/archive/wise_00.yml diff --git a/yml/archive/wise_01.yml b/python/lsst/sdm_schemas/schemas/archive/wise_01.yml similarity index 100% rename from yml/archive/wise_01.yml rename to python/lsst/sdm_schemas/schemas/archive/wise_01.yml diff --git a/yml/cdb_latiss.yaml b/python/lsst/sdm_schemas/schemas/cdb_latiss.yaml similarity index 100% rename from yml/cdb_latiss.yaml rename to python/lsst/sdm_schemas/schemas/cdb_latiss.yaml diff --git a/yml/cdb_lsstcomcam.yaml b/python/lsst/sdm_schemas/schemas/cdb_lsstcomcam.yaml similarity index 100% rename from yml/cdb_lsstcomcam.yaml rename to python/lsst/sdm_schemas/schemas/cdb_lsstcomcam.yaml diff --git a/yml/cdb_lsstcomcamsim.yaml b/python/lsst/sdm_schemas/schemas/cdb_lsstcomcamsim.yaml similarity index 100% rename from yml/cdb_lsstcomcamsim.yaml rename to python/lsst/sdm_schemas/schemas/cdb_lsstcomcamsim.yaml diff --git a/yml/cdb_startrackerfast.yaml b/python/lsst/sdm_schemas/schemas/cdb_startrackerfast.yaml similarity index 100% rename from yml/cdb_startrackerfast.yaml rename to python/lsst/sdm_schemas/schemas/cdb_startrackerfast.yaml diff --git a/yml/cdb_startrackernarrow.yaml b/python/lsst/sdm_schemas/schemas/cdb_startrackernarrow.yaml similarity index 100% rename from yml/cdb_startrackernarrow.yaml rename to python/lsst/sdm_schemas/schemas/cdb_startrackernarrow.yaml diff --git a/yml/cdb_startrackerwide.yaml b/python/lsst/sdm_schemas/schemas/cdb_startrackerwide.yaml similarity index 100% rename from yml/cdb_startrackerwide.yaml rename to python/lsst/sdm_schemas/schemas/cdb_startrackerwide.yaml diff --git a/yml/dp02_dc2.yaml b/python/lsst/sdm_schemas/schemas/dp02_dc2.yaml similarity index 100% rename from yml/dp02_dc2.yaml rename to python/lsst/sdm_schemas/schemas/dp02_dc2.yaml diff --git a/yml/dp02_obscore.yaml b/python/lsst/sdm_schemas/schemas/dp02_obscore.yaml similarity index 100% rename from yml/dp02_obscore.yaml rename to python/lsst/sdm_schemas/schemas/dp02_obscore.yaml diff --git a/yml/dp03_10yr.yaml b/python/lsst/sdm_schemas/schemas/dp03_10yr.yaml similarity index 100% rename from yml/dp03_10yr.yaml rename to python/lsst/sdm_schemas/schemas/dp03_10yr.yaml diff --git a/yml/dp03_1yr.yaml b/python/lsst/sdm_schemas/schemas/dp03_1yr.yaml similarity index 100% rename from yml/dp03_1yr.yaml rename to python/lsst/sdm_schemas/schemas/dp03_1yr.yaml diff --git a/yml/hsc.yaml b/python/lsst/sdm_schemas/schemas/hsc.yaml similarity index 100% rename from yml/hsc.yaml rename to python/lsst/sdm_schemas/schemas/hsc.yaml diff --git a/yml/imsim.yaml b/python/lsst/sdm_schemas/schemas/imsim.yaml similarity index 100% rename from yml/imsim.yaml rename to python/lsst/sdm_schemas/schemas/imsim.yaml diff --git a/yml/ivoa/ObsCore-v1.1-descriptions.csv b/python/lsst/sdm_schemas/schemas/ivoa/ObsCore-v1.1-descriptions.csv similarity index 100% rename from yml/ivoa/ObsCore-v1.1-descriptions.csv rename to python/lsst/sdm_schemas/schemas/ivoa/ObsCore-v1.1-descriptions.csv diff --git a/yml/ivoa/ObsCore-v1.1-mandatory.csv b/python/lsst/sdm_schemas/schemas/ivoa/ObsCore-v1.1-mandatory.csv similarity index 100% rename from yml/ivoa/ObsCore-v1.1-mandatory.csv rename to python/lsst/sdm_schemas/schemas/ivoa/ObsCore-v1.1-mandatory.csv diff --git a/yml/ivoa/ObsCore-v1.1-optional.csv b/python/lsst/sdm_schemas/schemas/ivoa/ObsCore-v1.1-optional.csv similarity index 100% rename from yml/ivoa/ObsCore-v1.1-optional.csv rename to python/lsst/sdm_schemas/schemas/ivoa/ObsCore-v1.1-optional.csv diff --git a/yml/ivoa/README.md b/python/lsst/sdm_schemas/schemas/ivoa/README.md similarity index 100% rename from yml/ivoa/README.md rename to python/lsst/sdm_schemas/schemas/ivoa/README.md diff --git a/yml/ivoa/make_obscore_nominal.py b/python/lsst/sdm_schemas/schemas/ivoa/make_obscore_nominal.py similarity index 100% rename from yml/ivoa/make_obscore_nominal.py rename to python/lsst/sdm_schemas/schemas/ivoa/make_obscore_nominal.py diff --git a/yml/ivoa/obscore_nominal.yaml b/python/lsst/sdm_schemas/schemas/ivoa/obscore_nominal.yaml similarity index 100% rename from yml/ivoa/obscore_nominal.yaml rename to python/lsst/sdm_schemas/schemas/ivoa/obscore_nominal.yaml diff --git a/yml/obsloctap.yaml b/python/lsst/sdm_schemas/schemas/obsloctap.yaml similarity index 100% rename from yml/obsloctap.yaml rename to python/lsst/sdm_schemas/schemas/obsloctap.yaml diff --git a/yml/oga_live_obscore.yaml b/python/lsst/sdm_schemas/schemas/oga_live_obscore.yaml similarity index 100% rename from yml/oga_live_obscore.yaml rename to python/lsst/sdm_schemas/schemas/oga_live_obscore.yaml From 6a369080719f0e7b33e799b1cd4966e70d74c9df Mon Sep 17 00:00:00 2001 From: Jeremy McCormick Date: Fri, 25 Oct 2024 17:15:58 -0500 Subject: [PATCH 3/7] Add link from yml directory to schemas in the Python source tree --- yml | 1 + 1 file changed, 1 insertion(+) create mode 120000 yml diff --git a/yml b/yml new file mode 120000 index 00000000..ea8ae4d6 --- /dev/null +++ b/yml @@ -0,0 +1 @@ +./python/lsst/sdm_schemas/schemas/ \ No newline at end of file From dbe8ba943e5204ad62a1d361aad63375f720788f Mon Sep 17 00:00:00 2001 From: Jeremy McCormick Date: Mon, 28 Oct 2024 11:48:50 -0500 Subject: [PATCH 4/7] Update the repository's top-level README after moving schema files Add updated links to schemas in the repository. Add some missing external links. Improve the grammar and wording of the text. --- README.md | 97 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 54 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 2ee11aeb..fa3913ec 100644 --- a/README.md +++ b/README.md @@ -1,59 +1,70 @@ Science Data Model Schemas ========================== -This repository stores schema definitions for user-facing data products (the -Science Data Model, or SDM). These schema definitions are YAML files that are -designed to be read with felis (https://github.com/lsst-dm/felis), which can -create derived schema formats from the YAML definitions (e.g., SQL CREATE TABLE -statements, or IVOA TAP_SCHEMA table contents.) These YAML schema definitions -serve as the "source of truth" for other downstream usage, including -verification of the catalog outputs from Rubin Observatory science pipelines. - -The schemas in this repository serve several different purposes: - - * `yml/imsim.yaml` describes the outputs of the pipelines for LSSTCam-imSim. - It is used to generate the data preview schemas, which are fixed at the time - of their release, while this one will continue to evolve. This schema is - used by `ci_imsim`, which verifies that the schema of the pipeline output - files agree with the contents of this file. - - * The various `dp0X` schemas represent what is actively being served by the - various data previews. These are created from `yml/imsim.yaml` at a - specific point in time. - - * `yml/apdb.yaml` describes the schema used in the Alert Production Database (APDB). - This schema reflects the current schema that is used for Alert Production with `ap_pipe`. - It is also used in nightly `ap_verify` runs for continuous integration. - Previous processing runs may differ from the current schema. - The user-queryable Prompt Products Database (PPDB) is expected to have a very similar schema to the APDB. - - * `yml/hsc.yaml` describes the outputs of the latest data release production - pipelines for HyperSuprimeCam. This schema is used by `ci_hsc`, which - verifies that the schema of the pipeline output files agree with the - contents of this file. - -Release assets +This repository stores schema definitions for user-facing data +products comprising the Science Data Model (SDM) of the +[Rubin Observatory](https://rubinobservatory.org/). +These schemas are defined in YAML files designed to be read with +[Felis](https://github.com/lsst-dm/felis), a tool which can convert this data +to derived formats including SQL DDL and +[TAP_SCHEMA](https://www.ivoa.net/documents/TAP/20180830/PR-TAP-1.1-20180830.html#tth_sEc4). +These schema definitions serve as the "source of truth" for the observatory's core data models. + +Schemas +------- + +The following schemas are maintained in this repository under the [schemas](./python/lsst/sdm_schemas/schemas) directory: + + * [imsim](./python/lsst/sdm_schemas/schemas/imsim.yaml) describes the outputs + of the pipelines for LSSTCam-imSim, used to generate the data preview + schemas, which are fixed at the time of their release, while `imsim` will + continue to evolve. This schema is used by + [ci_imsim](https://github.com/lsst/ci_imsim) to verify that the tabular data + of the pipeline output files is conformant with the schema definition. + + * The various Data Preview (DP) schemas such as + [dp02_dc2](./python/lsst/sdm_schemas/schemas/dp02_dc2.yaml) represent + content that is being actively served by the various data previews. + These are created from `imsim` at a specific point in time. + + * [apdb](./python/lsst/sdm_schemas/schemas/apdb.yaml) describes the schema + of the Alert Production Database (APDB) used for Alert Production with + `ap_pipe` and for nightly `ap_verify` runs within continuous integration. + Previous processing runs may differ from the current schema. The + user-queryable Prompt Products Database (PPDB) is expected to have a very similar schema to the APDB. + + * [hsc](./python/lsst/sdm_schemas/schemas/hsc.yaml) describes the outputs of + the latest data release production pipelines for HyperSuprimeCam. This + schema is used by [ci_hsc](https://github.com/lsst/ci_hsc) for verification + of its output files. + +* The various `cdb` schemas such as + [cdb_latiss](./python/lsst/sdm_schemas/schemas/cdb_latiss.yaml) describe the + data model of the [Consolidated Database](https://github.com/lsst-dm/consdb) + or ConsDB, an image metadata database containing summarizations of + Engineering Facilities Database (EFD) telemetry by exposure and visit time windows. + +Release Assets -------------- -Each release of `sdm_schemas` includes the following additional release -assets, generated automatically via GitHub Actions when a new tag is created. +Each release of `sdm_schemas` includes the following additional assets, +generated automatically via GitHub Actions when a new tag is created: * `datalink-columns.zip` contains a set of YAML files with a restricted - subset of the Felis schema. Currently, they identify the principal and + subset of the Felis schema. Currently, these identify the principal and minimal columns for a subset of the tables defined by the schema in this repository. Principal columns are those for which the `principal` - flag is set in the TAP schema, and have the meaning defined in the + flag is set in the TAP schema, defined in the [IVOA TAP specification](https://www.ivoa.net/documents/TAP/20190927/REC-TAP-1.1.html#tth_sEc4.3). - Minimal columns are still experimental and in flux. These files are + The minimal columns are still experimental and in flux. These files are intended for use with the [datalinker](https://github.com/lsst-sqre/datalinker) service of a Rubin Science Platform deployment. * `datalink-snippets.zip` contains a JSON manifest and a set of XML files - that define VOTables following the IVOA DataLink specification. This - release asset is intended for use with the TAP service of a Rubin - Science Platform deployment and is used to add DataLink records to the - responses from a TAP query. Those DataLink records, in turn, provide - links to operations that a client may wish to perform on those results, - such as closely-related TAP queries. + that define VOTables following the IVOA DataLink specification and are + intended to be used by the TAP service of a Rubin Science Platform + deployment for adding DataLink records to the responses from a TAP query. + Those DataLink records, in turn, provide links to operations that a client + may wish to perform on those results, such as closely-related TAP queries. From d8bcff8e28a00611f7e5859ebec7379180c97121 Mon Sep 17 00:00:00 2001 From: Jeremy McCormick Date: Mon, 28 Oct 2024 12:26:40 -0500 Subject: [PATCH 5/7] Remove unneeded README from schema dir The top-level README should have all the required information. --- python/lsst/sdm_schemas/schemas/README.md | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 python/lsst/sdm_schemas/schemas/README.md diff --git a/python/lsst/sdm_schemas/schemas/README.md b/python/lsst/sdm_schemas/schemas/README.md deleted file mode 100644 index 33f73f4f..00000000 --- a/python/lsst/sdm_schemas/schemas/README.md +++ /dev/null @@ -1,13 +0,0 @@ -Rubin Data Model Definitions -============================ - -The files in this directory contain definitions, -[in the "Felis" format](https://felis.lsst.io/), -of the data models for the Rubin Observatory's public and principal internal -catalog and observation metadata tabular data products, -including IVOA-related metadata. - -See [the main README for this repository](../README.md) for more information. - -Note that only the ".yaml" extension for YAML files is supported by the -build mechanisms for this repository. From a2f1b977a3f160e8123d006c580a58667bce3edb Mon Sep 17 00:00:00 2001 From: Jeremy McCormick Date: Mon, 28 Oct 2024 14:20:46 -0500 Subject: [PATCH 6/7] Remove custom install script --- ups/eupspkg.cfg.sh | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 ups/eupspkg.cfg.sh diff --git a/ups/eupspkg.cfg.sh b/ups/eupspkg.cfg.sh deleted file mode 100644 index 20d57ce9..00000000 --- a/ups/eupspkg.cfg.sh +++ /dev/null @@ -1,7 +0,0 @@ -install() -{ - default_install - mkdir -p "$PREFIX" - cp -a ./yml "$PREFIX" - install_ups -} From f25aabb51ea96087e5e769d5affc5c4992e1764a Mon Sep 17 00:00:00 2001 From: Jeremy McCormick Date: Mon, 28 Oct 2024 16:20:03 -0500 Subject: [PATCH 7/7] Use psycopg2-binary dependency instead of psycopg2 --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index de897b88..dbf01a4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "lsst-felis @ git+http://github.com/lsst/felis.git@main", "click", "pyyaml", - "psycopg2", + "psycopg2-binary", "pymysql", "cryptography" ] diff --git a/requirements.txt b/requirements.txt index 60056fda..f848768a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ lsst-felis @ git+http://github.com/lsst/felis.git@main click pyyaml -psycopg2 +psycopg2-binary pymysql cryptography