refactor: persistence layer clean-up

DEV_ENV.md

@@ -51,12 +51,14 @@ The dev environment is initialized with AWS Secrets/S3 data in the [scripts/setu
 
 ### Make targets for running tests in dev
 
-| Command                      | Description                                                                                  | Notes |
-|------------------------------|----------------------------------------------------------------------------------------------| ----- |
-| `make local-unit-test`       | Run backend tests in the local dockerized environment, against mock of persistence layer     |       |
-| `make local-integration-test`      | Run backend tests in the local dockerized environment, against dockerized database instance. |       |
-| `make local-functional-test` | Run functional tests in local dockerized environment                                         |       |
-| `make local-smoke-test`      | Run frontend/e2e tests in the local dockerized environment                                   |       |
+| Command                             | Description                                                                               | Notes |
+|-------------------------------------|-------------------------------------------------------------------------------------------| ----- |
+| `make local-unit-test`              | Run backend tests in the local dockerized environment, against mock of persistence layer  |       |
+| `make local-integration-test-backend` | Run backend tests in the local dockerized environment, against dockerized database instance. |       |
+| `make local-unit-test-processing`   | Run dataset processing upload job unit tests in the 'processing' docker environment.      |       |
+| `make local-unit-test-wmg-backend`  | Run wmg backend unit tests in the local dockerized environment.                           |       |
+| `make local-functional-test`        | Run backend functional tests in local dockerized environment                    |       |
+| `make local-smoke-test`             | Run frontend/e2e tests in the local dockerized environment                                |       |
 
 ### External dependencies
 

backend/database/README.md

@@ -16,7 +16,7 @@
 5. In the generated file, update the `Revision ID` and the `revision` (used by Alembic) to include the migration count.
    For example `Revision ID: a8cd0dc08805` becomes `Revision ID: 18_a8cd0dc08805` and `revision = "a8cd0dc08805"` becomes `revision = "18_a8cd0dc08805"`
 6. [Test your migration](#test-a-migration)
-7. Check that [corpora_orm.py](../common/corpora_orm.py) matches up with your changes.
+7. Check that [orm.py](../layers/persistence/orm.py) matches up with your changes.
 8. Once you've completed the changes, create a PR to get the functions reviewed.
 9. Once the PR is merged, migrations will be run as part of the deployment process to each env.
 10. [Connect to Remote RDS](#connect-to-remote-rds) to single-cell-dev
@@ -29,7 +29,7 @@ DEPLOYMENT_STAGE=test make db/migrate
 
 ## How to autogenerate migration script
 
-1. Make changes to the ORM class(es) in [corpora_orm.py](../common/corpora_orm.py)
+1. Make changes to the ORM class(es) in [orm.py](../layers/persistence/orm.py)
 2. [Connect to Remote RDS](#connect-to-remote-rds). Note, generally, you would be connecting to prod
    (`AWS_PROFILE=single-cell-prod DEPLOYMENT_STAGE=prod`) since we want to generate
    a migration from the database schema currently deployed in prod. However, if there are migrations haven't been

backend/database/versions/34_2be441104b48_standardize_db_fields.py

@@ -0,0 +1,60 @@
+"""standardize db fields
+
+Revision ID: 34_2be441104b48
+Revises: 33_c5aaf6e2ca9e
+Create Date: 2023-01-09 16:40:13.184800
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '34_2be441104b48'
+down_revision = '33_c5aaf6e2ca9e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('Collection', sa.Column('tombstone', sa.BOOLEAN(), nullable=True), schema='persistence_schema')
+    op.drop_column('Collection', 'tombstoned', schema='persistence_schema')
+    op.add_column('CollectionVersion', sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), schema='persistence_schema')
+    op.drop_column('CollectionVersion', 'version_id', schema='persistence_schema')
+    op.add_column('CollectionVersion', sa.Column('collection_metadata', postgresql.JSON(astext_type=sa.Text()), nullable=True), schema='persistence_schema')
+    op.drop_column('CollectionVersion', 'metadata', schema='persistence_schema')
+    op.add_column('Dataset', sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), schema='persistence_schema')
+    op.drop_column('Dataset', 'dataset_id', schema='persistence_schema')
+    op.add_column('Dataset', sa.Column('version_id', postgresql.UUID(as_uuid=True), nullable=True), schema='persistence_schema')
+    op.drop_column('Dataset', 'dataset_version_id', schema='persistence_schema')
+    op.add_column('DatasetVersion', sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), schema='persistence_schema')
+    op.drop_column('DatasetVersion', 'version_id', schema='persistence_schema')
+    op.add_column('DatasetVersion', sa.Column('dataset_metadata', postgresql.JSON(astext_type=sa.Text()), nullable=True), schema='persistence_schema')
+    op.drop_column('DatasetVersion', 'metadata', schema='persistence_schema')
+    op.drop_constraint('DatasetVersion_dataset_id_fkey', 'DatasetVersion', schema='persistence_schema',
+                       type_='foreignkey')
+    op.create_foreign_key(None, 'DatasetVersion', 'Dataset', ['dataset_id'], ['id'], source_schema='persistence_schema',
+                          referent_schema='persistence_schema')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('DatasetVersion', sa.Column('version_id', postgresql.UUID(), autoincrement=False, nullable=False), schema='persistence_schema')
+    op.add_column('DatasetVersion', sa.Column('metadata', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True), schema='persistence_schema')
+    op.drop_constraint(None, 'DatasetVersion', schema='persistence_schema', type_='foreignkey')
+    op.create_foreign_key('DatasetVersion_dataset_id_fkey', 'DatasetVersion', 'Dataset', ['dataset_id'], ['dataset_id'], source_schema='persistence_schema', referent_schema='persistence_schema')
+    op.drop_column('DatasetVersion', 'dataset_metadata', schema='persistence_schema')
+    op.drop_column('DatasetVersion', 'id', schema='persistence_schema')
+    op.add_column('Dataset', sa.Column('dataset_id', postgresql.UUID(), autoincrement=False, nullable=False), schema='persistence_schema')
+    op.add_column('Dataset', sa.Column('dataset_version_id', postgresql.UUID(), autoincrement=False, nullable=True), schema='persistence_schema')
+    op.drop_column('Dataset', 'version_id', schema='persistence_schema')
+    op.drop_column('Dataset', 'id', schema='persistence_schema')
+    op.add_column('CollectionVersion', sa.Column('version_id', postgresql.UUID(), autoincrement=False, nullable=False), schema='persistence_schema')
+    op.add_column('CollectionVersion', sa.Column('metadata', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True), schema='persistence_schema')
+    op.drop_column('CollectionVersion', 'collection_metadata', schema='persistence_schema')
+    op.drop_column('CollectionVersion', 'id', schema='persistence_schema')
+    op.add_column('Collection', sa.Column('tombstoned', sa.BOOLEAN(), autoincrement=False, nullable=True), schema='persistence_schema')
+    op.drop_column('Collection', 'tombstone', schema='persistence_schema')
+    # ### end Alembic commands ###