Resolves #3 adding gene expr correlation with str length in crc suppo…

…rt + docker-compose + alembic migrations (optional) (#4)

.devcontainer/devcontainer.json

@@ -0,0 +1,29 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+	"name": "Existing Dockerfile",
+	"build": {
+		// Sets the run context to one level up instead of the .devcontainer folder.
+		"context": "..",
+		// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+		"dockerfile": "../Dockerfile"
+	},
+	"features": {
+		"ghcr.io/devcontainers/features/python:1": {}
+	}
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Uncomment the next line to run commands after the container is created.
+	// "postCreateCommand": "cat /etc/os-release",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "devcontainer"
+}

.dockerignore

@@ -0,0 +1,28 @@
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+**/db/
+LICENSE
+README.md

.env.example

@@ -0,0 +1,7 @@
+# Rename this file to .env and docker compose wil automatically use envornment variables fro it
+DATABASE_URL=postgresql://webstr:webstr@db:5432/webstr
+# DATABASE_URL=sqlite:///db/debug.sqlite
+PYTHONPATH="./"  
+WEBSTR_DATABASE_DATA_UPGRADE=True
+WEBSTR_DEVELOPMENT=False
+WEBSTR_SOURCE_MOUNT_PATH=/temp/src #/usr/src/strs

.gitignore

@@ -4,4 +4,9 @@
 
 strAPI/local_queries.ipynb
 data/repeats
+db/docker_data/
 .venv/
+db/debug.db
+
+*/migrations/versions0/*
+*.env

.vscode/launch.json

@@ -0,0 +1,46 @@
+{
+    "configurations": [
+        {
+            "name": "Python: main",
+            "type": "python",
+            "request": "launch",
+            "program": "strAPI/main.py",
+            "cwd": "${workspaceFolder}",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "env": {
+               // "PYTHONPATH": "${workspaceFolder}/strAPI/"
+            }
+        },
+        {
+            "name": "Docker: Python - Fastapi",
+            "type": "docker",
+            "request": "launch",
+            "preLaunchTask": "docker-run: debug",
+            "python": {
+                "pathMappings": [
+                    {
+                        "localRoot": "${workspaceFolder}",
+                        "remoteRoot": "/usr/src/strs"
+                    }
+                ],
+                "projectType": "fastapi"
+            },
+        },
+        {
+            "name": "Python: Remote Attach 5000",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "0.0.0.0",
+                "port": 5000
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/usr/src/app"
+                }
+            ]
+        }
+    ]
+}

.vscode/tasks.json

@@ -0,0 +1,43 @@
+{
+	"version": "2.0.0",
+	"tasks": [
+		{
+			"type": "docker-build",
+			"label": "docker-build",
+			"platform": "python",
+			"dockerBuild": {
+				"tag": "webstrapi:latest",
+				"dockerfile": "${workspaceFolder}/Dockerfile",
+				"context": "${workspaceFolder}",
+				"pull": true,
+			}
+		},
+		{
+			"type": "docker-run",
+			"label": "docker-run: debug",
+			"dependsOn": [
+				"docker-build"
+			],
+			"dockerRun": {
+				"env": {
+				  "DATABASE_URL": "sqlite:///db/debug.db",
+				  "WEBSTR_DATABAS_UPGRADE": "True"
+				},
+				"mounts": [
+					"source=${workspaceFolder},target=/app,type=bind,consistency=cached"
+				],
+			},
+			"python": {
+				"args": [
+					"strAPI.main:app",
+					"--host",
+					"0.0.0.0",
+					"--port",
+					"5000",
+					"--reload"
+				],
+				"module": "uvicorn",
+			}
+		}
+	]
+}

Dockerfile

@@ -1,18 +1,33 @@
-FROM python:3.8-slim-buster
+FROM python:3.8-slim-buster as base
 LABEL maintainer="merenlin -- follow me on medium https://medium.com/@merenlin"
 
-RUN apt-get update && apt-get install -y python3-dev build-essential
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
 
 RUN mkdir -p /usr/src/strs
 WORKDIR /usr/src/strs
 
 COPY requirements.txt .
-RUN pip3 install -r requirements.txt
+
+RUN apt-get update && \
+    apt-get install -y gcc python3-dev build-essential  && \
+    python -m pip install -r requirements.txt --no-cache-dir  && \
+    apt-get remove -y gcc python3-dev build-essential && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV DATABASE_URL=sqlite:///db/debug.db
+
+FROM base as prod
+LABEL maintainer="merenlin -- follow me on medium https://medium.com/@merenlin"
+
+RUN adduser -u 1000 --disabled-password --gecos "" appuser && chown -R appuser /usr/src/strs
+USER appuser
 
 COPY . .
 
-EXPOSE 5000
+ARG PORT=5000
+EXPOSE $PORT
 
-CMD uvicorn strAPI.main:app --host=0.0.0.0 --port=${PORT:-5000}
+CMD bash entrypoint.sh
 
 

README.md

@@ -23,10 +23,17 @@ Yes! It is possible and we encourage it if you would like to add your own data t
 
 ### Instructions on how to set-up webSTR-API locally (for development): 
 
-#### Step 0: Set up the database
+#### Step 0: (docker and local ways) Set up the database 
  Install and configure PostgreSQL on your machine and create an empty database called strdb. We provide an sql_dump backup of the current version of the database on request. Restore the database from this backup. 
-
-#### Step 1: Install all the requirements
+
+ Alternatively you can use `docker compose` with provided docker-compose.yml file to set up the PostgreSQL database.
+ 1. rename `.env.example` to `.env`
+ 2. copy backup to folder ./db/docker_data/pgdata/webstr_backup.dump
+ 2. `docker compose up -d db`
+ 3. `docker exec -it webstr-api-db-1 pg_restore -d strdb /var/lib/postgresql/data/pgdata/webstr_backup.dump`
+
+#### Step 1: (only for non-docker way) Install all the requirements 
+
 
 a) Set up python3 and virtualenv on your machine:
 [For Mac, follow instructions here.](https://gist.github.com/pandafulmanda/730a9355e088a9970b18275cb9eadef3)
@@ -38,22 +45,40 @@ b) Create new virtual env and install all the requirements with the following co
 `pip install -r requirements.txt`
 
 
-#### Step 2: Set environmental variable DATABASE_URL on your machine (or your IDE) to 
-
+#### Step 2: (only for non-docker way) Set environmental variable DATABASE_URL on your machine (or your IDE) to 
 `export DATABASE_URL="postgres://postgres:YOURPASSWORD@localhost:5432/strdb"`
 
 Note that this is using the default user postgres, if you created your db on a different user, adjust this variable accordingly. 
 
 Optional: add this line to `~/.bashrc` and restart your terminal. 
 
-#### Step 3: Start API server
+#### Step 3: (only for non-docker way) Start API server  
 
 Run the following command from the root folder of this repo: 
 
 `uvicorn strAPI.main:app --host=0.0.0.0 --port=${PORT:-5000} --reload`
 
-#### Step 4: You can now access the api at `http://0.0.0.0:5000` 
+#### Step 4: You can now access the api at `http://localhost:5000` 
+
+***
 
+### How to build and run application using docker way
+
+#### Step 0: Set up the database 
+##### if you want to use PostgreSQL and backup
+See docker part of Step 1 from the previous instructions
+##### if sqlite is enough
+Change database url in .env file to DATABASE_URL=sqlite:///db/debug.sqlite
+Set WEBSTR_DATABASE_DATA_UPGRADE=True and WEBSTR_DEVELOPMENT=True in .env file
+#### Step 1: Start containers
+Run `docker compose up`
+
+Now you can access api on localhost:5000 and frontend on localhost:5001
+
+##### debugging in docker container
+If you want to debug the code and you use VSCode - you can run code in container using vscode and tasks defined in launch.json in .vscode folder. Or you can use devcontaienr plugin to do the same. This is a bit more advanced, so you need to study how debuggin in containers works in vscode a bit before that. And you will have to stop webster-api-api-1 container first if you have it running through docker compose.
+
+***
 
 ## How to build the database from scratch or import my own data to WebSTR? 
 
@@ -76,3 +101,14 @@ Explore "database_setup" directory for different utilities to import data into t
 * If you would like **to import a new reference panel** we recommend making a csv corresponding to the repeats table structure and importing it directly to SQL to save time. Alternatively see  ` insert_repeats.py ` and 
 ` import_data_ensembltrs.py `  utilities that we made for repeats data coming  in different formats. Feel free to contact us for more details if you would like to make your own reference STR panel. 
 
+***
+
+### Database migrations using Alembic - Proof of Concept, not used in production.
+@slmjy added alembic migrations as a proof of concept to the codebase.
+Following changes were introduced:
+1. Database migrations in /database_setup/migrations
+2. alembic.ini and env.py files 
+3. entrypoint.sh can run alembic migratiosn given environment variable WEBSTR_DATABASE_MIGRATE is set to True
+4. in database.py there is a disabled check if database is on the latest version
+
+If someone wants to start using alembic migrations, they can enable the version check and start generating new migrations using alembic and using them in production.