Dockerized worked examples

.circleci/config.yml

@@ -129,25 +129,17 @@ executors:
           REDIS_PASSWORD: "fm_redis"
 
 jobs:
-  set_docker_tag_for_ci_images:
-    docker: *base_docker
-    steps:
-      - run:
-          name: Set a common docker tag
-          command: |
-            echo "export SAFE_TAG=${CIRCLE_TAG:-$(echo -n $CIRCLE_BRANCH | md5sum | cut -d" " -f1)}"| tee -a $BASH_ENV /home/circleci/.env
-      - persist_to_workspace:
-          root: /home/circleci/
-          paths:
-            - .env
-
   build_flowdb:
     machine:
       image: circleci/classic:201808-01
     working_directory: /home/circleci/project
     environment: *flowdb_env
     steps:
       - checkout
+      - run:
+          name: Set a common docker tag
+          command: |
+            echo "export SAFE_TAG=${CIRCLE_TAG:-$(echo -n $CIRCLE_BRANCH | md5sum | cut -d" " -f1)}"| tee -a $BASH_ENV /home/circleci/.env
       - attach_workspace:
           at: /home/circleci/
       - restore_cache:
@@ -462,7 +454,9 @@ jobs:
     parameters:
       component:
         type: enum
-        enum: ["flowmachine", "flowauth", "flowapi"]
+        enum: ["flowmachine", "flowauth", "flowapi", "flowkit-examples"]
+      component_path:
+        type: string
     docker:
       - image: circleci/python:3.7
     working_directory: /home/circleci/project/
@@ -479,7 +473,7 @@ jobs:
           name: Build <<parameters.component>> image
           command: |
             echo "Tagging as $CIRCLE_SHA1"
-            docker build -t flowminder/<<parameters.component>>:$CIRCLE_SHA1 ./<<parameters.component>>
+            docker build -t flowminder/<<parameters.component>>:$CIRCLE_SHA1 <<parameters.component_path>>
       - run:
           name: Push images to Docker cloud
           command: |
@@ -518,6 +512,18 @@ jobs:
           path: /home/circleci/project/
       - run: ./quick_start.sh
       - run: ./quick_start.sh stop
+      - run: ./quick_start.sh examples smaller_data
+      - run: ./quick_start.sh stop
+      #- run: ./quick_start.sh larger_data # Synth data spinup with this data volume is very slow so not testing it right now on CI
+      #- run: ./quick_start.sh stop
+      - run:
+          name: Check all containers are down
+          command: |
+            RUNNING=`docker ps -q`
+            if [[ "$RUNNING" != "" ]]
+            then
+              exit 1
+            fi
 
   build_docs:
     parameters:
@@ -631,7 +637,7 @@ jobs:
           command: |
             export DOCKER_USER=$DOCKER_CLOUD_USER
             export DOCKER_PASS=$DOCKER_CLOUD_PASSWORD
-            for IMAGE in flowapi flowmachine flowdb flowdb-synthetic-data flowdb-testdata flowauth; do
+            for IMAGE in flowapi flowmachine flowdb flowdb-synthetic-data flowdb-testdata flowauth flowkit-examples; do
               docker-retag flowminder/$IMAGE:$CIRCLE_SHA1 ${<< parameters.tag >>:-latest}
             done
 
@@ -642,11 +648,7 @@ jobs:
 workflows:
   run_build_pipeline:
     jobs:
-      - set_docker_tag_for_ci_images:
-          <<: *run_always_org_context
       - build_flowdb:
-          requires:
-            - set_docker_tag_for_ci_images
           <<: *run_always_org_context
       - install_flowmachine_deps:
           <<: *run_always_org_context
@@ -679,11 +681,15 @@ workflows:
           requires:
             - build_flowdb
           <<: *run_always_org_context
+      - build_docker_image:
+          name: build_examples
+          component: flowkit-examples
+          component_path: .
+          <<: *run_always_org_context
       - build_docker_image:
           name: build_flowmachine
           component: flowmachine
-          requires:
-            - run_flowmachine_tests
+          component_path: flowmachine
           <<: *run_always_org_context
       - build_docs:
           name: build_docs
@@ -694,36 +700,32 @@ workflows:
       - build_docker_image:
           name: build_flowauth
           component: flowauth
-          requires:
-            - run_flowauth_backend_tests
-            - run_flowauth_frontend_tests
+          component_path: flowauth
           <<: *run_always_org_context
       - build_docker_image:
           name: build_flowapi
           component: flowapi
-          requires:
-            - run_flowkit_api_tests
+          component_path: flowapi
           <<: *run_always_org_context
       - build_python_wheel:
           name: build_flowclient_wheel
           project_name: flowclient
           requires:
-            - run_flowclient_tests
-            - integration_tests
+            - install_flowmachine_deps
           <<: *run_always_org_context
       - build_python_wheel:
           name: build_flowmachine_wheel
           project_name: flowmachine
           requires:
-            - run_flowmachine_tests
-            - integration_tests
+            - install_flowmachine_deps
           <<: *run_always_org_context
       - test_quickstart:
           requires:
             - build_flowdb
             - build_flowapi
             - build_flowmachine
             - build_flowauth
+            - build_examples
           <<: *run_always_org_context
       - integration_tests:
           requires:
@@ -734,32 +736,40 @@ workflows:
           name: retag_master_branch
           requires:
             - test_quickstart
+            - integration_tests
             - build_docs
+            - run_flowkit_api_tests
+            - run_flowauth_backend_tests
+            - run_flowauth_frontend_tests
+            - run_flowclient_tests
           <<: *master_only_org_context
       - retag_images:
           name: retag_tagged_build
           requires:
             - test_quickstart
+            - integration_tests
             - build_docs
+            - run_flowkit_api_tests
+            - run_flowauth_backend_tests
+            - run_flowauth_frontend_tests
+            - run_flowclient_tests
           tag: CIRCLE_TAG
           <<: *tag_only_org_context
       - push_wheel:
           name: push_flowclient_wheel
           project_name: flowclient
           requires:
-            - build_flowclient_wheel
+            - retag_tagged_build
           <<: *tag_only_org_context
       - push_wheel:
           name: push_flowmachine_wheel
           project_name: flowmachine
           requires:
-            - build_flowmachine_wheel
+            - retag_tagged_build
           <<: *tag_only_org_context
       - build_docs:
           name: Deploy docs
           deploy: true
           requires:
-            - build_docs
             - retag_tagged_build
-            - push_flowclient_wheel
           <<: *tag_only_org_context

.dockerignore

@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# This .dockerignore file follows the "whitelisting" approach described here:
+#    https://youknowfordevs.com/2018/12/07/getting-control-of-your-dockerignore-files.html
+
+#
+# First, exclude everything by default:
+#
+
+*
+
+#
+# Now un-exclude only those folders and files that are needed for building the image:
+#
+!docs/source/worked_examples/*.ipynb
+
+!flowmachine/flowmachine
+!flowmachine/Pipfile*
+!flowmachine/setup.cfg
+!flowmachine/setup.py
+!flowmachine/README.md
+!flowmachine/versioneer.py
+!flowmachine/MANIFEST.in
+
+!flowclient/flowclient
+!flowclient/Pipfile*
+!flowclient/setup.cfg
+!flowclient/setup.py
+!flowclient/README.md
+!flowclient/versioneer.py
+!flowclient/MANIFEST.in

CHANGELOG.md

@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Error is displayed in the add group form of Flowauth if group name already exists. [#709](https://github.com/Flowminder/FlowKit/issues/709)
 
 ### Added
+- FlowKit's worked examples are now Dockerized, and available as part of the quick setup script [#614](https://github.com/Flowminder/FlowKit/issues/614)
 
 ### Changed
 

Dockerfile

@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Worked examples docker container
+# Provides all worked examples from the FlowKit docs in a ready-to-go JupyterLab install
+#
+
+FROM jupyter/scipy-notebook
+
+RUN rm -rf /home/$NB_USER/work
+
+COPY docs/source/worked_examples/*.ipynb /home/$NB_USER/
+COPY flowmachine /flowmachine
+COPY flowclient /flowclient
+RUN pip install folium /flowclient /flowmachine && \
+    fix-permissions $CONDA_DIR && \
+    fix-permissions /home/$NB_USER && \
+    cd /home/$NB_USER/ && jupyter trust -y *

Makefile

@@ -21,7 +21,7 @@
 DOCKER_COMPOSE_FILE ?= docker-compose.yml
 DOCKER_COMPOSE_FILE_BUILD ?= docker-compose-build.yml
 FLOWDB_SERVICES ?= flowdb_testdata
-DOCKER_SERVICES ?= $(FLOWDB_SERVICES) flowapi flowmachine flowauth flowmachine_query_locker
+DOCKER_SERVICES ?= $(FLOWDB_SERVICES) flowapi flowmachine flowauth flowmachine_query_locker worked_examples
 export DOCKER_FLOWDB_HOST=$(word 1, $(FLOWDB_SERVICES))
 
 
@@ -108,6 +108,16 @@ flowauth-build:
 	docker-compose -f $(DOCKER_COMPOSE_FILE) -f $(DOCKER_COMPOSE_FILE_BUILD) build flowauth
 
 
+worked_examples-up: worked_examples-build
+	docker-compose -f $(DOCKER_COMPOSE_FILE) -f $(DOCKER_COMPOSE_FILE_BUILD) up -d --build worked_examples
+
+worked_examples-down:
+	docker-compose -f $(DOCKER_COMPOSE_FILE) rm -f -s -v worked_examples
+
+worked_examples-build:
+	docker-compose -f $(DOCKER_COMPOSE_FILE) -f $(DOCKER_COMPOSE_FILE_BUILD) build worked_examples
+
+
 flowmachine_query_locker-up:
 	docker-compose -f $(DOCKER_COMPOSE_FILE) up -d flowmachine_query_locker
 

development_environment

@@ -116,3 +116,7 @@ OUTPUT_ROOT_DIR=/docker-entrypoint-initdb.d
 # Integration tests
 
 FLOWKIT_INTEGRATION_TESTS_DISABLE_AUTOSTART_SERVERS=FALSE
+
+# Worked examples
+
+WORKED_EXAMPLES_PORT=8888

docker-compose-build.yml

@@ -5,7 +5,7 @@
 # DOCKER COMPOSE BUILDER FOR FLOWKIT
 #
 
-version: '3.7'
+version: '3.5'
 
 services:
 
@@ -33,6 +33,13 @@ services:
       context: ./flowmachine
       dockerfile: Dockerfile
 
+  worked_examples:
+    container_name: worked_examples
+    image: flowminder/flowkit-examples:${CONTAINER_TAG:-latest}
+    build:
+      context: .
+      dockerfile: Dockerfile
+
   flowapi:
     image: flowminder/flowapi:${CONTAINER_TAG:-latest}
     build:

docker-compose.yml

@@ -5,7 +5,7 @@
 # DOCKER COMPOSE FOR FLOWKIT
 #
 
-version: '3.7'
+version: '3.5'
 
 networks:
   db:
@@ -113,6 +113,29 @@ services:
       - db
       - redis
 
+  worked_examples:
+    container_name: worked_examples
+    image: flowminder/flowkit-examples:${CONTAINER_TAG:-latest}
+    ports:
+      - ${WORKED_EXAMPLES_PORT:?Must set WORKED_EXAMPLES_PORT env var}:8888
+    tty: true
+    stdin_open: true
+    environment:
+      - FLOWDB_PORT=5432
+      - FLOWDB_HOST=${DOCKER_FLOWDB_HOST:?Must set DOCKER_FLOWDB_HOST env var}
+      - FLOWMACHINE_FLOWDB_USER=${FLOWMACHINE_FLOWDB_USER:?Must set FLOWMACHINE_FLOWDB_USER env var}
+      - FLOWMACHINE_FLOWDB_PASSWORD=${FLOWMACHINE_FLOWDB_PASSWORD:?Must set FLOWMACHINE_FLOWDB_PASSWORD env var}
+      - REDIS_HOST=query_locker
+      - REDIS_PORT=6379
+      - REDIS_PASSWORD=${REDIS_PASSWORD:?Must set REDIS_PASSWORD env var}
+      - FLOWAPI_URL=http://flowapi:9090
+    restart: always
+    networks:
+      - db
+      - redis
+    command:
+      ["start.sh", "jupyter", "lab", "--LabApp.token=''"]
+
   flowapi:
     container_name: flowapi
     image: flowminder/flowapi:${CONTAINER_TAG:-latest}