feat(zkstack_cli): use docker-managed volumes (#3140)

## What ❔

Host-bound volumes never get cleaned even if you do `docker compose down
-v`. Existing dev flows don't seem to rely on volumes being on the host
machine, so this is PoC of how we can move to Docker-managed volumes.

## Why ❔

Avoid pesky bugs that prevent user from deleting `./volumes`, rely on
Docker to persist and dispose of data as need be

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zkstack dev fmt` and `zkstack dev
lint`.

.github/workflows/build-contract-verifier-template.yml

@@ -101,7 +101,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run pre_download_compilers.sh

.github/workflows/build-core-template.yml

@@ -114,7 +114,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run pre_download_compilers.sh

.github/workflows/build-local-node-docker.yml

@@ -50,7 +50,6 @@ jobs:
 
       - name: start-services
         run: |
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
         

.github/workflows/build-prover-template.yml

@@ -75,7 +75,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run sccache --start-server

.github/workflows/build-witness-generator-template.yml

@@ -75,7 +75,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run sccache --start-server

.github/workflows/ci-common-reusable.yml

@@ -27,7 +27,6 @@ jobs:
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
-          mkdir -p ./volumes/postgres
           docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
       
       - name: Install zkstack

.github/workflows/ci-prover-e2e.yml

@@ -29,7 +29,6 @@ jobs:
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
-          mkdir -p ./volumes/postgres ./volumes/reth/data
           docker-compose -f ${RUNNER_COMPOSE_FILE} --profile runner up -d --wait
           ci_run sccache --start-server
 

.github/workflows/ci-prover-reusable.yml

@@ -27,7 +27,6 @@ jobs:
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
-          mkdir -p ./volumes/postgres
           docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
 
       - name: Install zkstack
@@ -68,7 +67,6 @@ jobs:
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
-          mkdir -p ./volumes/postgres
           docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
 
       - name: Install zkstack

bin/ci_localnet_up

@@ -4,6 +4,5 @@ set -e
 
 cd $ZKSYNC_HOME
 
-mkdir -p ./volumes/postgres ./volumes/reth/data
 run_retried docker-compose pull
 docker-compose --profile runner up -d --wait

docker-compose-gpu-runner-cuda-12-0.yml

@@ -6,8 +6,8 @@ services:
     ports:
       - 127.0.0.1:8545:8545
     volumes:
-      - type: bind
-        source: ./volumes/reth/data
+      - type: volume
+        source: reth-data
         target: /rethdata
       - type: bind
         source: ./etc/reth/chaindata
@@ -69,3 +69,7 @@ services:
     environment:
       # We bind only to 127.0.0.1, so setting insecure password is acceptable here
       - POSTGRES_PASSWORD=notsecurepassword
+
+volumes:
+  postgres-data:
+  reth-data:

docker-compose-runner-nightly.yml

@@ -1,4 +1,3 @@
-version: '3.2'
 services:
   zk:
     image: ghcr.io/matter-labs/zk-environment:latest2.0-lightweight-nightly
@@ -15,3 +14,7 @@ services:
     extends:
       file: docker-compose.yml
       service: reth
+
+volumes:
+  postgres-data:
+  reth-data:

docker-compose-unit-tests.yml

@@ -1,4 +1,3 @@
-version: '3.2'
 name: unit_tests
 services:
   # An instance of postgres configured to execute Rust unit-tests, tuned for performance.

docker-compose.yml

@@ -1,13 +1,12 @@
-version: '3.2'
 services:
   reth:
     restart: always
     image: "ghcr.io/paradigmxyz/reth:v1.0.6"
     ports:
       - 127.0.0.1:8545:8545
     volumes:
-      - type: bind
-        source: ./volumes/reth/data
+      - type: volume
+        source: reth-data
         target: /rethdata
       - type: bind
         source: ./etc/reth/chaindata
@@ -22,8 +21,8 @@ services:
     ports:
       - 127.0.0.1:5432:5432
     volumes:
-      - type: bind
-        source: ./volumes/postgres
+      - type: volume
+        source: postgres-data
         target: /var/lib/postgresql/data
     environment:
       # We bind only to 127.0.0.1, so setting insecure password is acceptable here
@@ -56,3 +55,7 @@ services:
     profiles:
       - runner
     network_mode: host
+
+volumes:
+  postgres-data:
+  reth-data:

zkstack_cli/crates/common/src/docker.rs

@@ -14,7 +14,11 @@ pub fn up(shell: &Shell, docker_compose_file: &str, detach: bool) -> anyhow::Res
 }
 
 pub fn down(shell: &Shell, docker_compose_file: &str) -> anyhow::Result<()> {
-    Ok(Cmd::new(cmd!(shell, "docker compose -f {docker_compose_file} down")).run()?)
+    Ok(Cmd::new(cmd!(
+        shell,
+        "docker compose -f {docker_compose_file} down -v"
+    ))
+    .run()?)
 }
 
 pub fn run(shell: &Shell, docker_image: &str, docker_args: Vec<String>) -> anyhow::Result<()> {

zkstack_cli/crates/zkstack/src/commands/containers.rs

@@ -36,10 +36,6 @@ pub fn run(shell: &Shell, args: ContainersArgs) -> anyhow::Result<()> {
 }
 
 pub fn initialize_docker(shell: &Shell, ecosystem: &EcosystemConfig) -> anyhow::Result<()> {
-    if !shell.path_exists("volumes") {
-        create_docker_folders(shell)?;
-    };
-
     if !shell.path_exists(DOCKER_COMPOSE_FILE) {
         copy_dockerfile(shell, ecosystem.link_to_code.clone())?;
     };
@@ -75,14 +71,6 @@ pub fn start_containers(shell: &Shell, observability: bool) -> anyhow::Result<()
     Ok(())
 }
 
-fn create_docker_folders(shell: &Shell) -> anyhow::Result<()> {
-    shell.create_dir("volumes")?;
-    shell.create_dir("volumes/postgres")?;
-    shell.create_dir("volumes/reth")?;
-    shell.create_dir("volumes/reth/data")?;
-    Ok(())
-}
-
 fn copy_dockerfile(shell: &Shell, link_to_code: PathBuf) -> anyhow::Result<()> {
     let docker_compose_file = link_to_code.join(DOCKER_COMPOSE_FILE);
 

zkstack_cli/crates/zkstack/src/commands/dev/commands/clean/mod.rs

@@ -5,8 +5,7 @@ use config::{EcosystemConfig, DOCKER_COMPOSE_FILE};
 use xshell::Shell;
 
 use crate::commands::dev::messages::{
-    MSG_CONTRACTS_CLEANING, MSG_CONTRACTS_CLEANING_FINISHED, MSG_DOCKER_COMPOSE_CLEANED,
-    MSG_DOCKER_COMPOSE_DOWN, MSG_DOCKER_COMPOSE_REMOVE_VOLUMES,
+    MSG_CONTRACTS_CLEANING, MSG_CONTRACTS_CLEANING_FINISHED, MSG_DOCKER_COMPOSE_DOWN,
 };
 
 #[derive(Subcommand, Debug)]
@@ -35,9 +34,6 @@ pub fn run(shell: &Shell, args: CleanCommands) -> anyhow::Result<()> {
 pub fn containers(shell: &Shell) -> anyhow::Result<()> {
     logger::info(MSG_DOCKER_COMPOSE_DOWN);
     docker::down(shell, DOCKER_COMPOSE_FILE)?;
-    logger::info(MSG_DOCKER_COMPOSE_REMOVE_VOLUMES);
-    shell.remove_path("volumes")?;
-    logger::info(MSG_DOCKER_COMPOSE_CLEANED);
     Ok(())
 }
 

zkstack_cli/crates/zkstack/src/commands/dev/messages.rs

@@ -157,9 +157,7 @@ pub(super) const MSG_UPGRADE_TEST_RUN_INFO: &str = "Running upgrade test";
 pub(super) const MSG_UPGRADE_TEST_RUN_SUCCESS: &str = "Upgrade test ran successfully";
 
 // Cleaning related messages
-pub(super) const MSG_DOCKER_COMPOSE_DOWN: &str = "docker compose down";
-pub(super) const MSG_DOCKER_COMPOSE_REMOVE_VOLUMES: &str = "docker compose remove volumes";
-pub(super) const MSG_DOCKER_COMPOSE_CLEANED: &str = "docker compose network cleaned";
+pub(super) const MSG_DOCKER_COMPOSE_DOWN: &str = "docker compose down -v";
 pub(super) const MSG_CONTRACTS_CLEANING: &str =
     "Removing contracts building and deployment artifacts";
 pub(super) const MSG_CONTRACTS_CLEANING_FINISHED: &str =