tests: Add a way to pause the e2e docker container for debugging. (#1412

)

Makefile

@@ -75,9 +75,15 @@ integration: cmd/algorand-indexer/algorand-indexer
 	curl -s https://algorand-testdata.s3.amazonaws.com/indexer/test_blockdata/create_destroy.tar.bz2 -o test/blockdata/create_destroy.tar.bz2
 	test/postgres_integration_test.sh
 
+# note: when running e2e tests manually be sure to set the e2e filename:
+# 	'export CI_E2E_FILENAME=rel-nightly'
+# To keep the container running at exit set 'export EXTRA="--keep-alive"',
+# once the container is paused use 'docker exec <id> bash' to inspect temp
+# files in `/tmp/*/'
 e2e: cmd/algorand-indexer/algorand-indexer conduit-docs
 	cd e2e_tests/docker/indexer/ && docker-compose build --build-arg GO_IMAGE=${GO_IMAGE} && docker-compose up --exit-code-from e2e
 
+# note: when running e2e tests manually be sure to set the e2e filename: 'export CI_E2E_FILENAME=rel-nightly'
 e2e-conduit: conduit
 	cd third_party/go-algorand && make install
 	export PATH=$(PATH):$(shell go env GOPATH)/bin; pip3 install e2e_tests/ && e2econduit --s3-source-net ${CI_E2E_FILENAME} --conduit-bin cmd/conduit/conduit

e2e_tests/docker/indexer/Dockerfile

@@ -25,4 +25,4 @@ RUN pip3 install ./
 ENV INDEXER_DATA="${HOME}/indexer/"
 WORKDIR /opt/go/indexer
 # Run test script
-ENTRYPOINT ["/bin/bash", "-c", "sleep 5 && e2elive --connection-string \"$CONNECTION_STRING\" --s3-source-net \"$CI_E2E_FILENAME\" --indexer-bin /opt/go/indexer/cmd/algorand-indexer/algorand-indexer --indexer-port 9890"]
+ENTRYPOINT ["/bin/bash", "-c", "sleep 5 && e2elive $EXTRA --connection-string \"$CONNECTION_STRING\" --s3-source-net \"$CI_E2E_FILENAME\" --indexer-bin /opt/go/indexer/cmd/algorand-indexer/algorand-indexer --indexer-port 9890"]

e2e_tests/docker/indexer/docker-compose.yml

@@ -8,6 +8,8 @@ services:
     environment:
       CONNECTION_STRING: "host=e2e-db port=5432 user=algorand password=algorand dbname=indexer_db sslmode=disable"
       CI_E2E_FILENAME: "${CI_E2E_FILENAME}"
+      # Add --keep-alive to this variable to pause the container on an error
+      EXTRA: "${EXTRA}"
 
   e2e-db:
     image: "postgres:13-alpine"

e2e_tests/src/e2e_indexer/e2elive.py

@@ -36,6 +36,8 @@ def main():
 
     ap = argparse.ArgumentParser()
     ap.add_argument("--keep-temps", default=False, action="store_true")
+    # keep alive is convenient in docker environments to prevent exiting after an error
+    ap.add_argument("--keep-alive", default=False, action="store_true")
     ap.add_argument(
         "--indexer-bin",
         default=None,
@@ -78,6 +80,9 @@ def main():
         atexit.register(shutil.rmtree, tempdir, onerror=logger.error)
     else:
         logger.info("leaving temp dir %r", tempdir)
+    if args.keep_alive:
+        # register after keep_temps so that the temp files are still there.
+        atexitrun(["sleep", "infinity"])
     if not (source_is_tar or (sourcenet and os.path.isdir(sourcenet))):
         tarname = args.s3_source_net
         if not tarname:
@@ -123,14 +128,18 @@ def main():
         xrun(["goal", "network", "start", "-r", tempnet])
     except Exception:
         logger.error("failed to start private network, looking for node.log")
+        found = False
         for root, dirs, files in os.walk(tempnet):
             for f in files:
                 if f == "node.log":
+                    found = True
                     p = os.path.join(root, f)
                     logger.error("found node.log: {}".format(p))
                     with open(p) as nf:
                         for line in nf:
                             logger.error("   {}".format(line))
+        if found is False:
+            logger.error("unable to find node.log")
         raise
 
     atexitrun(["goal", "network", "stop", "-r", tempnet])