From d42f7e4a1c638ae9e22c2c276815c483e525b89a Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Fri, 17 Dec 2021 17:06:32 -0800 Subject: [PATCH 01/11] packer snap --- .circleci/config.yml | 22 +++++++++ .gitignore | 1 + scripts/snap-lotus-entrypoint.sh | 10 ++++ snap/snapcraft.yaml | 40 +++++++++++---- tools/packer/lotus-snap.pkr.hcl | 84 ++++++++++++++++++++++++++++++++ tools/packer/lotus.pkr.hcl | 10 ---- tools/packer/setup-snap.sh | 34 +++++++++++++ 7 files changed, 181 insertions(+), 20 deletions(-) create mode 100755 scripts/snap-lotus-entrypoint.sh create mode 100644 tools/packer/lotus-snap.pkr.hcl create mode 100644 tools/packer/setup-snap.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index a4a88a09089..92ce5bc6b8d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -705,6 +705,17 @@ jobs: - packer/build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG" + publish-packer-snap: + description: build packer image with snap. mainnet only. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." + - packer/build: + template: tools/packer/lotus-snap.pkr.hcl publish-dockerhub: description: publish to dockerhub machine: @@ -1042,6 +1053,7 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish-packer-snap nightly: triggers: @@ -1058,3 +1070,13 @@ workflows: - publish-dockerhub: name: publish-dockerhub-nightly tag: nightly + monthly: + triggers: + - schedule: + cron: "0 0 1 * *" + filters: + branches: + only: + - master + jobs: + - publish-packer-snap diff --git a/.gitignore b/.gitignore index 467f315b8ef..33fbffa3c19 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ build/paramfetch.sh /bundle /darwin /linux +*.snap *-fuzz.zip /chain/types/work_msg/ diff --git a/scripts/snap-lotus-entrypoint.sh b/scripts/snap-lotus-entrypoint.sh new file mode 100755 index 00000000000..a3ab04c5b34 --- /dev/null +++ b/scripts/snap-lotus-entrypoint.sh @@ -0,0 +1,10 @@ +LOTUS_IMPORT_SNAPSHOT="https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car" +LOTUS_BINARY=$(dirname "$0")/lotus +GATE="$LOTUS_PATH"/date_initialized +if [ ! -f "$GATE" ]; then + echo importing minimal snapshot + $LOTUS_BINARY daemon --import-snapshot "$LOTUS_IMPORT_SNAPSHOT" --halt-after-import + # Block future inits + date > "$GATE" +fi +$LOTUS_BINARY daemon $ARGS diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 472621c2a48..08854555ffc 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -17,7 +17,6 @@ description: | https://github.com/filecoin-project/lotus -grade: devel confinement: strict parts: @@ -39,6 +38,15 @@ parts: override-build: | LDFLAGS="" make lotus lotus-miner lotus-worker cp lotus lotus-miner lotus-worker $SNAPCRAFT_PART_INSTALL + cp scripts/snap-lotus-entrypoint.sh $SNAPCRAFT_PART_INSTALL + +layout: + /var/lib/lotus: + symlink: $SNAP_COMMON/lotus + /var/lib/lotus-miner: + symlink: $SNAP_COMMON/lotus-miner + /var/lib/lotus-worker: + symlink: $SNAP_COMMON/lotus-worker apps: lotus: @@ -49,9 +57,9 @@ apps: - home environment: FIL_PROOFS_PARAMETER_CACHE: $SNAP_USER_COMMON/filecoin-proof-parameters - LOTUS_PATH: $SNAP_USER_COMMON/lotus - LOTUS_MINER_PATH: $SNAP_USER_COMMON/lotus-miner - LOTUS_WORKER_PATH: $SNAP_USER_COMMON/lotus-worker + LOTUS_PATH: $SNAP_COMMON/lotus + LOTUS_MINER_PATH: $SNAP_COMMON/lotus-miner + LOTUS_WORKER_PATH: $SNAP_COMMON/lotus-worker lotus-miner: command: lotus-miner plugs: @@ -60,9 +68,9 @@ apps: - opengl environment: FIL_PROOFS_PARAMETER_CACHE: $SNAP_USER_COMMON/filecoin-proof-parameters - LOTUS_PATH: $SNAP_USER_COMMON/lotus - LOTUS_MINER_PATH: $SNAP_USER_COMMON/lotus-miner - LOTUS_WORKER_PATH: $SNAP_USER_COMMON/lotus-worker + LOTUS_PATH: $SNAP_COMMON/lotus + LOTUS_MINER_PATH: $SNAP_COMMON/lotus-miner + LOTUS_WORKER_PATH: $SNAP_COMMON/lotus-worker lotus-worker: command: lotus-worker plugs: @@ -71,6 +79,18 @@ apps: - opengl environment: FIL_PROOFS_PARAMETER_CACHE: $SNAP_USER_COMMON/filecoin-proof-parameters - LOTUS_PATH: $SNAP_USER_COMMON/lotus - LOTUS_MINER_PATH: $SNAP_USER_COMMON/lotus-miner - LOTUS_WORKER_PATH: $SNAP_USER_COMMON/lotus-worker + LOTUS_PATH: $SNAP_COMMON/lotus + LOTUS_MINER_PATH: $SNAP_COMMON/lotus-miner + LOTUS_WORKER_PATH: $SNAP_COMMON/lotus-worker + lotus-daemon: + command: snap-lotus-entrypoint.sh + daemon: simple + install-mode: enable + plugs: + - network + - network-bind + environment: + FIL_PROOFS_PARAMETER_CACHE: $SNAP_COMMON/filecoin-proof-parameters + LOTUS_PATH: $SNAP_COMMON/lotus + LOTUS_MINER_PATH: $SNAP_COMMON/lotus-miner + LOTUS_WORKER_PATH: $SNAP_COMMON/lotus-worker diff --git a/tools/packer/lotus-snap.pkr.hcl b/tools/packer/lotus-snap.pkr.hcl new file mode 100644 index 00000000000..ef0f52f8cec --- /dev/null +++ b/tools/packer/lotus-snap.pkr.hcl @@ -0,0 +1,84 @@ +variable "ci_workspace_bins" { + type = string + default = "./linux" +} + +variable "lotus_network" { + type = string + default = "mainnet" +} + +locals { + timestamp = regex_replace(timestamp(), "[- TZ:]", "") +} + +source "amazon-ebs" "lotus" { + ami_name = "lotus-${var.lotus_network}-snap-${local.timestamp}" + ami_regions = [ + "ap-east-1", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + ] + ami_groups = [ + # This causes the ami to be publicly-accessable. + "all", + ] + ami_description = "Lotus Filecoin AMI" + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 100 + delete_on_termination = true + } + + instance_type = "t2.micro" + source_ami_filter { + filters = { + name = "ubuntu/images/*ubuntu-focal-20.04-amd64-server-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["099720109477"] + } + ssh_username = "ubuntu" +} + +source "digitalocean" "lotus" { + droplet_name = "lotus-snap" + size = "s-1vcpu-1gb" + region = "nyc3" + image = "ubuntu-20-04-x64" + snapshot_name = "lotus-${var.lotus_network}-snap-${local.timestamp}" + ssh_username = "root" +} + +build { + sources = [ + "source.amazon-ebs.lotus", + "source.digitalocean.lotus", + ] + + provisioner "file" { + source = "./tools/packer/etc/motd" + destination = "motd" + } + # build it. + provisioner "shell" { + script = "./tools/packer/setup-snap.sh" + } +} diff --git a/tools/packer/lotus.pkr.hcl b/tools/packer/lotus.pkr.hcl index 8ef41613b14..cfaca83986e 100644 --- a/tools/packer/lotus.pkr.hcl +++ b/tools/packer/lotus.pkr.hcl @@ -63,19 +63,9 @@ source "amazon-ebs" "lotus" { ssh_username = "ubuntu" } -source "digitalocean" "lotus" { - droplet_name = "lotus-${var.lotus_network}" - size = "s-1vcpu-1gb" - region = "nyc3" - image = "ubuntu-20-04-x64" - snapshot_name = "lotus-${var.lotus_network}-${var.git_tag}-${local.timestamp}" - ssh_username = "root" -} - build { sources = [ "source.amazon-ebs.lotus", - "source.digitalocean.lotus", ] # Lotus software (from CI workspace) diff --git a/tools/packer/setup-snap.sh b/tools/packer/setup-snap.sh new file mode 100644 index 00000000000..7b543bc364d --- /dev/null +++ b/tools/packer/setup-snap.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# This script is executed by packer to setup the image. +# When this script is run, packer will have already copied binaries into the home directory of +# whichever user it has access too. This script is executed from within the home directory of that +# user. Bear in mind that different cloud providers, and different images on the same cloud +# provider will have a different initial user account. + +set -x + +# Become root, if we aren't already. +# Docker images will already be root. AMIs will have an SSH user account. +UID=$(id -u) +if [ x$UID != x0 ] +then + printf -v cmd_str '%q ' "$0" "$@" + exec sudo su -c "$cmd_str" +fi + +MANAGED_FILES=( + /etc/motd +) + +snap install filecoin-lotus + +snap alias lotus-filecoin.lotus lotus +snap alias lotus-filecoin.lotus-miner lotus-miner +snap alias lotus-filecoin.lotus-miner lotus-worker + +# Setup firewall +yes | ufw enable +ufw default deny incoming +ufw default allow outgoing +ufw allow ssh From dedf46ed1f5d5ec73edff67b5e1a928aaff06435 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Fri, 17 Dec 2021 17:12:10 -0800 Subject: [PATCH 02/11] use correct snap name --- tools/packer/setup-snap.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/packer/setup-snap.sh b/tools/packer/setup-snap.sh index 7b543bc364d..29e0bf26dfd 100644 --- a/tools/packer/setup-snap.sh +++ b/tools/packer/setup-snap.sh @@ -7,6 +7,7 @@ # provider will have a different initial user account. set -x +set -e # Become root, if we aren't already. # Docker images will already be root. AMIs will have an SSH user account. @@ -21,7 +22,7 @@ MANAGED_FILES=( /etc/motd ) -snap install filecoin-lotus +snap install lotus-filecoin snap alias lotus-filecoin.lotus lotus snap alias lotus-filecoin.lotus-miner lotus-miner From d874c633d784c4f591a15c05eb5ddde1d8407ac8 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Fri, 17 Dec 2021 17:14:53 -0800 Subject: [PATCH 03/11] set -e after switch user --- tools/packer/setup-snap.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/packer/setup-snap.sh b/tools/packer/setup-snap.sh index 29e0bf26dfd..7546023f265 100644 --- a/tools/packer/setup-snap.sh +++ b/tools/packer/setup-snap.sh @@ -7,7 +7,6 @@ # provider will have a different initial user account. set -x -set -e # Become root, if we aren't already. # Docker images will already be root. AMIs will have an SSH user account. @@ -18,6 +17,8 @@ then exec sudo su -c "$cmd_str" fi +set -e + MANAGED_FILES=( /etc/motd ) From a03e9e34ab6e9ee61d3b8f3a6890ac2717aea40e Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Fri, 17 Dec 2021 17:30:23 -0800 Subject: [PATCH 04/11] reinstall snapd --- tools/packer/setup-snap.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/packer/setup-snap.sh b/tools/packer/setup-snap.sh index 7546023f265..f8f94981616 100644 --- a/tools/packer/setup-snap.sh +++ b/tools/packer/setup-snap.sh @@ -10,7 +10,6 @@ set -x # Become root, if we aren't already. # Docker images will already be root. AMIs will have an SSH user account. -UID=$(id -u) if [ x$UID != x0 ] then printf -v cmd_str '%q ' "$0" "$@" @@ -23,6 +22,10 @@ MANAGED_FILES=( /etc/motd ) +# this is required on digitalocean, which does not have snap seeded correctly at this phase. +apt update +apt reinstall snapd + snap install lotus-filecoin snap alias lotus-filecoin.lotus lotus From d53a9e877abdf179042a930fea32bc7d9993c8e8 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Fri, 17 Dec 2021 17:49:32 -0800 Subject: [PATCH 05/11] cleanup script --- tools/packer/setup-snap.sh | 4 ++++ tools/packer/setup.sh | 2 ++ 2 files changed, 6 insertions(+) diff --git a/tools/packer/setup-snap.sh b/tools/packer/setup-snap.sh index f8f94981616..69c91cb604e 100644 --- a/tools/packer/setup-snap.sh +++ b/tools/packer/setup-snap.sh @@ -32,8 +32,12 @@ snap alias lotus-filecoin.lotus lotus snap alias lotus-filecoin.lotus-miner lotus-miner snap alias lotus-filecoin.lotus-miner lotus-worker +snap stop lotus-filecoin.lotus-daemon + # Setup firewall yes | ufw enable ufw default deny incoming ufw default allow outgoing ufw allow ssh + +curl -L https://raw.githubusercontent.com/digitalocean/marketplace-partners/master/scripts/90-cleanup.sh | bash diff --git a/tools/packer/setup.sh b/tools/packer/setup.sh index 6c0742254ff..2b190045e1b 100644 --- a/tools/packer/setup.sh +++ b/tools/packer/setup.sh @@ -65,3 +65,5 @@ ufw default deny incoming ufw default allow outgoing ufw allow ssh ufw allow 5678 #libp2p + +curl -L https://raw.githubusercontent.com/digitalocean/marketplace-partners/master/scripts/90-cleanup.sh | bash From 0e7e2529516fe22586ee2fffe6629aa3a493beae Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Fri, 17 Dec 2021 18:07:54 -0800 Subject: [PATCH 06/11] set +e before cleanup --- tools/packer/setup-snap.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/packer/setup-snap.sh b/tools/packer/setup-snap.sh index 69c91cb604e..462d393f8c1 100644 --- a/tools/packer/setup-snap.sh +++ b/tools/packer/setup-snap.sh @@ -40,4 +40,6 @@ ufw default deny incoming ufw default allow outgoing ufw allow ssh +set +e + curl -L https://raw.githubusercontent.com/digitalocean/marketplace-partners/master/scripts/90-cleanup.sh | bash From 70b8557b639e338f9bf78e6b56ce7c5073ce0043 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Fri, 17 Dec 2021 18:12:23 -0800 Subject: [PATCH 07/11] tmp, don't stop --- tools/packer/setup-snap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/packer/setup-snap.sh b/tools/packer/setup-snap.sh index 462d393f8c1..d72ca876249 100644 --- a/tools/packer/setup-snap.sh +++ b/tools/packer/setup-snap.sh @@ -32,7 +32,7 @@ snap alias lotus-filecoin.lotus lotus snap alias lotus-filecoin.lotus-miner lotus-miner snap alias lotus-filecoin.lotus-miner lotus-worker -snap stop lotus-filecoin.lotus-daemon +#snap stop lotus-filecoin.lotus-daemon # Setup firewall yes | ufw enable From 1cd93e63fdee10666cc0371cae2a535c64cd136d Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Tue, 18 Jan 2022 17:35:34 -0800 Subject: [PATCH 08/11] edge --- .circleci/config.yml | 8 +++++++- tools/packer/lotus-snap.pkr.hcl | 2 +- tools/packer/setup-snap.sh | 7 ++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a66a852986..a13a5796bee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1058,7 +1058,13 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish-packer-snap + - publish-snapcaft: + name: publish-snapcraft-temporary + channel: edge + - publish-packer-snap: + name: publish-packer-snap-temporary + requires: + - publish-snapcraft-temporary nightly: triggers: diff --git a/tools/packer/lotus-snap.pkr.hcl b/tools/packer/lotus-snap.pkr.hcl index ef0f52f8cec..8ceb743d088 100644 --- a/tools/packer/lotus-snap.pkr.hcl +++ b/tools/packer/lotus-snap.pkr.hcl @@ -10,7 +10,7 @@ variable "lotus_network" { locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") -} +} source "amazon-ebs" "lotus" { ami_name = "lotus-${var.lotus_network}-snap-${local.timestamp}" diff --git a/tools/packer/setup-snap.sh b/tools/packer/setup-snap.sh index d72ca876249..e4f838d9bed 100644 --- a/tools/packer/setup-snap.sh +++ b/tools/packer/setup-snap.sh @@ -26,13 +26,14 @@ MANAGED_FILES=( apt update apt reinstall snapd -snap install lotus-filecoin +snap install lotus-filecoin --channel=edge snap alias lotus-filecoin.lotus lotus +snap alias lotus-file.con.lotus-daemon lotus-daemon snap alias lotus-filecoin.lotus-miner lotus-miner -snap alias lotus-filecoin.lotus-miner lotus-worker +snap alias lotus-filecoin.lotus-worker lotus-worker -#snap stop lotus-filecoin.lotus-daemon +snap stop lotus-filecoin.lotus-daemon # Setup firewall yes | ufw enable From afac38e4920b9503333f90fbc6e3fed50d47ed1b Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Tue, 18 Jan 2022 17:36:44 -0800 Subject: [PATCH 09/11] spelling --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a13a5796bee..d967837dc35 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1058,7 +1058,7 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish-snapcaft: + - publish-snapcraft: name: publish-snapcraft-temporary channel: edge - publish-packer-snap: From 8cbf1a3e1b1f4a4d245be76b9a316a70fa87b5a9 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 19 Jan 2022 12:30:36 -0800 Subject: [PATCH 10/11] undo tmp edits --- .circleci/config.yml | 7 ------- tools/packer/setup-snap.sh | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d967837dc35..1bfe821da83 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1058,13 +1058,6 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish-snapcraft: - name: publish-snapcraft-temporary - channel: edge - - publish-packer-snap: - name: publish-packer-snap-temporary - requires: - - publish-snapcraft-temporary nightly: triggers: diff --git a/tools/packer/setup-snap.sh b/tools/packer/setup-snap.sh index e4f838d9bed..2e921f8da9f 100644 --- a/tools/packer/setup-snap.sh +++ b/tools/packer/setup-snap.sh @@ -26,7 +26,7 @@ MANAGED_FILES=( apt update apt reinstall snapd -snap install lotus-filecoin --channel=edge +snap install lotus-filecoin snap alias lotus-filecoin.lotus lotus snap alias lotus-file.con.lotus-daemon lotus-daemon From cdd2d5cc6a1d7c414a2827b8362f9210164bf554 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Fri, 18 Feb 2022 15:37:53 -0800 Subject: [PATCH 11/11] update template --- .circleci/template.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.circleci/template.yml b/.circleci/template.yml index 8f5995d56d1..9feecc3b544 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -705,6 +705,17 @@ jobs: - packer/build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG" + publish-packer-snap: + description: build packer image with snap. mainnet only. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." + - packer/build: + template: tools/packer/lotus-snap.pkr.hcl publish-dockerhub: description: publish to dockerhub machine: @@ -898,3 +909,13 @@ workflows: - publish-dockerhub: name: publish-dockerhub-nightly tag: nightly + monthly: + triggers: + - schedule: + cron: "0 0 1 * *" + filters: + branches: + only: + - master + jobs: + - publish-packer-snap