From ae3e0ebb91e3842d74e2e3964001a6adf0c10654 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 18 Jan 2021 08:41:46 -0500 Subject: [PATCH 01/13] nixos/nomad: add settingsFiles configuration option to nomad service --- nixos/modules/services/networking/nomad.nix | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 6c151175e5bd3..c36e574c2dad5 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -49,6 +49,18 @@ in ''; }; + settingsFiles = mkOption { + type = types.listOf (types.oneOf [ types.path types.str ]); + default = []; + description = '' + Additional settings files used to configure nomad. These files + will be watched for changes. + ''; + example = literalExample '' + [ "/etc/awesome.nomad.json" ] + ''; + }; + settings = mkOption { type = format.type; default = {}; @@ -89,7 +101,8 @@ in wantedBy = [ "multi-user.target" ]; wants = [ "network-online.target" ]; after = [ "network-online.target" ]; - restartTriggers = [ config.environment.etc."nomad.json".source ]; + restartTriggers = [ config.environment.etc."nomad.json".source ] + ++ cfg.settingsFiles; path = cfg.extraPackages ++ (with pkgs; [ # Client mode requires at least the following: @@ -101,7 +114,8 @@ in serviceConfig = { DynamicUser = cfg.dropPrivileges; ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; - ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json"; + ExecStart = "${cfg.package}/bin/nomad agent" + + concatMapStrings (file: " -config=${file}") (["/etc/nomad.json"] ++ cfg.settingsFiles); KillMode = "process"; KillSignal = "SIGINT"; LimitNOFILE = 65536; From 0cf1dbc6b914633480ab7290e0d18f149548c162 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 18 Jan 2021 08:44:30 -0500 Subject: [PATCH 02/13] nixos/nomad: revert small argument passing change --- nixos/modules/services/networking/nomad.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index c36e574c2dad5..a6c9e796390ef 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -114,8 +114,8 @@ in serviceConfig = { DynamicUser = cfg.dropPrivileges; ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; - ExecStart = "${cfg.package}/bin/nomad agent" + - concatMapStrings (file: " -config=${file}") (["/etc/nomad.json"] ++ cfg.settingsFiles); + ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json" + + concatMapStrings (file: " -config=${file}") cfg.settingsFiles; KillMode = "process"; KillSignal = "SIGINT"; LimitNOFILE = 65536; From 74a9853023bfd0642f70d5ee7a542dcff307d977 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 20 Jan 2021 17:13:46 -0500 Subject: [PATCH 03/13] Apply suggestions from code review Co-authored-by: Robert Hensing --- nixos/modules/services/networking/nomad.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index a6c9e796390ef..3816f26fcbb04 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -49,15 +49,15 @@ in ''; }; - settingsFiles = mkOption { - type = types.listOf (types.oneOf [ types.path types.str ]); + extraSettingsPaths = mkOption { + type = types.listOf types.path; default = []; description = '' Additional settings files used to configure nomad. These files will be watched for changes. ''; example = literalExample '' - [ "/etc/awesome.nomad.json" ] + [ "/etc/nomad-mutable.json" "/run/keys/nomad-with-secrets.json" ] ''; }; From b1ee24e93974844c1e138dc1c30486eb5c10f160 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Wed, 20 Jan 2021 20:01:22 -0500 Subject: [PATCH 04/13] nixos/nomad: refer to extraSettingsPaths variable --- nixos/modules/services/networking/nomad.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 3816f26fcbb04..68ecaaec4ad58 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -102,7 +102,7 @@ in wants = [ "network-online.target" ]; after = [ "network-online.target" ]; restartTriggers = [ config.environment.etc."nomad.json".source ] - ++ cfg.settingsFiles; + ++ cfg.extraSettingsPaths; path = cfg.extraPackages ++ (with pkgs; [ # Client mode requires at least the following: @@ -115,7 +115,7 @@ in DynamicUser = cfg.dropPrivileges; ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json" + - concatMapStrings (file: " -config=${file}") cfg.settingsFiles; + concatMapStrings (file: " -config=${file}") cfg.extraSettingsPaths; KillMode = "process"; KillSignal = "SIGINT"; LimitNOFILE = 65536; From 8709da69ab521d08bb9705a0179cce49242198cf Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 12:13:45 -0500 Subject: [PATCH 05/13] nixos/nomad: add simple test --- nixos/tests/all-tests.nix | 1 + nixos/tests/nomad.nix | 60 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 nixos/tests/nomad.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 966c78446574c..523d3c051e049 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -272,6 +272,7 @@ in nginx-variants = handleTest ./nginx-variants.nix {}; nix-ssh-serve = handleTest ./nix-ssh-serve.nix {}; nixos-generate-config = handleTest ./nixos-generate-config.nix {}; + nomad = handleTest ./nomad.nix {}; novacomd = handleTestOn ["x86_64-linux"] ./novacomd.nix {}; nsd = handleTest ./nsd.nix {}; nzbget = handleTest ./nzbget.nix {}; diff --git a/nixos/tests/nomad.nix b/nixos/tests/nomad.nix new file mode 100644 index 0000000000000..de5b495b0b0d3 --- /dev/null +++ b/nixos/tests/nomad.nix @@ -0,0 +1,60 @@ +import ./make-test-python.nix ( + { lib, ... }: { + name = "nomad"; + nodes = { + server = { config, pkgs, lib, ... }: { + networking = { + interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{ + address = "192.168.1.1"; + prefixLength = 16; + }]; + + firewall = { + allowedTCPPorts = [ 4646 4647 4648 ]; + allowedUDPPorts = [ 4646 4647 4648 ]; + }; + }; + + environment.etc."nomad.custom.json".source = + (pkgs.formats.json { }).generate "nomad.custom.json" { + region = "universe"; + datacenter = "earth"; + }; + + services.nomad = { + enable = true; + + settings = { + data_dir = "/var/lib/nomad"; + + server = { + enabled = true; + bootstrap_expect = 1; + }; + }; + + extraSettingsPaths = [ config.environment.etc."nomad.custom.json".source ]; + enableDocker = false; + }; + }; + }; + + testScript = '' + server.wait_for_unit("nomad.service") + + # wait for healthy server + server.wait_until_succeeds( + "[ $(nomad operator raft list-peers | grep true | wc -l) == 1 ]" + ) + + # wait for server liveness + server.succeed("[ $(nomad server members | grep -o alive | wc -l) == 1 ]") + + # check the region + server.succeed("nomad server members | grep -o universe") + + # check the datacenter + server.succeed("[ $(nomad server members | grep -o earth | wc -l) == 1 ]") + ''; + } +) From fc102bfe2843babb93e48b02a311f1ed2114cefb Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 12:14:26 -0500 Subject: [PATCH 06/13] nixos/nomad: remove cfg.extraSettingsPaths from restartTriggers --- nixos/modules/services/networking/nomad.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 68ecaaec4ad58..f65194866ae62 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -101,8 +101,7 @@ in wantedBy = [ "multi-user.target" ]; wants = [ "network-online.target" ]; after = [ "network-online.target" ]; - restartTriggers = [ config.environment.etc."nomad.json".source ] - ++ cfg.extraSettingsPaths; + restartTriggers = [ config.environment.etc."nomad.json".source ]; path = cfg.extraPackages ++ (with pkgs; [ # Client mode requires at least the following: From 18262fefe28ffb66d91110f3ae21828ed7af08a3 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 12:17:00 -0500 Subject: [PATCH 07/13] nixos/nomad: remove documentation about restarts --- nixos/modules/services/networking/nomad.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index f65194866ae62..3f40cb2ab6fa3 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -53,8 +53,7 @@ in type = types.listOf types.path; default = []; description = '' - Additional settings files used to configure nomad. These files - will be watched for changes. + Additional settings files used to configure nomad. ''; example = literalExample '' [ "/etc/nomad-mutable.json" "/run/keys/nomad-with-secrets.json" ] From f20c0fdb2f7d977cf07285e6f7f8681bf810eea3 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 12:19:51 -0500 Subject: [PATCH 08/13] nixos/nomad: add documentation calling out the ability to use directories --- nixos/modules/services/networking/nomad.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 3f40cb2ab6fa3..bef4eaf871571 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -53,10 +53,10 @@ in type = types.listOf types.path; default = []; description = '' - Additional settings files used to configure nomad. + Additional settings paths used to configure nomad. These can be files or directories. ''; example = literalExample '' - [ "/etc/nomad-mutable.json" "/run/keys/nomad-with-secrets.json" ] + [ "/etc/nomad-mutable.json" "/run/keys/nomad-with-secrets.json" "/etc/nomad/config.d" ] ''; }; From 984c07d21c793d07d3d54994461406a6e6325b1c Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 12:21:16 -0500 Subject: [PATCH 09/13] nixos/nomad: remove unnecessary open firewall ports --- nixos/tests/nomad.nix | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nixos/tests/nomad.nix b/nixos/tests/nomad.nix index de5b495b0b0d3..849cade218023 100644 --- a/nixos/tests/nomad.nix +++ b/nixos/tests/nomad.nix @@ -8,11 +8,6 @@ import ./make-test-python.nix ( address = "192.168.1.1"; prefixLength = 16; }]; - - firewall = { - allowedTCPPorts = [ 4646 4647 4648 ]; - allowedUDPPorts = [ 4646 4647 4648 ]; - }; }; environment.etc."nomad.custom.json".source = From 180006b23f82c790451d9f22e3110530643d116b Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 12:46:10 -0500 Subject: [PATCH 10/13] nixos/nomad: make variable name accurate --- nixos/modules/services/networking/nomad.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index bef4eaf871571..dafdae0c327b9 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -113,7 +113,7 @@ in DynamicUser = cfg.dropPrivileges; ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json" + - concatMapStrings (file: " -config=${file}") cfg.extraSettingsPaths; + concatMapStrings (path: " -config=${path}") cfg.extraSettingsPaths; KillMode = "process"; KillSignal = "SIGINT"; LimitNOFILE = 65536; From 528fb0020399a9b1fa2e12274352aac5ee461a52 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Sat, 23 Jan 2021 12:53:59 -0500 Subject: [PATCH 11/13] nixos/nomad: use non store path for test Co-authored-by: Robert Hensing --- nixos/tests/nomad.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/tests/nomad.nix b/nixos/tests/nomad.nix index 849cade218023..f23d655808400 100644 --- a/nixos/tests/nomad.nix +++ b/nixos/tests/nomad.nix @@ -28,7 +28,7 @@ import ./make-test-python.nix ( }; }; - extraSettingsPaths = [ config.environment.etc."nomad.custom.json".source ]; + extraSettingsPaths = [ "/etc/nomad.custom.json" ]; enableDocker = false; }; }; From bdc00d6c9eb1dae4c5489e97779eb9dd49b00677 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 12:55:47 -0500 Subject: [PATCH 12/13] nixos/tests: remove unnecessary config argument from nomad test --- nixos/tests/nomad.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/tests/nomad.nix b/nixos/tests/nomad.nix index f23d655808400..d6460efa89013 100644 --- a/nixos/tests/nomad.nix +++ b/nixos/tests/nomad.nix @@ -2,7 +2,7 @@ import ./make-test-python.nix ( { lib, ... }: { name = "nomad"; nodes = { - server = { config, pkgs, lib, ... }: { + server = { pkgs, lib, ... }: { networking = { interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{ address = "192.168.1.1"; From 0c52d114d9e97281c324f3e77b47a6bc6b0aeee4 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 17:16:28 -0500 Subject: [PATCH 13/13] nixos/tests: remove data_dir argument from test --- nixos/tests/nomad.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/nixos/tests/nomad.nix b/nixos/tests/nomad.nix index d6460efa89013..bd052152bd653 100644 --- a/nixos/tests/nomad.nix +++ b/nixos/tests/nomad.nix @@ -20,8 +20,6 @@ import ./make-test-python.nix ( enable = true; settings = { - data_dir = "/var/lib/nomad"; - server = { enabled = true; bootstrap_expect = 1;