From ca7d2369af87b0ed84e5aa3bcbfb0daff30ccc6a Mon Sep 17 00:00:00 2001 From: Michele Catalano Date: Fri, 6 Apr 2018 15:11:52 +0200 Subject: [PATCH 01/11] Add garbage collector to docker registry --- .../modules/services/misc/docker-registry.nix | 40 +++++++++++++++++-- nixos/tests/docker-registry.nix | 2 +- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/nixos/modules/services/misc/docker-registry.nix b/nixos/modules/services/misc/docker-registry.nix index a44162409fc0f..41468f89f2598 100644 --- a/nixos/modules/services/misc/docker-registry.nix +++ b/nixos/modules/services/misc/docker-registry.nix @@ -42,6 +42,8 @@ let }; }; + configFile = pkgs.writeText "docker-registry-config.yml" (builtins.toJSON (registryConfig // cfg.extraConfig)); + in { options.services.dockerRegistry = { enable = mkEnableOption "Docker Registry"; @@ -95,6 +97,24 @@ in { default = {}; type = types.attrsOf types.str; }; + + enableGarbageCollect = mkOption { + description = "Enable run GarbageCollect automatic every given time."; + default = false; + type = types.bool; + }; + + garbageCollectDates = mkOption { + default = "daily"; + type = types.str; + description = '' + Specification (in the format described by + systemd.time + 7) of the time at + which the garbage collect will occur. + '' + }; + }; }; config = mkIf cfg.enable { @@ -102,9 +122,7 @@ in { description = "Docker Container Registry"; wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; - script = let - configFile = pkgs.writeText "docker-registry-config.yml" (builtins.toJSON (registryConfig // cfg.extraConfig)); - in '' + script = '' ${pkgs.docker-distribution}/bin/registry serve ${configFile} ''; @@ -114,6 +132,22 @@ in { }; }; + systemd.services.docker-registry-garbage-collect = { + description = "Run Garbage Collection for docker registry"; + + restartIfChange = false; + unitConfig.X-StopOnRemoval = false; + + serviceConfig.Type = "oneshot"; + + script = '' + ${pkgs.docker-distribution}/bin/registry garbage-collect ${configFile} + ${lib.optionalString enableRedisCache '${pkgs.systemd}/bin/systemctl restart docker-registry'} + ''; + + startAt = optional cfg.enableGarbageCollect cfg.garbageCollectDates; + }; + users.extraUsers.docker-registry = { createHome = true; home = cfg.storagePath; diff --git a/nixos/tests/docker-registry.nix b/nixos/tests/docker-registry.nix index 943773ee39184..1092a8e6a6548 100644 --- a/nixos/tests/docker-registry.nix +++ b/nixos/tests/docker-registry.nix @@ -3,7 +3,7 @@ import ./make-test.nix ({ pkgs, ...} : { name = "docker-registry"; meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ globin ma27 ]; + maintainers = [ globin ma27 ironpinguin ]; }; nodes = { From 179ef9a37a0dcb51fc0fbcd40e1a6bd8024dccf2 Mon Sep 17 00:00:00 2001 From: Michele Catalano Date: Fri, 6 Apr 2018 16:31:50 +0200 Subject: [PATCH 02/11] Extend Test with run gc - 1. delete uploaded manifest - 2. run gc to remove all depending blobs from filesystem (and restart registry) - 3. check if blobs are deleted - 4. push again the image - 5. check if blobs exists in filesystem (means that the registry was correct restarted) --- nixos/tests/docker-registry.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nixos/tests/docker-registry.nix b/nixos/tests/docker-registry.nix index 1092a8e6a6548..47019c920f407 100644 --- a/nixos/tests/docker-registry.nix +++ b/nixos/tests/docker-registry.nix @@ -43,7 +43,15 @@ import ./make-test.nix ({ pkgs, ...} : { $client2->succeed("docker images | grep scratch"); $client2->succeed( - 'curl -fsS -X DELETE registry:8080/v2/scratch/manifests/$(curl registry:8080/v2/scratch/manifests/latest | jq ".fsLayers[0].blobSum" | sed -e \'s/"//g\')' + 'curl -fsS -X DELETE registry:8080/v2/scratch/manifests/$(curl -fsS -I -H"Accept: application/vnd.docker.distribution.manifest.v2+json" registry:8080/v2/scratch/manifests/latest | grep Docker-Content | sed -e\'Docker-Content-Digest: //\' | tr -d \'\r\')' ); + + $registry->succeed("systemctl start docker-registry-garbage-collect"); + $registry->waitForUnit("docker-registry.service"); + + $registry->fail("ls store/docker/registry/v2/blobs/sha256/**/data"); + + $client1->succeed("docker push registry:8080/scratch"); + $registry->succeed("ls store/docker/registry/v2/blobs/sha256/**/data"); ''; }) From 17d348a7928b845ba70b9d4bdf6981b863a8c91d Mon Sep 17 00:00:00 2001 From: Michele Catalano Date: Sat, 7 Apr 2018 09:50:05 +0200 Subject: [PATCH 03/11] change to mkEnableOption and fix typos - Also change enableRedisCache to mkEnableOption --- nixos/modules/services/misc/docker-registry.nix | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/nixos/modules/services/misc/docker-registry.nix b/nixos/modules/services/misc/docker-registry.nix index 41468f89f2598..f7d733b8c4325 100644 --- a/nixos/modules/services/misc/docker-registry.nix +++ b/nixos/modules/services/misc/docker-registry.nix @@ -72,11 +72,7 @@ in { description = "Enable delete for manifests and blobs."; }; - enableRedisCache = mkOption { - type = types.bool; - default = false; - description = "Enable redis as blob cache instade of inmemory."; - }; + enableRedisCache = mkEnableOption "redis as blob cache"; redisUrl = mkOption { type = types.str; @@ -98,11 +94,7 @@ in { type = types.attrsOf types.str; }; - enableGarbageCollect = mkOption { - description = "Enable run GarbageCollect automatic every given time."; - default = false; - type = types.bool; - }; + enableGarbageCollect = mkEnableOption "garbage collect"; garbageCollectDates = mkOption { default = "daily"; @@ -113,7 +105,6 @@ in { 7) of the time at which the garbage collect will occur. '' - }; }; }; @@ -142,7 +133,7 @@ in { script = '' ${pkgs.docker-distribution}/bin/registry garbage-collect ${configFile} - ${lib.optionalString enableRedisCache '${pkgs.systemd}/bin/systemctl restart docker-registry'} + ${lib.optionalString enableRedisCache "${pkgs.systemd}/bin/systemctl restart docker-registry"} ''; startAt = optional cfg.enableGarbageCollect cfg.garbageCollectDates; From db9f58bb2200e1a800b6453cfe7f205f79974c82 Mon Sep 17 00:00:00 2001 From: Michele Catalano Date: Sat, 7 Apr 2018 10:19:04 +0200 Subject: [PATCH 04/11] Fix copy & paste string --- nixos/tests/docker-registry.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos/tests/docker-registry.nix b/nixos/tests/docker-registry.nix index 47019c920f407..839e114830718 100644 --- a/nixos/tests/docker-registry.nix +++ b/nixos/tests/docker-registry.nix @@ -49,9 +49,9 @@ import ./make-test.nix ({ pkgs, ...} : { $registry->succeed("systemctl start docker-registry-garbage-collect"); $registry->waitForUnit("docker-registry.service"); - $registry->fail("ls store/docker/registry/v2/blobs/sha256/**/data"); + $registry->fail("ls /var/lib/registry/docker/registry/v2/blobs/sha256/**/data"); $client1->succeed("docker push registry:8080/scratch"); - $registry->succeed("ls store/docker/registry/v2/blobs/sha256/**/data"); + $registry->succeed("ls /var/lib/registry/docker/registry/v2/blobs/sha256/**/data"); ''; }) From ad71bb441fecb020c0b1ef3656e48e54378487b2 Mon Sep 17 00:00:00 2001 From: Michele Catalano Date: Sat, 7 Apr 2018 10:30:19 +0200 Subject: [PATCH 05/11] Fix a other typo --- nixos/modules/services/misc/docker-registry.nix | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nixos/modules/services/misc/docker-registry.nix b/nixos/modules/services/misc/docker-registry.nix index f7d733b8c4325..7e3806a53b750 100644 --- a/nixos/modules/services/misc/docker-registry.nix +++ b/nixos/modules/services/misc/docker-registry.nix @@ -98,13 +98,13 @@ in { garbageCollectDates = mkOption { default = "daily"; - type = types.str; - description = '' - Specification (in the format described by - systemd.time - 7) of the time at - which the garbage collect will occur. - '' + type = types.str; + description = '' + Specification (in the format described by + systemd.time + 7) of the time at + which the garbage collect will occur. + ''; }; }; From 6a8194f53b91c9478af6b59c1f5df864d3757af7 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 7 Apr 2018 21:49:22 +0200 Subject: [PATCH 06/11] fix evaluation + applying minor fixes --- nixos/modules/services/misc/docker-registry.nix | 4 ++-- nixos/tests/docker-registry.nix | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nixos/modules/services/misc/docker-registry.nix b/nixos/modules/services/misc/docker-registry.nix index 7e3806a53b750..e73ba4e80be03 100644 --- a/nixos/modules/services/misc/docker-registry.nix +++ b/nixos/modules/services/misc/docker-registry.nix @@ -126,14 +126,14 @@ in { systemd.services.docker-registry-garbage-collect = { description = "Run Garbage Collection for docker registry"; - restartIfChange = false; + restartIfChanged = false; unitConfig.X-StopOnRemoval = false; serviceConfig.Type = "oneshot"; script = '' ${pkgs.docker-distribution}/bin/registry garbage-collect ${configFile} - ${lib.optionalString enableRedisCache "${pkgs.systemd}/bin/systemctl restart docker-registry"} + ${optionalString cfg.enableRedisCache "${pkgs.systemd}/bin/systemctl restart docker-registry"} ''; startAt = optional cfg.enableGarbageCollect cfg.garbageCollectDates; diff --git a/nixos/tests/docker-registry.nix b/nixos/tests/docker-registry.nix index 839e114830718..8089372cb1a10 100644 --- a/nixos/tests/docker-registry.nix +++ b/nixos/tests/docker-registry.nix @@ -12,6 +12,7 @@ import ./make-test.nix ({ pkgs, ...} : { services.dockerRegistry.enableDelete = true; services.dockerRegistry.port = 8080; services.dockerRegistry.listenAddress = "0.0.0.0"; + services.dockerRegistry.enableGarbageCollect = true; networking.firewall.allowedTCPPorts = [ 8080 ]; }; @@ -23,7 +24,6 @@ import ./make-test.nix ({ pkgs, ...} : { client2 = { config, pkgs, ...}: { virtualisation.docker.enable = true; virtualisation.docker.extraOptions = "--insecure-registry registry:8080"; - environment.systemPackages = [ pkgs.jq ]; }; }; @@ -31,7 +31,7 @@ import ./make-test.nix ({ pkgs, ...} : { $client1->start(); $client1->waitForUnit("docker.service"); $client1->succeed("tar cv --files-from /dev/null | docker import - scratch"); - $client1->succeed("docker tag scratch registry:8080/scratch"); + $client1->succeed("docker tag scratch registry:8080/scratch:latest"); $registry->start(); $registry->waitForUnit("docker-registry.service"); @@ -43,7 +43,7 @@ import ./make-test.nix ({ pkgs, ...} : { $client2->succeed("docker images | grep scratch"); $client2->succeed( - 'curl -fsS -X DELETE registry:8080/v2/scratch/manifests/$(curl -fsS -I -H"Accept: application/vnd.docker.distribution.manifest.v2+json" registry:8080/v2/scratch/manifests/latest | grep Docker-Content | sed -e\'Docker-Content-Digest: //\' | tr -d \'\r\')' + 'curl -fsS -X DELETE registry:8080/v2/scratch/manifests/$(curl -fsS -I -H"Accept: application/vnd.docker.distribution.manifest.v2+json" registry:8080/v2/scratch/manifests/latest | grep docker-content-digest | sed -e \'s/docker-content-digest: //\' | tr -d \'\r\')' ); $registry->succeed("systemctl start docker-registry-garbage-collect"); From 7c31c36f8a1b2c50b31abf1b01cdee82eb9280d1 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 8 Apr 2018 13:52:05 +0200 Subject: [PATCH 07/11] start fixing tests --- nixos/modules/services/misc/docker-registry.nix | 5 ++++- nixos/tests/docker-registry.nix | 13 +++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/nixos/modules/services/misc/docker-registry.nix b/nixos/modules/services/misc/docker-registry.nix index e73ba4e80be03..b36ae561ddfdb 100644 --- a/nixos/modules/services/misc/docker-registry.nix +++ b/nixos/modules/services/misc/docker-registry.nix @@ -132,8 +132,11 @@ in { serviceConfig.Type = "oneshot"; script = '' + ${pkgs.systemd}/bin/systemctl stop docker-registry.service ${pkgs.docker-distribution}/bin/registry garbage-collect ${configFile} - ${optionalString cfg.enableRedisCache "${pkgs.systemd}/bin/systemctl restart docker-registry"} + ${pkgs.systemd}/bin/systemctl start docker-registry.service + + ls /var/lib/docker-registry/docker/registry/v2/blobs/sha256 ''; startAt = optional cfg.enableGarbageCollect cfg.garbageCollectDates; diff --git a/nixos/tests/docker-registry.nix b/nixos/tests/docker-registry.nix index 8089372cb1a10..1e500e8a56023 100644 --- a/nixos/tests/docker-registry.nix +++ b/nixos/tests/docker-registry.nix @@ -31,7 +31,7 @@ import ./make-test.nix ({ pkgs, ...} : { $client1->start(); $client1->waitForUnit("docker.service"); $client1->succeed("tar cv --files-from /dev/null | docker import - scratch"); - $client1->succeed("docker tag scratch registry:8080/scratch:latest"); + $client1->succeed("docker tag scratch registry:8080/scratch"); $registry->start(); $registry->waitForUnit("docker-registry.service"); @@ -43,15 +43,20 @@ import ./make-test.nix ({ pkgs, ...} : { $client2->succeed("docker images | grep scratch"); $client2->succeed( - 'curl -fsS -X DELETE registry:8080/v2/scratch/manifests/$(curl -fsS -I -H"Accept: application/vnd.docker.distribution.manifest.v2+json" registry:8080/v2/scratch/manifests/latest | grep docker-content-digest | sed -e \'s/docker-content-digest: //\' | tr -d \'\r\')' + 'curl -fsS -X DELETE registry:8080/v2/scratch/manifests/$(curl -fsS -I -H"Accept: application/vnd.docker.distribution.manifest.v2+json" registry:8080/v2/scratch/manifests/latest | grep Docker-Content-Digest | sed -e \'s/Docker-Content-Digest: //\' | tr -d \'\r\')' ); $registry->succeed("systemctl start docker-registry-garbage-collect"); $registry->waitForUnit("docker-registry.service"); - $registry->fail("ls /var/lib/registry/docker/registry/v2/blobs/sha256/**/data"); + $registry->fail( + 'bash -c ls -ld /var/lib/docker-registry/docker/registry/v2/blobs/sha256/*/*/data' + ); $client1->succeed("docker push registry:8080/scratch"); - $registry->succeed("ls /var/lib/registry/docker/registry/v2/blobs/sha256/**/data"); + $registry->execute("ls -l /var/lib/docker-registry/docker/registry/v2/blobs/sha256"); + $registry->succeed( + 'bash -c ls -ld /var/lib/docker-registry/docker/registry/v2/blobs/sha256/*/*/data' + ); ''; }) From fa6528806080c989c59d683c90aae89a469d1ef3 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 8 Apr 2018 15:59:55 +0200 Subject: [PATCH 08/11] stupid me --- nixos/modules/services/misc/docker-registry.nix | 2 -- nixos/tests/docker-registry.nix | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/nixos/modules/services/misc/docker-registry.nix b/nixos/modules/services/misc/docker-registry.nix index b36ae561ddfdb..10afc571a323f 100644 --- a/nixos/modules/services/misc/docker-registry.nix +++ b/nixos/modules/services/misc/docker-registry.nix @@ -135,8 +135,6 @@ in { ${pkgs.systemd}/bin/systemctl stop docker-registry.service ${pkgs.docker-distribution}/bin/registry garbage-collect ${configFile} ${pkgs.systemd}/bin/systemctl start docker-registry.service - - ls /var/lib/docker-registry/docker/registry/v2/blobs/sha256 ''; startAt = optional cfg.enableGarbageCollect cfg.garbageCollectDates; diff --git a/nixos/tests/docker-registry.nix b/nixos/tests/docker-registry.nix index 1e500e8a56023..ee79735b43619 100644 --- a/nixos/tests/docker-registry.nix +++ b/nixos/tests/docker-registry.nix @@ -50,13 +50,12 @@ import ./make-test.nix ({ pkgs, ...} : { $registry->waitForUnit("docker-registry.service"); $registry->fail( - 'bash -c ls -ld /var/lib/docker-registry/docker/registry/v2/blobs/sha256/*/*/data' + 'ls -l /var/lib/docker-registry/docker/registry/v2/blobs/sha256/*/*/data' ); $client1->succeed("docker push registry:8080/scratch"); - $registry->execute("ls -l /var/lib/docker-registry/docker/registry/v2/blobs/sha256"); $registry->succeed( - 'bash -c ls -ld /var/lib/docker-registry/docker/registry/v2/blobs/sha256/*/*/data' + 'ls -l /var/lib/docker-registry/docker/registry/v2/blobs/sha256/*/*/data' ); ''; }) From ef2fd100eff19350c3f4f3c8a0ff57b1d4065cb9 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 8 Apr 2018 16:07:19 +0200 Subject: [PATCH 09/11] review comment --- nixos/tests/docker-registry.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/tests/docker-registry.nix b/nixos/tests/docker-registry.nix index ee79735b43619..27eec26fb2210 100644 --- a/nixos/tests/docker-registry.nix +++ b/nixos/tests/docker-registry.nix @@ -46,7 +46,7 @@ import ./make-test.nix ({ pkgs, ...} : { 'curl -fsS -X DELETE registry:8080/v2/scratch/manifests/$(curl -fsS -I -H"Accept: application/vnd.docker.distribution.manifest.v2+json" registry:8080/v2/scratch/manifests/latest | grep Docker-Content-Digest | sed -e \'s/Docker-Content-Digest: //\' | tr -d \'\r\')' ); - $registry->succeed("systemctl start docker-registry-garbage-collect"); + $registry->systemctl("start docker-registry-garbage-collect"); $registry->waitForUnit("docker-registry.service"); $registry->fail( From 8d05940648c45b1dbd9eb5fb3f7ebe6188db287b Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 8 Apr 2018 16:42:52 +0200 Subject: [PATCH 10/11] just restart in the end --- nixos/modules/services/misc/docker-registry.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nixos/modules/services/misc/docker-registry.nix b/nixos/modules/services/misc/docker-registry.nix index 10afc571a323f..4fce9dd382ff3 100644 --- a/nixos/modules/services/misc/docker-registry.nix +++ b/nixos/modules/services/misc/docker-registry.nix @@ -132,9 +132,8 @@ in { serviceConfig.Type = "oneshot"; script = '' - ${pkgs.systemd}/bin/systemctl stop docker-registry.service ${pkgs.docker-distribution}/bin/registry garbage-collect ${configFile} - ${pkgs.systemd}/bin/systemctl start docker-registry.service + ${pkgs.systemd}/bin/systemctl restart docker-registry.service ''; startAt = optional cfg.enableGarbageCollect cfg.garbageCollectDates; From 384749ab26c84ef86fe8532eb9f872d248b70cbf Mon Sep 17 00:00:00 2001 From: System administrator Date: Sun, 8 Apr 2018 15:04:27 +0000 Subject: [PATCH 11/11] Add wait for open docker-registry port --- nixos/tests/docker-registry.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nixos/tests/docker-registry.nix b/nixos/tests/docker-registry.nix index 27eec26fb2210..1fbd199c7bc4f 100644 --- a/nixos/tests/docker-registry.nix +++ b/nixos/tests/docker-registry.nix @@ -35,6 +35,7 @@ import ./make-test.nix ({ pkgs, ...} : { $registry->start(); $registry->waitForUnit("docker-registry.service"); + $registry->waitForOpenPort("8080"); $client1->succeed("docker push registry:8080/scratch"); $client2->start(); @@ -46,7 +47,8 @@ import ./make-test.nix ({ pkgs, ...} : { 'curl -fsS -X DELETE registry:8080/v2/scratch/manifests/$(curl -fsS -I -H"Accept: application/vnd.docker.distribution.manifest.v2+json" registry:8080/v2/scratch/manifests/latest | grep Docker-Content-Digest | sed -e \'s/Docker-Content-Digest: //\' | tr -d \'\r\')' ); - $registry->systemctl("start docker-registry-garbage-collect"); + $registry->systemctl("start docker-registry-garbage-collect.service"); + $registry->waitUntilFails("systemctl status docker-registry-garbage-collect.service"); $registry->waitForUnit("docker-registry.service"); $registry->fail(