From 77c0d440a5dbfe6d8107d1602b78768ce693a198 Mon Sep 17 00:00:00 2001 From: Paulo Lieuthier Date: Mon, 12 Apr 2021 08:29:06 -0300 Subject: [PATCH 1/6] Match file path relative to config file's directory (2) (#853) * Match file path relative to config file's directory * Update README.rst Signed-off-by: Carsten Skov --- README.rst | 2 +- config/config.go | 15 +++++++++++-- config/config_test.go | 51 ++++++++++++++++++++++++++++--------------- 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index b05fcf107..f94e69795 100644 --- a/README.rst +++ b/README.rst @@ -650,7 +650,7 @@ and its KMS and PGP keys are used to encrypt the file. It should be noted that the looking up of ``.sops.yaml`` is from the working directory (CWD) instead of the directory of the encrypting file (see `Issue 242 `_). -The path_regex checks the full path of the encrypting file. Here is another example: +The path_regex checks the path of the encrypting file relative to the .sops.yaml config file. Here is another example: * files located under directory **development** should use one set of KMS A * files located under directory **production** should use another set of KMS B diff --git a/config/config.go b/config/config.go index e89336ddc..061d038e2 100644 --- a/config/config.go +++ b/config/config.go @@ -8,7 +8,9 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "regexp" + "strings" "github.com/sirupsen/logrus" "go.mozilla.org/sops/v3" @@ -313,12 +315,20 @@ func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptio return config, nil } -func parseCreationRuleForFile(conf *configFile, filePath string, kmsEncryptionContext map[string]*string) (*Config, error) { +func parseCreationRuleForFile(conf *configFile, confPath, filePath string, kmsEncryptionContext map[string]*string) (*Config, error) { // If config file doesn't contain CreationRules (it's empty or only contains DestionationRules), assume it does not exist if conf.CreationRules == nil { return nil, nil } + configDir, err := filepath.Abs(filepath.Dir(confPath)) + if err != nil { + return nil, err + } + + // compare file path relative to path of config file + filePath = strings.TrimPrefix(filePath, configDir + string(filepath.Separator)) + var rule *creationRule for _, r := range conf.CreationRules { @@ -356,7 +366,8 @@ func LoadCreationRuleForFile(confPath string, filePath string, kmsEncryptionCont if err != nil { return nil, err } - return parseCreationRuleForFile(conf, filePath, kmsEncryptionContext) + + return parseCreationRuleForFile(conf, confPath, filePath, kmsEncryptionContext) } // LoadDestinationRuleForFile works the same as LoadCreationRuleForFile, but gets the "creation_rule" from the matching destination_rule's diff --git a/config/config_test.go b/config/config_test.go index ac8aca6f3..a653fcb8e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -75,6 +75,15 @@ creation_rules: hc_vault_uris: https://foz:443/v1/foz/keys/foz `) +var sampleConfigWithAmbiguousPath = []byte(` +creation_rules: + - path_regex: foo/* + kms: "1" + pgp: "2" + gcp_kms: "3" + hc_vault_uris: http://4:8200/v1/4/keys/4 +`) + var sampleConfigWithGroups = []byte(` creation_rules: - path_regex: foobar* @@ -299,12 +308,12 @@ func TestLoadConfigFileWithGroups(t *testing.T) { } func TestLoadConfigFileWithNoMatchingRules(t *testing.T) { - _, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithNoMatchingRules, t), "foobar2000", nil) + _, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithNoMatchingRules, t), "/conf/path", "foobar2000", nil) assert.NotNil(t, err) } func TestLoadConfigFileWithInvalidComplicatedRegexp(t *testing.T) { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithInvalidComplicatedRegexp, t), "stage/prod/api.yml", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithInvalidComplicatedRegexp, t), "/conf/path", "stage/prod/api.yml", nil) assert.Equal(t, "can not compile regexp: error parsing regexp: invalid escape sequence: `\\K`", err.Error()) assert.Nil(t, conf) } @@ -315,58 +324,58 @@ func TestLoadConfigFileWithComplicatedRegexp(t *testing.T) { "stage/dev/feature-foo.yml": "dev-feature", "stage/dev/api.yml": "dev", } { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithComplicatedRegexp, t), filePath, nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithComplicatedRegexp, t), "/conf/path", filePath, nil) assert.Nil(t, err) assert.Equal(t, k, conf.KeyGroups[0][0].ToString()) } } func TestLoadEmptyConfigFile(t *testing.T) { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleEmptyConfig, t), "foobar2000", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleEmptyConfig, t), "/conf/path", "foobar2000", nil) assert.Nil(t, conf) assert.Nil(t, err) } func TestLoadConfigFileWithEmptyCreationRules(t *testing.T) { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithEmptyCreationRules, t), "foobar2000", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithEmptyCreationRules, t), "/conf/path", "foobar2000", nil) assert.Nil(t, conf) assert.Nil(t, err) } func TestLoadConfigFileWithOnlyDestinationRules(t *testing.T) { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithOnlyDestinationRules, t), "foobar2000", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithOnlyDestinationRules, t), "/conf/path", "foobar2000", nil) assert.Nil(t, conf) assert.Nil(t, err) } func TestKeyGroupsForFile(t *testing.T) { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "foobar2000", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "/conf/path", "foobar2000", nil) assert.Nil(t, err) assert.Equal(t, "2", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "1", conf.KeyGroups[0][1].ToString()) - conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "whatever", nil) + conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "/conf/path", "whatever", nil) assert.Nil(t, err) assert.Equal(t, "bar", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "foo", conf.KeyGroups[0][1].ToString()) } func TestKeyGroupsForFileWithPath(t *testing.T) { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithPath, t), "foo/bar2000", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithPath, t), "/conf/path", "foo/bar2000", nil) assert.Nil(t, err) assert.Equal(t, "2", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "1", conf.KeyGroups[0][1].ToString()) - conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfigWithPath, t), "somefilename.yml", nil) + conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfigWithPath, t), "/conf/path", "somefilename.yml", nil) assert.Nil(t, err) assert.Equal(t, "baggins", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "bilbo", conf.KeyGroups[0][1].ToString()) - conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "whatever", nil) + conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "/conf/path", "whatever", nil) assert.Nil(t, err) assert.Equal(t, "bar", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "foo", conf.KeyGroups[0][1].ToString()) } func TestKeyGroupsForFileWithGroups(t *testing.T) { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithGroups, t), "whatever", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithGroups, t), "/conf/path", "whatever", nil) assert.Nil(t, err) assert.Equal(t, "bar", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "foo", conf.KeyGroups[0][1].ToString()) @@ -375,31 +384,39 @@ func TestKeyGroupsForFileWithGroups(t *testing.T) { } func TestLoadConfigFileWithUnencryptedSuffix(t *testing.T) { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithSuffixParameters, t), "foobar", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithSuffixParameters, t), "/conf/path", "foobar", nil) assert.Nil(t, err) assert.Equal(t, "_unencrypted", conf.UnencryptedSuffix) } func TestLoadConfigFileWithEncryptedSuffix(t *testing.T) { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithSuffixParameters, t), "barfoo", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithSuffixParameters, t), "/conf/path", "barfoo", nil) assert.Nil(t, err) assert.Equal(t, "_enc", conf.EncryptedSuffix) } func TestLoadConfigFileWithUnencryptedRegex(t *testing.T) { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "barbar", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "/conf/path", "barbar", nil) assert.Equal(t, nil, err) assert.Equal(t, "^dec:", conf.UnencryptedRegex) } func TestLoadConfigFileWithEncryptedRegex(t *testing.T) { - conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "barbar", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "/conf/path", "barbar", nil) assert.Equal(t, nil, err) assert.Equal(t, "^enc:", conf.EncryptedRegex) } func TestLoadConfigFileWithInvalidParameters(t *testing.T) { - _, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithInvalidParameters, t), "foobar", nil) + _, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithInvalidParameters, t), "/conf/path", "foobar", nil) + assert.NotNil(t, err) +} + +func TestLoadConfigFileWithAmbiguousPath(t *testing.T) { + config := parseConfigFile(sampleConfigWithAmbiguousPath, t) + _, err := parseCreationRuleForFile(config, "/foo/config", "/foo/foo/bar", nil) + assert.Nil(t, err) + _, err = parseCreationRuleForFile(config, "/foo/config", "/foo/fuu/bar", nil) assert.NotNil(t, err) } From 75265f0dc3dcc50231f2c1284923316e283e8dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damien=20L=C3=A9ger?= Date: Tue, 20 Apr 2021 11:36:08 +0200 Subject: [PATCH 2/6] fix missing argument in encrypting with age section (#860) Signed-off-by: Carsten Skov --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f94e69795..852e06c09 100644 --- a/README.rst +++ b/README.rst @@ -189,7 +189,7 @@ the ``--age`` option or the **SOPS_AGE_RECIPIENTS** environment variable: .. code:: bash - $ sops --age age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw test.yaml > test.enc.yaml + $ sops --encrypt --age age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw test.yaml > test.enc.yaml When decrypting a file with the corresponding identity, sops will look for a text file name ``keys.txt`` located in a ``sops`` subdirectory of your user From 596ceac84b281600591cdec63890249ca81f25e7 Mon Sep 17 00:00:00 2001 From: Alexey Odinokov Date: Fri, 23 Apr 2021 00:51:02 -0500 Subject: [PATCH 3/6] Correct path to age keys.txt in documentation (#861) corrected path to keys based on the real behavior. see [1] [1] https://github.com/mozilla/sops/blob/master/age/keysource.go#L108 Signed-off-by: Carsten Skov --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 852e06c09..795318db6 100644 --- a/README.rst +++ b/README.rst @@ -193,9 +193,9 @@ the ``--age`` option or the **SOPS_AGE_RECIPIENTS** environment variable: When decrypting a file with the corresponding identity, sops will look for a text file name ``keys.txt`` located in a ``sops`` subdirectory of your user -configuration directory. On Linux, this would be ``$XDG_CONFIG_HOME/sops/keys.txt``. -On macOS, this would be ``$HOME/Library/Application Support/sops/keys.txt``. On -Windows, this would be ``%AppData%\sops\keys.txt``. You can specify the location +configuration directory. On Linux, this would be ``$XDG_CONFIG_HOME/sops/age/keys.txt``. +On macOS, this would be ``$HOME/Library/Application Support/sops/age/keys.txt``. On +Windows, this would be ``%AppData%\sops\age\keys.txt``. You can specify the location of this file manually by setting the environment variable **SOPS_AGE_KEY_FILE**. The contents of this key file should be a list of age X25519 identities, one From aed509670658f4753c7908771d45d2e6c80f113f Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 8 May 2021 19:43:55 +0200 Subject: [PATCH 4/6] Prevent comment duplication. (#866) Signed-off-by: Carsten Skov --- stores/yaml/store.go | 2 +- stores/yaml/store_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/stores/yaml/store.go b/stores/yaml/store.go index a3ae2fb46..d9e78b528 100644 --- a/stores/yaml/store.go +++ b/stores/yaml/store.go @@ -70,7 +70,7 @@ func (store Store) nodeToTreeValue(node *yaml.Node, commentsWereHandled bool) (i return result, nil case yaml.MappingNode: branch := make(sops.TreeBranch, 0) - return store.appendYamlNodeToTreeBranch(node, branch, false) + return store.appendYamlNodeToTreeBranch(node, branch, commentsWereHandled) case yaml.ScalarNode: var result interface{} node.Decode(&result) diff --git a/stores/yaml/store_test.go b/stores/yaml/store_test.go index be6b90eb2..05249f4a5 100644 --- a/stores/yaml/store_test.go +++ b/stores/yaml/store_test.go @@ -91,6 +91,26 @@ var COMMENT_5 = []byte(`# foo key: value `) +// The following is a regression test for https://github.com/mozilla/sops/issues/865 +var COMMENT_6 = []byte(`a: + - a + # I no longer get duplicated + - {} +`) + +var COMMENT_6_BRANCHES = sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "a", + Value: []interface{}{ + "a", + sops.Comment{" I no longer get duplicated"}, + sops.TreeBranch{}, + }, + }, + }, +} + func TestUnmarshalMetadataFromNonSOPSFile(t *testing.T) { data := []byte(`hello: 2`) _, err := (&Store{}).LoadEncryptedFile(data) @@ -178,6 +198,16 @@ func TestEmpty2(t *testing.T) { } */ +func TestComment6(t *testing.T) { + branches, err := (&Store{}).LoadPlainFile(COMMENT_6) + assert.Nil(t, err) + assert.Equal(t, COMMENT_6_BRANCHES, branches) + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + assert.Equal(t, string(COMMENT_6), string(bytes)) + assert.Equal(t, COMMENT_6, bytes) +} + func TestEmitValue(t *testing.T) { // First iteration: load and store bytes, err := (&Store{}).EmitValue(BRANCHES[0]) From 8e48c378ea032d595a5f209523da3c1f023d2386 Mon Sep 17 00:00:00 2001 From: ikedam Date: Sat, 24 Jul 2021 19:12:18 +0900 Subject: [PATCH 5/6] Use the key of release@mozilla.com for the unit test (#882) (#906) * `golang.org/x/crypto/openpgp` requires keys contain identity information. * A email address can have only a single key with identity information on keys.openpgp.org. Signed-off-by: Carsten Skov --- pgp/keysource_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pgp/keysource_test.go b/pgp/keysource_test.go index 0c10996d1..1e66056bc 100644 --- a/pgp/keysource_test.go +++ b/pgp/keysource_test.go @@ -44,7 +44,10 @@ func TestPGPKeySourceFromString(t *testing.T) { } func TestRetrievePGPKey(t *testing.T) { - fingerprint := "FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4" + // Requires a key available in https://keys.openpgp.org/ *with identity information* (that is, an email address). + // See https://keys.openpgp.org/about/faq#verify-multiple for details about identity information. + // We use the key of release@mozilla.com for here. + fingerprint := "14F26682D0916CDD81E37B6D61B7B526D98F0353" _, err := getKeyFromKeyServer(fingerprint) assert.NoError(t, err) } From 1c1b6189bc9e36408cf3815f976a05b693bcf48f Mon Sep 17 00:00:00 2001 From: Carsten Skov Date: Sat, 23 Sep 2023 08:23:51 +0200 Subject: [PATCH 6/6] Added example of having age recipients in .sops.yaml Fixed formatting for example of multiple age keys in .sops.yaml Added example of using updatekeys with age Apply suggestions from code review Apply suggestions from code review Co-authored-by: Felix Fontein Signed-off-by: Carsten Skov --- README.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.rst b/README.rst index 795318db6..0e853e8ff 100644 --- a/README.rst +++ b/README.rst @@ -204,7 +204,28 @@ identity will be tried in sequence until one is able to decrypt the data. Encrypting with SSH keys via age is not yet supported by sops. +A list of age recipients can be added to the ``.sops.yaml``: +.. code:: yaml + + creation_rules: + - age: >- + age1s3cqcks5genc6ru8chl0hkkd04zmxvczsvdxq99ekffe4gmvjpzsedk23c, + age1qe5lxzzeppw5k79vxn3872272sgy224g2nzqlzy3uljs84say3yqgvd0sw + +It is also possible to use ``updatekeys``, when adding or removing age recipients. For example: + +.. code:: sh + + $ sops updatekeys secret.enc.yaml + 2022/02/09 16:32:02 Syncing keys for file /iac/solution1/secret.enc.yaml + The following changes will be made to the file's groups: + Group 1 + age1s3cqcks5genc6ru8chl0hkkd04zmxvczsvdxq99ekffe4gmvjpzsedk23c + +++ age1qe5lxzzeppw5k79vxn3872272sgy224g2nzqlzy3uljs84say3yqgvd0sw + Is this okay? (y/n):y + 2022/02/09 16:32:04 File /iac/solution1/secret.enc.yaml synced with new keys + Encrypting using GCP KMS ~~~~~~~~~~~~~~~~~~~~~~~~ GCP KMS uses `Application Default Credentials