Do not discard comments at beginning of YAML documents

CHANGELOG.rst

@@ -1,6 +1,17 @@
 Changelog
 =========
 
+Develop branch
+--------------
+
+Changes:
+
+    * Comments at the beginning of YAML documents are now no longer discarded (#756)
+
+Bug fixes:
+
+    * Fix several problems with YAML comment handling described in #695 (#754)
+
 3.6.1
 -----
 Features:

functional-tests/res/comments.enc.yaml

@@ -1,12 +1,15 @@
+#ENC[AES256_GCM,data:IYA+b4ORDq8u9CBQolipWD4HRqoZyA==,iv:F8ldQqGng+WptHuBkFtjrGM+7sRZCsvd0FHq98lrpAE=,tag:ZHbLU9+CELinf5PhhuIzSQ==,type:comment]
 lorem: ENC[AES256_GCM,data:PhmSdTs=,iv:J5ugEWq6RfyNx+5zDXvcTdoQ18YYZkqesDED7LNzou4=,tag:0Qrom6J6aUnZMZzGz5XCxw==,type:str]
 #ENC[AES256_GCM,data:HiHCasVRzWUiFxKb3X/AcEeM,iv:bmNg+T91dqGk/CEtVH+FDC53osDCEPmWmJKpLyAU5OM=,tag:bTLDYxQSAfYDCBYccoUokQ==,type:comment]
 dolor: ENC[AES256_GCM,data:IgvT,iv:wtPNYbDTARFE810PH6ldOLzCDcAjkB/dzPsZjpgHcko=,tag:zwE8P+AwO1hrHkgF6pTbZw==,type:str]
 sops:
     kms: []
     gcp_kms: []
     azure_kv: []
-    lastmodified: '2017-08-16T03:41:16Z'
-    mac: ENC[AES256_GCM,data:3ngUnY2hkK6pkDbCeAnOHsi/M6bLnGk1vkd+EeGyN/efqJZmwH0+9hUdACNnwHzofIR6NbtCGZal+cSCuTGD4eDuqNV+LbwV1/EaaVZj9RktTNXq3STSXxfzYGoHV3NOMtBhq6sYhF0U72nunreCymm3QzOTylAa2HlmRs54axM=,iv:EMXphsMa+ELK8XXX3MDfFJe3jFgXzwCSwjxNR5ah14k=,tag:gakwLdPvwyihj+FkTG/2kQ==,type:str]
+    hc_vault: []
+    age: []
+    lastmodified: '2020-10-07T15:49:13Z'
+    mac: ENC[AES256_GCM,data:2dhyKdHYSynjXPwYrn9356wA7vRKw+T5qwBenI2vZrgthpQBOCQG4M6f7eeH3VLTxB4mN4CAchb25dsNRoGr6A38VruaSSAhPco3Rh4AlvKSvXuhgRnzZvNxE/bnHX1D4K5cdTb4FsJg/Ue1l7UcWrlrv1s3H3SwLHP/nf+suD0=,iv:6xBYURjjaQzlUOKOrs2NWOChiNFZVAGPJZQZ59MwX3o=,tag:uXD5VYme+c8eHcCc5TD2YA==,type:str]
     pgp:
     -   created_at: '2019-08-29T21:52:32Z'
         enc: |
@@ -24,4 +27,4 @@ sops:
             -----END PGP MESSAGE-----
         fp: FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4
     unencrypted_suffix: _unencrypted
-    version: 2.0.9
+    version: 3.6.1

functional-tests/res/comments.yaml

@@ -1,3 +1,4 @@
+# first comment in file
 lorem: ipsum
 # this-is-a-comment
 dolor: sit

functional-tests/res/comments_unencrypted_comments.yaml

@@ -1,3 +1,4 @@
+# first comment in file
 lorem: ENC[AES256_GCM,data:qVz4paM=,iv:0oGsaw71i3wZKmlyDl8uDhQT9XLvJt3oIyx514X44K8=,tag:acbMS613StWo1IVnKK+5uQ==,type:str]
 # this-is-a-comment
 dolor: ENC[AES256_GCM,data:21fI,iv:01LXdHZYwLTeyUB1YWIAM6KF8cPPVsw/RuQO+Ab4pgM=,tag:o1xnCIIoccWzdWxB2kZYKg==,type:str]

functional-tests/src/lib.rs

@@ -421,6 +421,7 @@ b: ba"#
             .output()
             .expect("Error running sops");
         assert!(output.status.success(), "SOPS didn't return successfully");
+        assert!(!String::from_utf8_lossy(&output.stdout).contains("first comment in file"), "Comment was not encrypted");
         assert!(!String::from_utf8_lossy(&output.stdout).contains("this-is-a-comment"), "Comment was not encrypted");
     }
 
@@ -446,6 +447,7 @@ b: ba"#
                     .output()
                     .expect("Error running sops");
         assert!(output.status.success(), "SOPS didn't return successfully");
+        assert!(String::from_utf8_lossy(&output.stdout).contains("first comment in file"), "Comment was not decrypted");
         assert!(String::from_utf8_lossy(&output.stdout).contains("this-is-a-comment"), "Comment was not decrypted");
     }
 
@@ -458,6 +460,7 @@ b: ba"#
                     .output()
                     .expect("Error running sops");
         assert!(output.status.success(), "SOPS didn't return successfully");
+        assert!(String::from_utf8_lossy(&output.stdout).contains("first comment in file"), "Comment was not decrypted");
         assert!(String::from_utf8_lossy(&output.stdout).contains("this-is-a-comment"), "Comment was not decrypted");
     }
 

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/lib/pq v1.2.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/go-wordwrap v1.0.0
-	github.com/mozilla-services/yaml v0.0.0-20201006214821-a130ed024dcf
+	github.com/mozilla-services/yaml v0.0.0-20201007153854-c369669a6625
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opencontainers/runc v0.1.1 // indirect

go.sum

@@ -213,6 +213,8 @@ github.com/mozilla-services/yaml v0.0.0-20191106225358-5c216288813c h1:yE1NxRAZA
 github.com/mozilla-services/yaml v0.0.0-20191106225358-5c216288813c/go.mod h1:Is/Ucts/yU/mWyGR8yELRoO46mejouKsJfQLAIfTR18=
 github.com/mozilla-services/yaml v0.0.0-20201006214821-a130ed024dcf h1:Lcffxn0MTcVD0v/kBCul62VDj0st4zxBSahSf03hGEc=
 github.com/mozilla-services/yaml v0.0.0-20201006214821-a130ed024dcf/go.mod h1:Is/Ucts/yU/mWyGR8yELRoO46mejouKsJfQLAIfTR18=
+github.com/mozilla-services/yaml v0.0.0-20201007153854-c369669a6625 h1:5IeGQzguDQ+EsTR5HE7tMYkZe09mqQ9cDypdKQEB5Kg=
+github.com/mozilla-services/yaml v0.0.0-20201007153854-c369669a6625/go.mod h1:Is/Ucts/yU/mWyGR8yELRoO46mejouKsJfQLAIfTR18=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

stores/yaml/store_test.go

@@ -17,6 +17,10 @@ key2: value2`)
 
 var BRANCHES = sops.TreeBranches{
 	sops.TreeBranch{
+		sops.TreeItem{
+			Key:   sops.Comment{" comment 0"},
+			Value: nil,
+		},
 		sops.TreeItem{
 			Key:   "key1",
 			Value: "value",
@@ -38,7 +42,8 @@ var BRANCHES = sops.TreeBranches{
 	},
 }
 
-var COMMENT_1 = []byte(`a:
+var COMMENT_1 = []byte(`# test
+a:
     b: null
     # foo
 `)
@@ -60,7 +65,9 @@ prometheus-node-exporter:
     - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/)
     - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$
 `)
-var COMMENT_3_OUT = []byte(`prometheus-node-exporter:
+var COMMENT_3_OUT = []byte(`## Configuration for prometheus-node-exporter subchart
+##
+prometheus-node-exporter:
     podLabels:
         ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards
         ##