Skip to content

Commit

Permalink
Implement sops publish command (getsops#473)
Browse files Browse the repository at this point in the history
* Implement `sops publish` command

Publishes a file to a pre-configured destination (this lives in the sops
config file). Additionally, support re-encryption rules that work
just like the creation rules. Initial support for S3/GCS.

This is a part of the sops-workspace v2.0 project

Includes the addition of a new dependency:
  github.com/googleapis/gax-go/v2

* code review changes; support global --verbose flag

* Switch to recreation_rule with full support

Reencryption rule is now recreation rule and supports everything that a
creation rule does. Now, when you load a config for a file, you load
either the creation rule or the destination rule. I'm not sure about
this style long term, but it allows for support to be added for the
recreation rules without a bigger refactor of how the config file works.

* split loadForFileFromBytes into two functions

remove branching based on destination rule or not, create one for
creation rules and one for destination rules

* pretty diff for keygroup updates in sops publish
  • Loading branch information
ajvb authored Jun 27, 2019
1 parent d61906a commit ebd153f
Show file tree
Hide file tree
Showing 38 changed files with 1,972 additions and 145 deletions.
25 changes: 25 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -791,6 +791,31 @@ By default ``sops`` just dumps all the output to the standard output. We can use
Beware using both ``--in-place`` and ``--output`` flags will result in an error.
Using the publish command
~~~~~~~~~~~~~~~~~~~~~~~~~
``sops publish $file`` publishes a file to a pre-configured destination (this lives in the sops
config file). Additionally, support re-encryption rules that work just like the creation rules.
This command requires a ``.sops.yaml`` configuration file. Below is an example:
.. code:: yaml
destination_rules:
- s3_bucket: "sops-secrets"
path_regex: s3/*
recreation_rule:
pgp: F69E4901EDBAD2D1753F8C67A64535C4163FB307
- gcs_bucket: "sops-secrets"
path_regex: gcs/*
recreation_rule:
pgp: F69E4901EDBAD2D1753F8C67A64535C4163FB307
The above configuration will place all files under ``s3/*`` into the S3 bucket ``sops-secrets`` and
will place all files under ``gcs/*`` into the GCS bucket ``sops-secrets``. As well, it will decrypt
these files and re-encrypt them using the ``F69E4901EDBAD2D1753F8C67A64535C4163FB307`` pgp key.
You would deploy a file to S3 with a command like: ``sops publish s3/app.yaml``
Important information on types
------------------------------
Expand Down
64 changes: 64 additions & 0 deletions cmd/sops/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@ import (
"strings"
"time"

"github.com/fatih/color"
wordwrap "github.com/mitchellh/go-wordwrap"
"go.mozilla.org/sops"
"go.mozilla.org/sops/cmd/sops/codes"
"go.mozilla.org/sops/keys"
"go.mozilla.org/sops/keyservice"
"go.mozilla.org/sops/kms"
"go.mozilla.org/sops/stores/dotenv"
Expand Down Expand Up @@ -339,3 +341,65 @@ func RecoverDataKeyFromBuggyKMS(opts GenericDecryptOpts, tree *sops.Tree) []byte

return nil
}

type Diff struct {
Common []keys.MasterKey
Added []keys.MasterKey
Removed []keys.MasterKey
}

func max(a, b int) int {
if a > b {
return a
}
return b
}

func DiffKeyGroups(ours, theirs []sops.KeyGroup) []Diff {
var diffs []Diff
for i := 0; i < max(len(ours), len(theirs)); i++ {
var diff Diff
var ourGroup, theirGroup sops.KeyGroup
if len(ours) > i {
ourGroup = ours[i]
}
if len(theirs) > i {
theirGroup = theirs[i]
}
ourKeys := make(map[string]struct{})
theirKeys := make(map[string]struct{})
for _, key := range ourGroup {
ourKeys[key.ToString()] = struct{}{}
}
for _, key := range theirGroup {
if _, ok := ourKeys[key.ToString()]; ok {
diff.Common = append(diff.Common, key)
} else {
diff.Added = append(diff.Added, key)
}
theirKeys[key.ToString()] = struct{}{}
}
for _, key := range ourGroup {
if _, ok := theirKeys[key.ToString()]; !ok {
diff.Removed = append(diff.Removed, key)
}
}
diffs = append(diffs, diff)
}
return diffs
}

func PrettyPrintDiffs(diffs []Diff) {
for i, diff := range diffs {
color.New(color.Underline).Printf("Group %d\n", i+1)
for _, c := range diff.Common {
fmt.Printf(" %s\n", c.ToString())
}
for _, c := range diff.Added {
color.New(color.FgGreen).Printf("+++ %s\n", c.ToString())
}
for _, c := range diff.Removed {
color.New(color.FgRed).Printf("--- %s\n", c.ToString())
}
}
}
46 changes: 45 additions & 1 deletion cmd/sops/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"go.mozilla.org/sops/cmd/sops/common"
"go.mozilla.org/sops/cmd/sops/subcommand/groups"
keyservicecmd "go.mozilla.org/sops/cmd/sops/subcommand/keyservice"
publishcmd "go.mozilla.org/sops/cmd/sops/subcommand/publish"
"go.mozilla.org/sops/cmd/sops/subcommand/updatekeys"
"go.mozilla.org/sops/config"
"go.mozilla.org/sops/gcpkms"
Expand Down Expand Up @@ -105,6 +106,49 @@ func main() {
For more information, see the README at github.com/mozilla/sops`
app.EnableBashCompletion = true
app.Commands = []cli.Command{
{
Name: "publish",
Usage: "Publish sops file to a configured destination",
ArgsUsage: `file`,
Flags: append([]cli.Flag{
cli.BoolFlag{
Name: "yes, y",
Usage: `pre-approve all changes and run non-interactively`,
},
cli.BoolFlag{
Name: "verbose",
Usage: "Enable verbose logging output",
},
}, keyserviceFlags...),
Action: func(c *cli.Context) error {
if c.Bool("verbose") || c.GlobalBool("verbose") {
logging.SetLevel(logrus.DebugLevel)
}
configPath, err := config.FindConfigFile(".")
if err != nil {
return common.NewExitError(err, codes.ErrorGeneric)
}
if c.NArg() < 1 {
return common.NewExitError("Error: no file specified", codes.NoFileSpecified)
}
fileName := c.Args()[0]
inputStore := inputStore(c, fileName)
err = publishcmd.Run(publishcmd.Opts{
ConfigPath: configPath,
InputPath: fileName,
InputStore: inputStore,
Cipher: aes.NewCipher(),
KeyServices: keyservices(c),
Interactive: !c.Bool("yes"),
})
if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil {
return cliErr
} else if err != nil {
return common.NewExitError(err, codes.ErrorGeneric)
}
return nil
},
},
{
Name: "keyservice",
Usage: "start a SOPS key service server",
Expand All @@ -129,7 +173,7 @@ func main() {
},
},
Action: func(c *cli.Context) error {
if c.Bool("verbose") {
if c.Bool("verbose") || c.GlobalBool("verbose") {
logging.SetLevel(logrus.DebugLevel)
}
err := keyservicecmd.Run(keyservicecmd.Opts{
Expand Down
151 changes: 151 additions & 0 deletions cmd/sops/subcommand/publish/publish.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
package publish

import (
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"

"go.mozilla.org/sops"
"go.mozilla.org/sops/cmd/sops/codes"
"go.mozilla.org/sops/cmd/sops/common"
"go.mozilla.org/sops/config"
"go.mozilla.org/sops/keyservice"
"go.mozilla.org/sops/logging"
"go.mozilla.org/sops/version"

"github.com/sirupsen/logrus"
)

var log *logrus.Logger

func init() {
log = logging.NewLogger("PUBLISH")
}

type Opts struct {
Interactive bool
Cipher sops.Cipher
ConfigPath string
InputPath string
KeyServices []keyservice.KeyServiceClient
InputStore sops.Store
}

func Run(opts Opts) error {
var fileContents []byte
path, err := filepath.Abs(opts.InputPath)
if err != nil {
return err
}
info, err := os.Stat(path)
if err != nil {
return err
}
if info.IsDir() {
return fmt.Errorf("can't operate on a directory")
}
_, fileName := filepath.Split(path)

conf, err := config.LoadDestinationRuleForFile(opts.ConfigPath, opts.InputPath, make(map[string]*string))
if err != nil {
return err
}
if conf.Destination == nil {
return errors.New("no destination configured for this file")
}

// Check that this is a sops-encrypted file
tree, err := common.LoadEncryptedFile(opts.InputStore, opts.InputPath)
if err != nil {
return err
}

// Re-encrypt if settings exist to do so
if len(conf.KeyGroups[0]) != 0 {
log.Debug("Re-encrypting tree before publishing")
_, err = common.DecryptTree(common.DecryptTreeOpts{
Cipher: opts.Cipher,
IgnoreMac: false,
Tree: tree,
KeyServices: opts.KeyServices,
})
if err != nil {
return err
}

diffs := common.DiffKeyGroups(tree.Metadata.KeyGroups, conf.KeyGroups)
keysWillChange := false
for _, diff := range diffs {
if len(diff.Added) > 0 || len(diff.Removed) > 0 {
keysWillChange = true
}
}
if keysWillChange {
fmt.Printf("The following changes will be made to the file's key groups:\n")
common.PrettyPrintDiffs(diffs)
}

tree.Metadata = sops.Metadata{
KeyGroups: conf.KeyGroups,
UnencryptedSuffix: conf.UnencryptedSuffix,
EncryptedSuffix: conf.EncryptedSuffix,
Version: version.Version,
ShamirThreshold: conf.ShamirThreshold,
}

dataKey, errs := tree.GenerateDataKeyWithKeyServices(opts.KeyServices)
if len(errs) > 0 {
err = fmt.Errorf("Could not generate data key: %s", errs)
return err
}

err = common.EncryptTree(common.EncryptTreeOpts{
DataKey: dataKey,
Tree: tree,
Cipher: opts.Cipher,
})
if err != nil {
return err
}

fileContents, err = opts.InputStore.EmitEncryptedFile(*tree)
if err != nil {
return common.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
}
} else {
fileContents, err = ioutil.ReadFile(path)
if err != nil {
return fmt.Errorf("could not read file: %s", err)
}
}

if opts.Interactive {
var response string
for response != "y" && response != "n" {
fmt.Printf("uploading %s to %s ? (y/n): ", path, conf.Destination.Path(fileName))
_, err := fmt.Scanln(&response)
if err != nil {
return err
}
}
if response == "n" {
return errors.New("Publish canceled")
}
}

err = conf.Destination.Upload(fileContents, fileName)
if err != nil {
return err
}

return nil
}

func min(a, b int) int {
if a < b {
return a
}
return b
}
Loading

0 comments on commit ebd153f

Please sign in to comment.