rpc: add

(squashes 51 commits, mainly to remove vendoring wackiness)

Makefile

@@ -6,7 +6,7 @@ NONVENDOR = ${shell find . -name '*.go' | grep -v vendor}
 BENCHDIRS= $(shell find . -name '*_test.go' | grep -v vendor | xargs grep '*testing.B' | cut -d: -f1 | xargs dirname | uniq)
 BENCH = .
 
-converge: $(shell find . -name '*.go')
+converge: $(shell find . -name '*.go') rpc/pb/root.pb.go rpc/pb/root.pb.gw.go
 	go build -ldflags="-s -w" .
 
 test: converge gotest samples/*.hcl samples/errors/*.hcl blackbox/*.sh
@@ -69,13 +69,16 @@ lint:
 
 vendor: ${NONVENDOR}
 	glide install --strip-vcs --strip-vendor --update-vendored
-	find vendor -not -name '*.go' -not -name '*.s' -not -name '*.pl' -not -name '*.c' -not -name LICENSE -type f -delete
+	make vendor-clean
 
 vendor-update: ${NOVENDOR}
 	glide update --strip-vcs --strip-vendor --update-vendored
-	find vendor -not -name '*.go' -not -name '*.s' -not -name '*.pl' -not -name '*.c' -not -name LICENSE -type f -delete
+	make vendor-clean
 
-xcompile: test
+vendor-clean: ${NOVENDOR}
+	find vendor -not -name '*.go' -not -name '*.s' -not -name '*.pl' -not -name '*.c' -not -name LICENSE -not -name '*.proto' -type f -delete
+
+xcompile: rpc/pb/root.pb.go rpc/pb/root.pb.gw.go test
 	@rm -rf build/
 	@mkdir -p build/
 	gox \
@@ -94,9 +97,27 @@ package: xcompile
     echo $$f; \
   done
 
+rpc/pb/root.pb.go: rpc/pb/root.proto
+	protoc -I rpc/pb \
+         -I vendor/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
+         --go_out=Mgoogle/api/annotations.proto=github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/google/api,plugins=grpc:rpc/pb \
+         rpc/pb/root.proto
+
+rpc/pb/root.pb.gw.go: rpc/pb/root.proto
+	protoc -I rpc/pb \
+         -I vendor/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
+         --grpc-gateway_out=logtostderr=true:rpc/pb \
+         rpc/pb/root.proto
+
+rpc/pb/root.swagger.json: rpc/pb/root.proto
+	protoc -I rpc/pb \
+         -I vendor/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
+         --swagger_out=logtostderr=true:rpc/pb \
+         rpc/pb/root.proto
+
 docs: docs_source/**/*
 	rm -rf docs || true
 	cd docs_source; make
 	mv docs_source/public docs
 
-.PHONY: test gotest vendor-update xcompile package samples/errors/*.hcl blackbox/*.sh lint bench license-check
+.PHONY: test gotest vendor-update vendor-clean xcompile package samples/errors/*.hcl blackbox/*.sh lint bench license-check

README.md

@@ -15,6 +15,9 @@ Converge is a configuration management tool.
         - [Server](#server)
             - [Module Hosting](#module-hosting)
             - [Binary Hosting](#binary-hosting)
+    - [Development](#development)
+        - [Tools](#tools)
+        - [RPC](#rpc)
     - [License](#license)
 
 <!-- markdown-toc end -->
@@ -185,6 +188,32 @@ bootstrap a new system without downloading the relevant version of converge over
 an external connection. It will be available at
 `http://your.host:8080/bootstrap/binary`.
 
+## Development
+
+### Tools
+
+For linting, you'll need:
+
+| tool | `go get` |
+| ==== | ======== |
+| `golint` | github.com/golang/lint/golint |
+| `go tool vet` | (built in) |
+| `gosimple` | honnef.co/go/simple/cmd/gosimple |
+| `unconvert` | github.com/mdempsky/unconvert |
+| `structcheck` | github.com/opennota/check/cmd/structcheck |
+| `varcheck` | github.com/opennota/check/cmd/varcheck |
+| `aligncheck` | github.com/opennota/check/cmd/aligncheck |
+| `gas` | github.com/HewlettPackard/gas |
+
+### RPC
+
+You'll need:
+
+- [Google's protobuf compiler](https://github.com/google/protobuf/releases), 3.0
+  or above.
+- The go protoc plugin: `go get -a github.com/golang/protobuf/protoc-gen-go`
+- The grpc gateway plugin(s): `go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger`
+
 ## License
 
 Converge is licensed under the Apache 2.0 license. See [LICENSE](LICENSE) for

apply/apply.go

@@ -30,81 +30,89 @@ var ErrTreeContainsErrors = errors.New("apply had errors, check graph")
 
 // Apply the actions in a Graph of resource.Tasks
 func Apply(ctx context.Context, in *graph.Graph) (*graph.Graph, error) {
-	var hasErrors error
+	return WithNotify(ctx, in, nil)
+}
 
-	out, err := in.Transform(ctx, func(id string, out *graph.Graph) error {
-		val := out.Get(id)
-		result, ok := val.(*plan.Result)
-		if !ok {
-			return fmt.Errorf("%s: could not get *plan.Result, was %T", id, val)
-		}
+// WithNotify is Apply, but with notification functions
+func WithNotify(ctx context.Context, in *graph.Graph, notify *graph.Notifier) (*graph.Graph, error) {
+	var hasErrors error
 
-		for _, depID := range graph.Targets(out.DownEdges(id)) {
-			dep, ok := out.Get(depID).(*Result)
+	out, err := in.Transform(
+		ctx,
+		notify.Transform(func(id string, out *graph.Graph) error {
+			val := out.Get(id)
+			result, ok := val.(*plan.Result)
 			if !ok {
-				return fmt.Errorf("graph walked out of order: %q before dependency %q", id, depID)
+				return fmt.Errorf("%s: could not get *plan.Result, was %T", id, val)
 			}
 
-			if err := dep.Error(); err != nil {
-				out.Add(
-					id,
-					&Result{
-						Ran:    false,
-						Status: &resource.Status{},
-						Plan:   result,
-						Err:    fmt.Errorf("error in dependency %q", depID),
-					},
-				)
-				// early return here after we set the signal error
-				hasErrors = ErrTreeContainsErrors
-				return nil
+			for _, depID := range graph.Targets(out.DownEdges(id)) {
+				dep, ok := out.Get(depID).(*Result)
+				if !ok {
+					return fmt.Errorf("graph walked out of order: %q before dependency %q", id, depID)
+				}
+
+				if err := dep.Error(); err != nil {
+					out.Add(
+						id,
+						&Result{
+							Ran:    false,
+							Status: &resource.Status{},
+							Plan:   result,
+							Err:    fmt.Errorf("error in dependency %q", depID),
+						},
+					)
+					// early return here after we set the signal error
+					hasErrors = ErrTreeContainsErrors
+					return nil
+				}
 			}
-		}
 
-		var newResult *Result
+			var newResult *Result
 
-		if result.Status.HasChanges() {
-			log.Printf("[DEBUG] applying %q\n", id)
+			if result.Status.HasChanges() {
+				log.Printf("[DEBUG] applying %q\n", id)
 
-			err := result.Task.Apply()
-			if err != nil {
-				err = errors.Wrapf(err, "error applying %s", id)
-			}
+				err := result.Task.Apply()
+				if err != nil {
+					err = errors.Wrapf(err, "error applying %s", id)
+				}
 
-			var status resource.TaskStatus
+				var status resource.TaskStatus
 
-			if err == nil {
-				status, err = result.Task.Check()
-				if err != nil {
-					err = errors.Wrapf(err, "error checking %s", id)
-				} else if status.HasChanges() {
-					err = fmt.Errorf("%s still needs to be changed after application", id)
+				if err == nil {
+					status, err = result.Task.Check()
+					if err != nil {
+						err = errors.Wrapf(err, "error checking %s", id)
+					} else if status.HasChanges() {
+						err = fmt.Errorf("%s still needs to be changed after application", id)
+					}
 				}
-			}
 
-			if err != nil {
-				hasErrors = ErrTreeContainsErrors
-			}
+				if err != nil {
+					hasErrors = ErrTreeContainsErrors
+				}
 
-			newResult = &Result{
-				Ran:    true,
-				Status: status,
-				Plan:   result,
-				Err:    err,
-			}
-		} else {
-			newResult = &Result{
-				Ran:    false,
-				Status: result.Status,
-				Plan:   result,
-				Err:    nil,
+				newResult = &Result{
+					Ran:    true,
+					Status: status,
+					Plan:   result,
+					Err:    err,
+				}
+			} else {
+				newResult = &Result{
+					Ran:    false,
+					Status: result.Status,
+					Plan:   result,
+					Err:    nil,
+				}
 			}
-		}
 
-		out.Add(id, newResult)
+			out.Add(id, newResult)
 
-		return nil
-	})
+			return nil
+		}),
+	)
 
 	if err != nil {
 		return out, err

blackbox/test_apply.sh

@@ -12,7 +12,7 @@ trap finish EXIT
 
 pushd $TMP
 
-$ROOT/converge apply -p filename=test.txt -p "message=x" $SOURCE
+$ROOT/converge apply --local -p filename=test.txt -p "message=x" $SOURCE
 
 if [ ! -f test.txt ]; then
     echo "test.txt doesn't exist"

blackbox/test_apply_remote.sh

@@ -2,13 +2,15 @@
 set -eo pipefail
 
 ROOT=$(pwd)
-SOURCE=${1:-http://localhost:8080/modules/basic.hcl}
+SOURCE=${1:-http://localhost:2694/api/v1/resources/modules/basic.hcl}
 
-$ROOT/converge server --root $ROOT/samples &
+$ROOT/converge server --no-token --root $ROOT/samples &
 PID=$!
 function finish {
     kill -2 $PID
 }
 trap finish EXIT
 
+sleep 0.5
+
 $ROOT/blackbox/test_apply.sh $SOURCE

blackbox/test_apply_sourcefile.sh

@@ -12,7 +12,7 @@ trap finish EXIT
 
 pushd $TMP
 
-$ROOT/converge apply -p "message=x" $SOURCE
+$ROOT/converge apply --local --rpc-addr=:8002 -p "message=x" $SOURCE
 
 if [ ! -f test.txt ]; then
     echo "test.txt doesn't exist"

blackbox/test_apply_sourcefile_remote.sh

@@ -2,13 +2,15 @@
 set -eo pipefail
 
 ROOT=$(pwd)
-SOURCE=${1:-http://localhost:8080/modules/sourceFile.hcl}
+SOURCE=${1:-http://localhost:2694/api/v1/resources/modules/sourceFile.hcl}
 
-$ROOT/converge server --root $ROOT/samples &
+$ROOT/converge server --no-token --root $ROOT/samples &
 PID=$!
 function finish {
     kill -2 $PID
 }
 trap finish EXIT
 
+sleep 0.5
+
 $ROOT/blackbox/test_apply.sh $SOURCE

blackbox/test_required_param.sh

@@ -13,7 +13,7 @@ pushd $TMP
 
 echo 'param "test" {}' > required_param.hcl
 
-if $ROOT/converge apply required_param.hcl; then
+if $ROOT/converge apply --local required_param.hcl; then
     echo "failed: apply without required param succeeded"
     exit 1
 else

blackbox/test_self_serve.sh

@@ -3,7 +3,7 @@ set -eo pipefail
 
 ROOT=$(pwd)
 
-$ROOT/converge server --root $ROOT/samples --self-serve &
+$ROOT/converge server --root $ROOT/samples --self-serve --no-token &
 PID=$!
 function finish {
     kill -2 $PID
@@ -12,7 +12,7 @@ trap finish EXIT
 
 sleep 0.5
 
-REMOTE_SUM=$(curl http://localhost:8080/bootstrap/binary | shasum | awk '{ print $1 }')
+REMOTE_SUM=$(curl http://localhost:2694/api/v1/resources/binary -H "Accept: text/plain" | shasum | awk '{ print $1 }')
 LOCAL_SUM=$(shasum $ROOT/converge | awk '{ print $1 }')
 
 if [[ "$REMOTE_SUM" == "$LOCAL_SUM" ]]; then

cmd/addrs.go

@@ -0,0 +1,21 @@
+// Copyright © 2016 Asteris, LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+const (
+	addrServer      = ":2693"
+	addrServerHTTP  = ":2694"
+	addrServerLocal = ":26930"
+)