Merge branch 'master' into getleafrange

* master: (94 commits)
  Complete TODO (#1632)
  fake_node_reader: Remove unused field (#1631)
  Parallelize VerifyMapLeavesResponse (#1615)
  Remove redundant root hash calculations (#1630)
  sequencer: Consolidate compact.Tree initialization (#1629)
  Remove unused NodeReader method (#1625)
  Fix bazel build (#1627)
  added/updated postgresql implementation of log_storage (#1571)
  Clean up compact.Tree tests (#1622)
  Remove old hash list format from compact.Tree (#1621)
  Mention protoc-gen-doc in README.md
  merkle: Add testonly package with golden hashes (#1620)
  compact: Simplify getting hashes in NewTreeWithState (#1618)
  compact: Implement Tree using Range (#1522)
  Guard verbose logging in merkle path code (#1604)
  Make MaskLeft of NodeID return an explicit copy. (#1612)
  Convert directly from Int.Bits() to NodeID (#1614)
  Couple of changes to make NodeIDs more frugal. (#1613)
  compact.Tree: Change semantic of adding leaves (#1592)
  MapHasher: Do not return error from HashLeaf (#1611)
  ...

.gitignore

@@ -13,6 +13,7 @@
 /ct_server
 /dump_tree
 /etcdiscover
+/licenses
 /loglb
 /maphammer
 /mapreplay

.golangci.yaml

@@ -12,6 +12,8 @@ linters-settings:
       - golang.org/x/net/context
       - github.com/gogo/protobuf/proto
       - github.com/google/certificate-transparency-go/trillian
+      - encoding/asn1
+      - crypto/x509
 
 linters:
   disable-all: true
@@ -27,5 +29,5 @@ linters:
     - deadcode
     - ineffassign
     - varcheck
-    # TODO(gbelvin): write license linter and commit to upstream. 
+    # TODO(gbelvin): write license linter and commit to upstream.
     # ./scripts/check_license.sh is run by ./scripts/presubmit.sh

.travis.yml

@@ -16,17 +16,21 @@ cache:
 
 
 env:
-  - WITH_COVERAGE=true
-  - GOFLAGS='-race'
-  - GOFLAGS='-race --tags=batched_queue'
-  - GOFLAGS='-race' WITH_ETCD=true
-  - GOFLAGS='-race --tags=pkcs11' WITH_PKCS11=true
-  - WITH_DOCKER_TESTS=true
+  - PRESUB_TESTS=true WITH_COVERAGE=true
+  - PRESUB_TESTS=true GOFLAGS='-race'
+  - PRESUB_TESTS=true GOFLAGS='-race --tags=batched_queue'
+  - PRESUB_TESTS=true GOFLAGS='-race' WITH_ETCD=true
+  - PRESUB_TESTS=true GOFLAGS='-race --tags=pkcs11' WITH_PKCS11=true
+  - INTEG_TESTS=true WITH_COVERAGE=true
+  - INTEG_TESTS=true GOFLAGS='-race'
+  - INTEG_TESTS=true GOFLAGS='-race --tags=batched_queue'
+  - INTEG_TESTS=true GOFLAGS='-race' WITH_ETCD=true
+  - INTEG_TESTS=true GOFLAGS='-race --tags=pkcs11' WITH_PKCS11=true
+  - BAZEL_TESTS=true
+  - DOCKER_TESTS=true
 
 matrix:
   fast_finish: true
-  allow_failures:
-    - env: WITH_DOCKER_TESTS=true
 
 services:
   - docker
@@ -61,24 +65,29 @@ install:
   - go get ${GOFLAGS} -d -t ./...
   - go get -d -t github.com/google/certificate-transparency-go/...
   - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.16.0
-  - go get github.com/golang/protobuf/proto
-  - go get github.com/golang/protobuf/protoc-gen-go
-  - go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
-  - go get github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc
-  - go get github.com/golang/mock/mockgen
-  - go get golang.org/x/tools/cmd/stringer
+  - go get
+      github.com/golang/mock/mockgen
+      github.com/golang/protobuf/proto
+      github.com/golang/protobuf/protoc-gen-go
+      github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
+      github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc
+      github.com/uber/prototool/cmd/prototool
+      golang.org/x/tools/cmd/stringer
   # install vendored etcd binary
-  - go install ./vendor/github.com/coreos/etcd/cmd/etcd
-  - go install ./vendor/github.com/coreos/etcd/cmd/etcdctl
+  - go install
+      ./vendor/github.com/coreos/etcd/cmd/etcd
+      ./vendor/github.com/coreos/etcd/cmd/etcdctl
   # install bazel
   - |
     (
-      BAZEL_VERSION='0.18.0'
-      URL="https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh"
-      wget -O install.sh ${URL}
-      chmod +x install.sh
-      ./install.sh --user
-      rm -f install.sh
+    if [[ "${BAZEL_TESTS}" == "true" ]]; then
+        BAZEL_VERSION='0.18.0'
+        URL="https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh"
+        wget -O install.sh ${URL}
+        chmod +x install.sh
+        ./install.sh --user
+        rm -f install.sh
+      fi
     )
 
 before_script:
@@ -90,30 +99,37 @@ script:
   - set -e
   - cd "$HOME/gopath/src/github.com/google/trillian"
   - export GO_TEST_TIMEOUT=20m
-  - ./scripts/presubmit.sh ${WITH_COVERAGE:+--coverage}
   - |
+    if [[ "${PRESUB_TESTS}" == "true" ]]; then
+      ./scripts/presubmit.sh ${WITH_COVERAGE:+--coverage}
       # Check re-generation didn't change anything. Skip protoc-generated files
       # because protoc is not deterministic when generating file descriptors.
       echo "Checking that generated files are the same as checked-in versions."
       git diff --exit-code -- ':!*.pb.go' ':!*_string.go'
+    fi
   - |
       if [[ "${WITH_ETCD}" == "true" ]]; then
         export ETCD_DIR="${GOPATH}/bin"
       fi
   - |
+    if [[ "${INTEG_TESTS}" == "true" ]]; then
       ./integration/integration_test.sh
       cd "$HOME/gopath/src/github.com/google/certificate-transparency-go"
       ./trillian/integration/integration_test.sh
       cd $HOME/gopath/src/github.com/google/trillian
       HAMMER_OPTS="--operations=50" ./integration/maphammer.sh 3
+    fi
   - |
       # TODO(RJPercival): Make docker-compose integration test work when PKCS#11
       # support is enabled. This requires running softhsm in a Docker container.
       # See https://github.com/rolandshoemaker/docker-hsm for an example.
-      if [[ "${WITH_DOCKER_TESTS}" == "true" ]]; then
+      if [[ "${DOCKER_TESTS}" == "true" ]]; then
         ./integration/docker_compose_integration_test.sh
       fi
-  - bazel --batch build //:*
+  - |
+      if [[ "${BAZEL_TESTS}" == "true" ]]; then
+        bazel --batch build //:*
+      fi
   - set +e
 
 after_success:

BUILD.bazel

@@ -18,8 +18,8 @@ proto_library(
     ],
     deps = [
         ":trillian_proto",
-        "@googleapi//google/api:annotations_proto",
-        "@googleapi//google/rpc:status_proto",
+        "@com_google_googleapis//google/api:annotations_proto",
+        "@com_google_googleapis//google/rpc:status_proto",
         "@com_google_protobuf//:field_mask_proto",
     ],
 )
@@ -32,8 +32,8 @@ proto_library(
     ],
     deps = [
         ":trillian_proto",
-        "@googleapi//google/api:annotations_proto",
-        "@googleapi//google/rpc:status_proto",
+        "@com_google_googleapis//google/api:annotations_proto",
+        "@com_google_googleapis//google/rpc:status_proto",
         "@com_google_protobuf//:api_proto",
         "@com_google_protobuf//:timestamp_proto",
     ],
@@ -43,9 +43,9 @@ proto_library(
 proto_library(
     name = "trillian_proto",
     srcs = [
-        "trillian.proto",
         "crypto/keyspb/keyspb.proto",
         "crypto/sigpb/sigpb.proto",
+        "trillian.proto",
     ],
     deps = [
         "@com_google_protobuf//:any_proto",
@@ -54,4 +54,3 @@ proto_library(
         "@com_google_protobuf//:timestamp_proto",
     ],
 )
-

CHANGELOG.md

@@ -4,6 +4,18 @@
 
 Not yet released; provisionally v2.0.0 (may change).
 
+### Add Profiling Flags to Binaries
+
+The `trillian_log_server`, `trillian_log_signer` and `trillian_map_server`
+binaries now have CPU and heap profiling flags. Profiling is off by default.
+For more details see the
+[Go Blog](https://blog.golang.org/profiling-go-programs).
+### Map performance tweaks
+
+The map mode has had some performance tweaks added:
+* A workaround for locking issues which affect the map when it's used in
+  single-transaction mode.
+
 ### Introduce BatchInclusionProof function
 
 Added a batch version of the Merkle Tree InclusionProof function.
@@ -70,7 +82,9 @@ if err != nil {
 }
 ```
 
-### Configurable number of connections for MySQL
+### MySQL changes
+
+#### Configurable number of connections for MySQL
 
 Two new flags have been added that limit connections to MySQL database servers:
 
@@ -82,6 +96,11 @@ database server will likely impose limits on the number of connections. The
 default limit on idle connections is controlled by
 [Go's `sql` package](https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns).
 
+#### Enfored no concurrent use of MySQL tx
+
+Concurrently using a single MySQL transaction can cause the driver to error
+out, so we now attempt to prevent this from happening.
+
 ### Removal of length limits for a tree's `display_name` and `description`
 
 Previously, these were restricted to 20 bytes and 200 bytes respectively. These
@@ -112,6 +131,7 @@ The [db\_server Docker image](examples/deployment/docker/db_server/Dockerfile)
 is now based on
 [the MySQL 5.7 image from the Google Cloud Marketplace](https://console.cloud.google.com/marketplace/details/google/mysql5),
 rather than the [official MySQL 5.7 image](https://hub.docker.com/_/mysql).
+This Dockerfile supercedes Dockerfile.db, which has been removed.
 
 There is now a [mysql.cnf file](examples/deployment/docker/db_server/mysql.cnf)
 alongside the Dockerfile that makes it easy to build the image with a custom
@@ -135,6 +155,8 @@ testing and experimental purposes:
 docker-compose -f examples/deployment/docker-compose.yml up
 ```
 
+Docker Compose v3.1 or higher is required.
+
 The Terraform, Kubernetes and Docker configuration files, as well as various
 scripts, all now use the same, consistently-named environment variables for
 MySQL-related data (e.g. `MYSQL_DATABASE`). The variable names are based on
@@ -150,7 +172,13 @@ Quota metrics with specs of the form `users/<user>/read` and
 `users/<user>/write` are no longer exported by the Trillian binaries (as they
 lead to excessive storage requirements for Trillian metrics).
 
-### Fix Operation Loop Hang
+### Resilience improvements in `log_signer`
+
+#### Add timeout to sequencing loop
+
+Added a timeout to the context in the sequencing loop, with a default of 60s.
+
+#### Fix Operation Loop Hang
 
 Resolved a bug that would hide errors and cause the `OperationLoop` to hang
 until process exit if any error occurred.
@@ -168,6 +196,11 @@ The CompactMerkleTree has been moved from `github.com/google/trillian/merkle` to
 A new powerful data structure named Compact Range has been added to the same
 package. It is a generalization of the previous compact Merkle tree structure.
 
+`AddLeaf*` methods of `compact.Tree` have been replaced with the corresponding
+`AppendLeaf*` methods, which do not report hashes of ephemeral nodes along the
+right border of the Merkle tree. The `CalculateRoot` method should be used in
+conjunction with appends if the caller needs to get those hashes.
+
 ### Storage API changes
 
 The internal storage API is modified so that the ReadOnlyTreeTX.ReadRevision and
@@ -179,6 +212,12 @@ The `SubtreeCache.GetNodeHash()` method is no longer exported.
 The memory storage provider has been refactored to make it more consistent with
 the other storage providers.
 
+The `LogMetadata.GetUnsequencedCounts()` method has been removed.
+
+`NodeReader.GetMerkleNodes` now must return `Node` objects in the same order as
+node IDs requested. Storage implementations known to us already adhere to this
+requirement.
+
 ### Maphammer improvements
 
 The maphammer test tool for the experimental Trillian Map has been enhanced.

CODEOWNERS

@@ -5,4 +5,5 @@
 # https://git-scm.com/docs/gitignore#_pattern_format
 
 # Default
-*.proto                    @Martin2112 @daviddrysdale @AlCutter
+/*.proto                         @Martin2112 @AlCutter
+/storage/storagepb/storage.proto @Martin2112 @AlCutter

README.md

@@ -118,8 +118,8 @@ running and configured to:
    --port=3306` connects OK)
  - not require a password for the `root` user
 
-You can then set up the [expected tables](storage/mysql/storage.sql) in a `test`
-database like so:
+You can then set up the [expected tables](storage/mysql/schema/storage.sql) in a
+`test` database like so:
 
 ```bash
 ./scripts/resetdb.sh
@@ -183,10 +183,9 @@ the original files; if you do, you'll need to install the prerequisites:
 
   - `mockgen` tool from https://github.com/golang/mock
   - `stringer` tool from https://golang.org/x/tools/cmd/stringer
-  - `protoc`, [Go support for protoc](https://github.com/golang/protobuf) and
-     [grpc-gateway](https://github.com/grpc-ecosystem/grpc-gateway) (see
-     documentation linked from the
-     [protobuf site](https://github.com/google/protobuf))
+  - `protoc`, [Go support for protoc](https://github.com/golang/protobuf),
+     [grpc-gateway](https://github.com/grpc-ecosystem/grpc-gateway) and
+     [protoc-gen-doc](https://github.com/pseudomuto/protoc-gen-doc).
   - protocol buffer definitions for standard Google APIs:
 
     ```bash
@@ -241,6 +240,11 @@ go install -ldflags "-X 'main.version=$(git describe --tags)' -X 'main.commit=$(
 cd -
 ```
 
+#### Install [prototool](https://github.com/uber/prototool#installation)
+```bash
+go get -u github.com/uber/prototool/cmd/prototool
+```
+
 #### Run code generation, build, test and linters
 ```bash
 ./scripts/presubmit.sh
@@ -249,6 +253,7 @@ cd -
 #### Or just run the linters alone
 ```bash
 golangci-lint run
+prototool lint
 ```
 
 

WORKSPACE

@@ -24,14 +24,22 @@ load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 protobuf_deps()
 
 http_archive(
-    name = "googleapi",
-    url = "https://github.com/googleapis/googleapis/archive/master.zip",
+    name = "com_google_googleapis",
     strip_prefix = "googleapis-master",
+    url = "https://github.com/googleapis/googleapis/archive/master.zip",
+)
+
+load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language")
+
+switched_rules_by_language(
+    name = "com_google_googleapis_imports",
+    go = True,
+    java = True,
+    python = True,
 )
 
 http_archive(
     name = "io_bazel_rules_go",
     strip_prefix = "rules_go-7d17d496a6b32f6a573c6c22e29c58204eddf3d4",
     urls = ["https://github.com/bazelbuild/rules_go/archive/7d17d496a6b32f6a573c6c22e29c58204eddf3d4.zip"],
 )
-