diff --git a/cmd/createtree/main.go b/cmd/createtree/main.go index d25bfec799..afd4200f8b 100644 --- a/cmd/createtree/main.go +++ b/cmd/createtree/main.go @@ -37,6 +37,7 @@ import ( "flag" "fmt" "os" + "time" "github.com/golang/glog" "github.com/golang/protobuf/ptypes" @@ -59,6 +60,7 @@ var ( signatureAlgorithm = flag.String("signature_algorithm", sigpb.DigitallySigned_RSA.String(), "Signature algorithm of the new tree") displayName = flag.String("display_name", "", "Display name of the new tree") description = flag.String("description", "", "Description of the new tree") + maxRootDuration = flag.Duration("max_root_duration", 0, "Interval after which a new signed root is produced despite no submissions; zero means never") privateKeyFormat = flag.String("private_key_format", "PrivateKey", "Type of private key to be used") pemKeyPath = flag.String("pem_key_path", "", "Path to the private key PEM file") @@ -72,6 +74,7 @@ var ( type createOpts struct { addr string treeState, treeType, hashStrategy, hashAlgorithm, sigAlgorithm, displayName, description string + maxRootDuration time.Duration privateKeyType, pemKeyPath, pemKeyPass string } @@ -130,14 +133,15 @@ func newRequest(opts *createOpts) (*trillian.CreateTreeRequest, error) { } tree := &trillian.Tree{ - TreeState: trillian.TreeState(ts), - TreeType: trillian.TreeType(tt), - HashStrategy: trillian.HashStrategy(hs), - HashAlgorithm: sigpb.DigitallySigned_HashAlgorithm(ha), - SignatureAlgorithm: sigpb.DigitallySigned_SignatureAlgorithm(sa), - DisplayName: opts.displayName, - Description: opts.description, - PrivateKey: pk, + TreeState: trillian.TreeState(ts), + TreeType: trillian.TreeType(tt), + HashStrategy: trillian.HashStrategy(hs), + HashAlgorithm: sigpb.DigitallySigned_HashAlgorithm(ha), + SignatureAlgorithm: sigpb.DigitallySigned_SignatureAlgorithm(sa), + DisplayName: opts.displayName, + Description: opts.description, + PrivateKey: pk, + MaxRootDurationMillis: opts.maxRootDuration.Nanoseconds() / int64(time.Millisecond), } return &trillian.CreateTreeRequest{Tree: tree}, nil } @@ -177,17 +181,18 @@ func newPK(opts *createOpts) (*any.Any, error) { func newOptsFromFlags() *createOpts { return &createOpts{ - addr: *adminServerAddr, - treeState: *treeState, - treeType: *treeType, - hashStrategy: *hashStrategy, - hashAlgorithm: *hashAlgorithm, - sigAlgorithm: *signatureAlgorithm, - displayName: *displayName, - description: *description, - privateKeyType: *privateKeyFormat, - pemKeyPath: *pemKeyPath, - pemKeyPass: *pemKeyPassword, + addr: *adminServerAddr, + treeState: *treeState, + treeType: *treeType, + hashStrategy: *hashStrategy, + hashAlgorithm: *hashAlgorithm, + sigAlgorithm: *signatureAlgorithm, + displayName: *displayName, + description: *description, + maxRootDuration: *maxRootDuration, + privateKeyType: *privateKeyFormat, + pemKeyPath: *pemKeyPath, + pemKeyPass: *pemKeyPassword, } } diff --git a/log/sequencer.go b/log/sequencer.go index 58b8804bd4..21e7d8827d 100644 --- a/log/sequencer.go +++ b/log/sequencer.go @@ -68,9 +68,6 @@ func createMetrics(mf monitoring.MetricFactory) { seqCommitLatency = mf.NewHistogram("sequencer_latency_commit", "Latency of commit part of sequencer batch operation in ms", logIDLabel) } -// TODO(daviddrysdale): Make this configurable -var maxRootDurationInterval = 12 * time.Hour - // TODO(Martin2112): Add admin support for safely changing params like guard window during operation // TODO(Martin2112): Add support for enabling and controlling sequencing as part of admin API @@ -89,6 +86,9 @@ type Sequencer struct { // sequencerGuardWindow is used to ensure entries newer than the guard window will not be // sequenced until they fall outside it. By default there is no guard window. sequencerGuardWindow time.Duration + // maxRootDurationInterval is used to ensure that a new signed log root is generated after a while, + // even if no entries have been added to the log. Zero duration disables this behavior. + maxRootDurationInterval time.Duration } // maxTreeDepth sets an upper limit on the size of Log trees. @@ -123,6 +123,13 @@ func (s *Sequencer) SetGuardWindow(sequencerGuardWindow time.Duration) { s.sequencerGuardWindow = sequencerGuardWindow } +// SetMaxRootDurationInterval changes the interval after which a log root is generated regardless of +// whether entries have been added to the log. The default is a zero interval, which +// disables the behavior. +func (s *Sequencer) SetMaxRootDurationInterval(interval time.Duration) { + s.maxRootDurationInterval = interval +} + // TODO: This currently doesn't use the batch api for fetching the required nodes. This // would be more efficient but requires refactoring. func (s Sequencer) buildMerkleTreeFromStorageAtRoot(ctx context.Context, root trillian.SignedLogRoot, tx storage.TreeTX) (*merkle.CompactMerkleTree, error) { @@ -270,7 +277,7 @@ func (s Sequencer) SequenceBatch(ctx context.Context, logID int64, limit int) (i if len(leaves) == 0 { nowNanos := s.timeSource.Now().UnixNano() interval := time.Duration(nowNanos - currentRoot.TimestampNanos) - if maxRootDurationInterval == 0 || interval < maxRootDurationInterval { + if s.maxRootDurationInterval == 0 || interval < s.maxRootDurationInterval { // We have nothing to integrate into the tree glog.V(1).Infof("No leaves sequenced in this signing operation.") return 0, tx.Commit() diff --git a/log/sequencer_test.go b/log/sequencer_test.go index cf5d34c9a9..6930f42bf2 100644 --- a/log/sequencer_test.go +++ b/log/sequencer_test.go @@ -335,14 +335,12 @@ func TestSequenceWithNothingQueuedNewRoot(t *testing.T) { }, } c, ctx := createTestContext(ctrl, params) - saved := maxRootDurationInterval - maxRootDurationInterval = 1 * time.Millisecond + c.sequencer.SetMaxRootDurationInterval(1 * time.Millisecond) leaves, err := c.sequencer.SequenceBatch(ctx, params.logID, 1) if leaves != 0 || err != nil { t.Errorf("SequenceBatch()=(%v,%v); want (0,nil)", leaves, err) } - maxRootDurationInterval = saved } // Tests that the guard interval is being passed to storage correctly. Actual operation of the diff --git a/server/admin/admin_server.go b/server/admin/admin_server.go index c305944282..a1ea8785eb 100644 --- a/server/admin/admin_server.go +++ b/server/admin/admin_server.go @@ -143,6 +143,10 @@ func (s *Server) CreateTree(ctx context.Context, request *trillian.CreateTreeReq tree.PublicKey = &keyspb.PublicKey{Der: publicKeyDER} } + if tree.MaxRootDurationMillis < 0 { + return nil, status.Error(codes.InvalidArgument, "the max root duration must be >= 0") + } + tx, err := s.registry.AdminStorage.Begin(ctx) if err != nil { return nil, err @@ -204,6 +208,8 @@ func applyUpdateMask(from, to *trillian.Tree, mask *field_mask.FieldMask) error to.Description = from.Description case "storage_settings": to.StorageSettings = from.StorageSettings + case "max_root_duration_millis": + to.MaxRootDurationMillis = from.MaxRootDurationMillis default: return status.Errorf(codes.InvalidArgument, "invalid update_mask path: %q", path) } diff --git a/server/admin/admin_server_test.go b/server/admin/admin_server_test.go index b227dedab0..f183c08098 100644 --- a/server/admin/admin_server_test.go +++ b/server/admin/admin_server_test.go @@ -314,6 +314,9 @@ func TestServer_CreateTree(t *testing.T) { keySignatureMismatch := validTree keySignatureMismatch.SignatureAlgorithm = sigpb.DigitallySigned_RSA + negRootDuration := validTree + negRootDuration.MaxRootDurationMillis = -1 + tests := []struct { desc string req *trillian.CreateTreeRequest @@ -340,6 +343,11 @@ func TestServer_CreateTree(t *testing.T) { req: &trillian.CreateTreeRequest{Tree: &omittedPrivateKey}, wantErr: true, }, + { + desc: "negativeMaxRootDuration", + req: &trillian.CreateTreeRequest{Tree: &negRootDuration}, + wantErr: true, + }, { desc: "privateKeySpec", req: &trillian.CreateTreeRequest{ @@ -477,7 +485,7 @@ func TestServer_CreateTree(t *testing.T) { reqCopy := proto.Clone(test.req).(*trillian.CreateTreeRequest) tree, err := s.CreateTree(ctx, reqCopy) if hasErr := err != nil; hasErr != test.wantErr { - t.Errorf("%v: CreateTree() = (_, %q), wantErr = %v", test.desc, err, test.wantErr) + t.Errorf("%v: CreateTree() = (_, %v), wantErr = %v", test.desc, err, test.wantErr) continue } else if hasErr { continue @@ -514,6 +522,7 @@ func TestServer_UpdateTree(t *testing.T) { existingTree.TreeId = 12345 existingTree.CreateTimeMillisSinceEpoch = 10 existingTree.UpdateTimeMillisSinceEpoch = 10 + existingTree.MaxRootDurationMillis = 1 // Any valid proto works here, the type doesn't matter for this test. settings, err := ptypes.MarshalAny(&keyspb.PEMKeyFile{}) @@ -523,12 +532,13 @@ func TestServer_UpdateTree(t *testing.T) { // successTree specifies changes in all rw fields successTree := &trillian.Tree{ - TreeState: trillian.TreeState_FROZEN, - DisplayName: "Brand New Tree Name", - Description: "Brand New Tree Desc", - StorageSettings: settings, + TreeState: trillian.TreeState_FROZEN, + DisplayName: "Brand New Tree Name", + Description: "Brand New Tree Desc", + StorageSettings: settings, + MaxRootDurationMillis: 2, } - successMask := &field_mask.FieldMask{Paths: []string{"tree_state", "display_name", "description", "storage_settings"}} + successMask := &field_mask.FieldMask{Paths: []string{"tree_state", "display_name", "description", "storage_settings", "max_root_duration_millis"}} successWant := existingTree successWant.TreeState = successTree.TreeState @@ -536,6 +546,7 @@ func TestServer_UpdateTree(t *testing.T) { successWant.Description = successTree.Description successWant.StorageSettings = successTree.StorageSettings successWant.PrivateKey = nil // redacted on responses + successWant.MaxRootDurationMillis = successTree.MaxRootDurationMillis tests := []struct { desc string diff --git a/server/sequencer_manager.go b/server/sequencer_manager.go index b8539c2ad6..d41f77f7b5 100644 --- a/server/sequencer_manager.go +++ b/server/sequencer_manager.go @@ -78,6 +78,7 @@ func (s *SequencerManager) ExecutePass(ctx context.Context, logID int64, info *L sequencer := log.NewSequencer( hasher, info.TimeSource, s.registry.LogStorage, signer, s.registry.MetricFactory, s.registry.QuotaManager) sequencer.SetGuardWindow(s.guardWindow) + sequencer.SetMaxRootDurationInterval(time.Duration(tree.MaxRootDurationMillis * int64(time.Millisecond))) leaves, err := sequencer.SequenceBatch(ctx, logID, info.BatchSize) if err != nil { diff --git a/storage/mysql/admin_storage.go b/storage/mysql/admin_storage.go index e5f1d0d189..2c8e20c260 100644 --- a/storage/mysql/admin_storage.go +++ b/storage/mysql/admin_storage.go @@ -46,7 +46,8 @@ const ( CreateTimeMillis, UpdateTimeMillis, PrivateKey, - PublicKey + PublicKey, + MaxRootDurationMillis FROM Trees` selectTreeByID = selectTrees + " WHERE TreeId = ?" ) @@ -156,7 +157,7 @@ func readTree(row row) (*trillian.Tree, error) { // Enums and Datetimes need an extra conversion step var treeState, treeType, hashStrategy, hashAlgorithm, signatureAlgorithm string - var createMillis, updateMillis int64 + var createMillis, updateMillis, maxRootDurationMillis int64 var displayName, description sql.NullString var privateKey, publicKey []byte err := row.Scan( @@ -172,6 +173,7 @@ func readTree(row row) (*trillian.Tree, error) { &updateMillis, &privateKey, &publicKey, + &maxRootDurationMillis, ) if err != nil { return nil, err @@ -222,6 +224,7 @@ func readTree(row row) (*trillian.Tree, error) { tree.CreateTimeMillisSinceEpoch = createMillis tree.UpdateTimeMillisSinceEpoch = updateMillis + tree.MaxRootDurationMillis = maxRootDurationMillis tree.PrivateKey = &any.Any{} if err := proto.Unmarshal(privateKey, tree.PrivateKey); err != nil { @@ -319,8 +322,9 @@ func (t *adminTX) CreateTree(ctx context.Context, tree *trillian.Tree) (*trillia CreateTimeMillis, UpdateTimeMillis, PrivateKey, - PublicKey) - VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + PublicKey, + MaxRootDurationMillis) + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) if err != nil { return nil, err } @@ -345,6 +349,7 @@ func (t *adminTX) CreateTree(ctx context.Context, tree *trillian.Tree) (*trillia newTree.UpdateTimeMillisSinceEpoch, privateKey, newTree.PublicKey.GetDer(), + newTree.MaxRootDurationMillis, ) if err != nil { return nil, err @@ -406,7 +411,7 @@ func (t *adminTX) UpdateTree(ctx context.Context, treeID int64, updateFunc func( stmt, err := t.tx.PrepareContext( ctx, `UPDATE Trees - SET TreeState = ?, DisplayName = ?, Description = ?, UpdateTimeMillis = ? + SET TreeState = ?, DisplayName = ?, Description = ?, UpdateTimeMillis = ?, MaxRootDurationMillis = ? WHERE TreeId = ?`) if err != nil { return nil, err @@ -419,6 +424,7 @@ func (t *adminTX) UpdateTree(ctx context.Context, treeID int64, updateFunc func( tree.DisplayName, tree.Description, tree.UpdateTimeMillisSinceEpoch, + tree.MaxRootDurationMillis, tree.TreeId); err != nil { return nil, err } diff --git a/storage/mysql/storage.sql b/storage/mysql/storage.sql index 5772ab5009..37924b6458 100644 --- a/storage/mysql/storage.sql +++ b/storage/mysql/storage.sql @@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS Trees( Description VARCHAR(200), CreateTimeMillis BIGINT NOT NULL, UpdateTimeMillis BIGINT NOT NULL, + MaxRootDurationMillis BIGINT NOT NULL, PrivateKey MEDIUMBLOB NOT NULL, PublicKey MEDIUMBLOB NOT NULL, PRIMARY KEY(TreeId) @@ -153,4 +154,3 @@ CREATE TABLE IF NOT EXISTS MapHead( UNIQUE INDEX TreeRevisionIdx(TreeId, MapRevision), FOREIGN KEY(TreeId) REFERENCES Trees(TreeId) ON DELETE CASCADE ); - diff --git a/storage/testonly/admin_storage_tester.go b/storage/testonly/admin_storage_tester.go index 99ad07fec4..961c296d51 100644 --- a/storage/testonly/admin_storage_tester.go +++ b/storage/testonly/admin_storage_tester.go @@ -88,6 +88,7 @@ var ( PublicKey: &keyspb.PublicKey{ Der: publicPEMToDER(ttestonly.DemoPublicKey), }, + MaxRootDurationMillis: 0, } // MapTree is a valid, MAP-type trillian.Tree for tests. @@ -105,6 +106,7 @@ var ( PublicKey: &keyspb.PublicKey{ Der: publicPEMToDER(ttestonly.DemoPublicKey), }, + MaxRootDurationMillis: 0, } ) diff --git a/storage/tree_validation.go b/storage/tree_validation.go index e218d1f2ec..20903c6840 100644 --- a/storage/tree_validation.go +++ b/storage/tree_validation.go @@ -108,6 +108,8 @@ func validateMutableTreeFields(tree *trillian.Tree) error { return errors.Errorf(errors.InvalidArgument, "display_name too big, max length is %v: %v", maxDisplayNameLength, tree.DisplayName) case len(tree.Description) > maxDescriptionLength: return errors.Errorf(errors.InvalidArgument, "description too big, max length is %v: %v", maxDescriptionLength, tree.Description) + case tree.MaxRootDurationMillis < 0: + return errors.Errorf(errors.InvalidArgument, "max_root_duration negative: %v", tree.MaxRootDurationMillis) } // Implementations may vary, so let's assume storage_settings is mutable. diff --git a/storage/tree_validation_test.go b/storage/tree_validation_test.go index 8410b87abb..dd75d42447 100644 --- a/storage/tree_validation_test.go +++ b/storage/tree_validation_test.go @@ -89,6 +89,9 @@ func TestValidateTreeForCreation(t *testing.T) { validSettings := newTree() validSettings.StorageSettings = settings + invalidRootDuration := newTree() + invalidRootDuration.MaxRootDurationMillis = -1 + tests := []struct { desc string tree *trillian.Tree @@ -191,6 +194,11 @@ func TestValidateTreeForCreation(t *testing.T) { desc: "validSettings", tree: validSettings, }, + { + desc: "invalidRootDuration", + tree: invalidRootDuration, + wantErr: true, + }, } for _, test := range tests { err := ValidateTreeForCreation(test.tree) @@ -239,6 +247,19 @@ func TestValidateTreeForUpdate(t *testing.T) { }, wantErr: true, }, + { + desc: "validRootDuration", + updatefn: func(tree *trillian.Tree) { + tree.MaxRootDurationMillis = 200 + }, + }, + { + desc: "invalidRootDuration", + updatefn: func(tree *trillian.Tree) { + tree.MaxRootDurationMillis = -200 + }, + wantErr: true, + }, // Changes on readonly fields { desc: "TreeId", @@ -334,14 +355,15 @@ func newTree() *trillian.Tree { } return &trillian.Tree{ - TreeState: trillian.TreeState_ACTIVE, - TreeType: trillian.TreeType_LOG, - HashStrategy: trillian.HashStrategy_RFC_6962, - HashAlgorithm: sigpb.DigitallySigned_SHA256, - SignatureAlgorithm: sigpb.DigitallySigned_ECDSA, - DisplayName: "Llamas Log", - Description: "Registry of publicly-owned llamas", - PrivateKey: privateKey, - PublicKey: &keyspb.PublicKey{Der: publicKeyPEM.Bytes}, + TreeState: trillian.TreeState_ACTIVE, + TreeType: trillian.TreeType_LOG, + HashStrategy: trillian.HashStrategy_RFC_6962, + HashAlgorithm: sigpb.DigitallySigned_SHA256, + SignatureAlgorithm: sigpb.DigitallySigned_ECDSA, + DisplayName: "Llamas Log", + Description: "Registry of publicly-owned llamas", + PrivateKey: privateKey, + PublicKey: &keyspb.PublicKey{Der: publicKeyPEM.Bytes}, + MaxRootDurationMillis: 1000, } } diff --git a/trillian.pb.go b/trillian.pb.go index 32aa9f82e2..28a7536ccf 100644 --- a/trillian.pb.go +++ b/trillian.pb.go @@ -168,6 +168,9 @@ type Tree struct { // The public key used for verifying tree heads and entry timestamps. // Readonly. PublicKey *keyspb.PublicKey `protobuf:"bytes,14,opt,name=public_key,json=publicKey" json:"public_key,omitempty"` + // Interval after which a new signed root is produced even if there have been + // no submission. If zero, this behavior is disabled. + MaxRootDurationMillis int64 `protobuf:"varint,15,opt,name=max_root_duration_millis,json=maxRootDurationMillis" json:"max_root_duration_millis,omitempty"` } func (m *Tree) Reset() { *m = Tree{} } @@ -266,6 +269,13 @@ func (m *Tree) GetPublicKey() *keyspb.PublicKey { return nil } +func (m *Tree) GetMaxRootDurationMillis() int64 { + if m != nil { + return m.MaxRootDurationMillis + } + return 0 +} + type SignedEntryTimestamp struct { TimestampNanos int64 `protobuf:"varint,1,opt,name=timestamp_nanos,json=timestampNanos" json:"timestamp_nanos,omitempty"` LogId int64 `protobuf:"varint,2,opt,name=log_id,json=logId" json:"log_id,omitempty"` diff --git a/trillian.proto b/trillian.proto index 6959844c37..ddcd7c19d6 100644 --- a/trillian.proto +++ b/trillian.proto @@ -144,6 +144,10 @@ message Tree { // The public key used for verifying tree heads and entry timestamps. // Readonly. keyspb.PublicKey public_key = 14; + + // Interval after which a new signed root is produced even if there have been + // no submission. If zero, this behavior is disabled. + int64 max_root_duration_millis = 15; } message SignedEntryTimestamp {