Skip to content

Commit

Permalink
update cache store naming
Browse files Browse the repository at this point in the history
Signed-off-by: Jörn Friedrich Dreyer <[email protected]>
  • Loading branch information
butonic committed Mar 20, 2023
1 parent 1069ba0 commit 8cf3f31
Show file tree
Hide file tree
Showing 40 changed files with 191 additions and 261 deletions.
9 changes: 9 additions & 0 deletions changelog/unreleased/change-cache-configuration.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
Change: Updatet Cache Configuration

We updated all cache related environment vars to more closely follow the go micro naming pattern:
- `{service}_CACHE_STORE_TYPE` becomes `{service}_CACHE_STORE` or `{service}_PERSISTENT_STORE`
- `{service}_CACHE_STORE_ADDRESS(ES)` becomes `{service}_CACHE_STORE_NODES`
- The `mem` store implmentation name changes to `memory`
We introduced `redis-sentinel` as a store implementation.

https://github.com/owncloud/ocis/pull/5829
8 changes: 4 additions & 4 deletions ocis-pkg/store/store.go → ocis-pkg/cache/cache.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package store
package cache

import (
"context"
Expand All @@ -9,8 +9,8 @@ import (
"github.com/go-micro/plugins/v4/store/redis"
redisopts "github.com/go-redis/redis/v8"
"github.com/nats-io/nats.go"
"github.com/owncloud/ocis/v2/ocis-pkg/store/etcd"
"github.com/owncloud/ocis/v2/ocis-pkg/store/memory"
"github.com/owncloud/ocis/v2/ocis-pkg/cache/etcd"
"github.com/owncloud/ocis/v2/ocis-pkg/cache/memory"
"go-micro.dev/v4/logger"
"go-micro.dev/v4/store"
)
Expand All @@ -27,7 +27,7 @@ const (
TypeNatsJS = "nats-js"
)

// Create returns a configured key-value store
// Create returns a configured key-value micro store
//
// Each microservice (or whatever piece is using the store) should use the
// options available in the interface's operations to choose the right database
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
6 changes: 3 additions & 3 deletions ocis-pkg/store/options.go → ocis-pkg/cache/options.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package store
package cache

import (
"context"
Expand All @@ -9,7 +9,7 @@ import (

type typeContextKey struct{}

// Type determines the implementation:
// Store determines the implementation:
// - "memory", for a in-memory implementation, which is also the default if noone matches
// - "noop", for a noop store (it does nothing)
// - "etcd", for etcd
Expand All @@ -18,7 +18,7 @@ type typeContextKey struct{}
// - "redis-sentinel", for redis-sentinel
// - "ocmem", custom in-memory implementation, with fixed size and optimized prefix
// and suffix search
func Type(val string) store.Option {
func Store(val string) store.Option {
return func(o *store.Options) {
if o.Context == nil {
o.Context = context.Background()
Expand Down
2 changes: 1 addition & 1 deletion ocis-pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ type Config struct {

Tracing *shared.Tracing `yaml:"tracing"`
Log *shared.Log `yaml:"log"`
CacheStore *shared.CacheStore `yaml:"cache_store"`
Cache *shared.Cache `yaml:"cache"`
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
GRPCServiceTLS *shared.GRPCServiceTLS `yaml:"grpc_service_tls"`
HTTPServiceTLS shared.HTTPServiceTLS `yaml:"http_service_tls"`
Expand Down
6 changes: 3 additions & 3 deletions ocis-pkg/config/parser/parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ func EnsureDefaults(cfg *config.Config) {
if cfg.TokenManager == nil {
cfg.TokenManager = &shared.TokenManager{}
}
if cfg.CacheStore == nil {
cfg.CacheStore = &shared.CacheStore{}
if cfg.Cache == nil {
cfg.Cache = &shared.Cache{}
}
if cfg.GRPCClientTLS == nil {
cfg.GRPCClientTLS = &shared.GRPCClientTLS{}
Expand All @@ -70,7 +70,7 @@ func EnsureCommons(cfg *config.Config) {

cfg.Commons.Log = structs.CopyOrZeroValue(cfg.Log)
cfg.Commons.Tracing = structs.CopyOrZeroValue(cfg.Tracing)
cfg.Commons.CacheStore = structs.CopyOrZeroValue(cfg.CacheStore)
cfg.Commons.Cache = structs.CopyOrZeroValue(cfg.Cache)

if cfg.GRPCClientTLS != nil {
cfg.Commons.GRPCClientTLS = cfg.GRPCClientTLS
Expand Down
36 changes: 0 additions & 36 deletions ocis-pkg/roles/cache.go

This file was deleted.

55 changes: 0 additions & 55 deletions ocis-pkg/roles/cache_test.go

This file was deleted.

22 changes: 11 additions & 11 deletions ocis-pkg/roles/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,34 +4,34 @@ import (
"context"
"time"

"github.com/owncloud/ocis/v2/ocis-pkg/cache"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
store "github.com/owncloud/ocis/v2/ocis-pkg/store"
settingsmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0"
settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
microstore "go-micro.dev/v4/store"
"go-micro.dev/v4/store"
"google.golang.org/protobuf/encoding/protojson"
)

const (
cacheDatabase = "ocis-pkg"
cacheTableName = "ocis-pkg/roles"
cacheTableName = "roles"
cacheTTL = time.Hour
)

// Manager manages a cache of roles by fetching unknown roles from the settings.RoleService.
type Manager struct {
logger log.Logger
cache microstore.Store
roleCache store.Store
roleService settingssvc.RoleService
}

// NewManager returns a new instance of Manager.
func NewManager(o ...Option) Manager {
opts := newOptions(o...)

nStore := store.Create(opts.storeOptions...)
nStore := cache.Create(opts.storeOptions...)
return Manager{
cache: nStore,
roleCache: nStore,
roleService: opts.roleService,
}
}
Expand All @@ -42,7 +42,7 @@ func (m *Manager) List(ctx context.Context, roleIDs []string) []*settingsmsg.Bun
result := make([]*settingsmsg.Bundle, 0)
lookup := make([]string, 0)
for _, roleID := range roleIDs {
if records, err := m.cache.Read(roleID, microstore.ReadFrom(cacheDatabase, cacheTableName)); err != nil {
if records, err := m.roleCache.Read(roleID, store.ReadFrom(cacheDatabase, cacheTableName)); err != nil {
lookup = append(lookup, roleID)
} else {
role := &settingsmsg.Bundle{}
Expand Down Expand Up @@ -77,15 +77,15 @@ func (m *Manager) List(ctx context.Context, roleIDs []string) []*settingsmsg.Bun
}
for _, role := range res.Bundles {
jsonbytes, _ := protojson.Marshal(role)
record := &microstore.Record{
record := &store.Record{
Key: role.Id,
Value: jsonbytes,
Expiry: cacheTTL,
}
err := m.cache.Write(
err := m.roleCache.Write(
record,
microstore.WriteTo(cacheDatabase, cacheTableName),
microstore.WriteTTL(cacheTTL),
store.WriteTo(cacheDatabase, cacheTableName),
store.WriteTTL(cacheTTL),
)
if err != nil {
m.logger.Debug().Err(err).Msg("failed to cache roles")
Expand Down
16 changes: 8 additions & 8 deletions ocis-pkg/shared/shared_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,21 +55,21 @@ type HTTPServiceTLS struct {
Key string `yaml:"key" env:"OCIS_HTTP_TLS_KEY" desc:"Path/File name for the TLS certificate key (in PEM format) for the server certificate to use for the http services."`
}

type CacheStore struct {
Type string `yaml:"type" env:"OCIS_CACHE_STORE_TYPE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Addresses []string `yaml:"addresses" env:"OCIS_CACHE_STORE_ADDRESSES" desc:"A comma separated list of addresses to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."`
Database string `yaml:"database" env:"OCIS_CACHE_STORE_DATABASE" desc:"The database name the configured store should use."`
Table string `yaml:"table" env:"OCIS_CACHE_STORE_TABLE" desc:"The database table the store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_STORE_TTL" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h."`
Size int `yaml:"size" env:"OCIS_CACHE_STORE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured."`
type Cache struct {
Store string `yaml:"store" env:"OCIS_CACHE_STORE;OCIS_CACHE_STORE_TYPE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;OCIS_CACHE_STORE_ADDRESSES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
Database string `yaml:"database" env:"OCIS_CACHE_STORE_DATABASE" desc:"The database name the configured store should use."`
Table string `yaml:"table" env:"OCIS_CACHE_STORE_TABLE" desc:"The database table the store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_STORE_TTL" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h."`
Size int `yaml:"size" env:"OCIS_CACHE_STORE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured."`
}

// Commons holds configuration that are common to all extensions. Each extension can then decide whether
// to overwrite its values.
type Commons struct {
Log *Log `yaml:"log"`
Tracing *Tracing `yaml:"tracing"`
CacheStore *CacheStore `yaml:"cache_store"`
Cache *Cache `yaml:"cache"`
GRPCClientTLS *GRPCClientTLS `yaml:"grpc_client_tls"`
GRPCServiceTLS *GRPCServiceTLS `yaml:"grpc_service_tls"`
HTTPServiceTLS HTTPServiceTLS `yaml:"http_service_tls"`
Expand Down
20 changes: 10 additions & 10 deletions services/eventhistory/pkg/command/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,17 @@ import (

"github.com/cs3org/reva/v2/pkg/events/stream"
"github.com/oklog/run"
"github.com/owncloud/ocis/v2/ocis-pkg/cache"
"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
ogrpc "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
"github.com/owncloud/ocis/v2/ocis-pkg/store"
"github.com/owncloud/ocis/v2/ocis-pkg/version"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config/parser"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/logging"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/metrics"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/server/grpc"
"github.com/urfave/cli/v2"
microstore "go-micro.dev/v4/store"
"go-micro.dev/v4/store"
)

// Server is the entrypoint for the server command.
Expand Down Expand Up @@ -55,13 +55,13 @@ func Server(cfg *config.Config) *cli.Command {
return err
}

st := store.Create(
store.Type(cfg.Store.Type),
store.TTL(cfg.Store.RecordExpiry),
store.Size(cfg.Store.Size),
microstore.Nodes(cfg.Store.Addresses...),
microstore.Database(cfg.Store.Database),
microstore.Table(cfg.Store.Table),
st := cache.Create(
cache.Store(cfg.Store.Store),
cache.TTL(cfg.Store.RecordExpiry),
cache.Size(cfg.Store.Size),
store.Nodes(cfg.Store.Nodes...),
store.Database(cfg.Store.Database),
store.Table(cfg.Store.Table),
)

service := grpc.NewService(
Expand All @@ -73,7 +73,7 @@ func Server(cfg *config.Config) *cli.Command {
grpc.Address(cfg.GRPC.Addr),
grpc.Metrics(metrics),
grpc.Consumer(consumer),
grpc.Store(st),
grpc.Persistence(st),
)

gr.Add(service.Run, func(err error) {
Expand Down
4 changes: 2 additions & 2 deletions services/eventhistory/pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ type GRPCConfig struct {

// Store configures the store to use
type Store struct {
Type string `yaml:"type" env:"OCIS_PERSISTENT_STORE_TYPE;EVENTHISTORY_STORE_TYPE" desc:"The type of the eventhistory store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Addresses []string `yaml:"addresses" env:"OCIS_PERSISTENT_STORE_ADDRESSES;EVENTHISTORY_STORE_ADDRESSES" desc:"A comma separated list of addresses to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."`
Store string `yaml:"store" env:"OCIS_PERSISTENT_STORE;EVENTHISTORY_STORE;OCIS_PERSISTENT_STORE_TYPE;EVENTHISTORY_STORE_TYPE" desc:"The type of the eventhistory store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_PERSISTENT_STORE_NODES;EVENTHISTORY_STORE_ADDRESSES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
Database string `yaml:"database" env:"EVENTHISTORY_STORE_DATABASE" desc:"The database name the configured store should use."`
Table string `yaml:"table" env:"EVENTHISTORY_STORE_TABLE" desc:"The database table the store should use."`
RecordExpiry time.Duration `yaml:"record_expiry" env:"EVENTHISTORY_RECORD_EXPIRY" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '336h' (2 weeks)."`
Expand Down
4 changes: 3 additions & 1 deletion services/eventhistory/pkg/config/defaults/defaultconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ func DefaultConfig() *config.Config {
EnableTLS: false,
},
Store: config.Store{
Type: "memory",
Store: "memory",
Database: "eventhistory",
Table: "events",
RecordExpiry: 336 * time.Hour,
},
GRPC: config.GRPCConfig{
Expand Down
26 changes: 13 additions & 13 deletions services/eventhistory/pkg/server/grpc/option.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,16 @@ type Option func(o *Options)

// Options defines the available options for this package.
type Options struct {
Name string
Address string
Logger log.Logger
Context context.Context
Config *config.Config
Metrics *metrics.Metrics
Namespace string
Flags []cli.Flag
Store store.Store
Consumer events.Consumer
Name string
Address string
Logger log.Logger
Context context.Context
Config *config.Config
Metrics *metrics.Metrics
Namespace string
Flags []cli.Flag
Persistence store.Store
Consumer events.Consumer
}

// newOptions initializes the available default options.
Expand Down Expand Up @@ -95,10 +95,10 @@ func Flags(flags []cli.Flag) Option {
}
}

// Store provides a function to configure the store
func Store(store store.Store) Option {
// Persistence provides a function to configure the store
func Persistence(store store.Store) Option {
return func(o *Options) {
o.Store = store
o.Persistence = store
}
}

Expand Down
Loading

0 comments on commit 8cf3f31

Please sign in to comment.