-
Notifications
You must be signed in to change notification settings - Fork 3.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Compactor: add per tenant compaction delete enabled flag (#6410)
* Add per tenant compaction delete enabled flag Signed-off-by: Michel Hollands <[email protected]> * Remove changes in wrong place Signed-off-by: Michel Hollands <[email protected]> * Add compactor deletion enabled field Signed-off-by: Michel Hollands <[email protected]> * Use limit in compactor Signed-off-by: Michel Hollands <[email protected]> * Use http middleware and add test Signed-off-by: Michel Hollands <[email protected]> * Fix lint issue Signed-off-by: Michel Hollands <[email protected]> * Add changelog Signed-off-by: Michel Hollands <[email protected]> * Revert to default setting if no override Signed-off-by: Michel Hollands <[email protected]> * Add default value command line option Signed-off-by: Michel Hollands <[email protected]> * Update the docs Signed-off-by: Michel Hollands <[email protected]> * Enable access to deletion API for integration test Signed-off-by: Michel Hollands <[email protected]> * Rename flag to allow_deletes Signed-off-by: Michel Hollands <[email protected]> * Update per review comments Signed-off-by: Michel Hollands <[email protected]>
- Loading branch information
1 parent
8ee0d62
commit b4e6c59
Showing
9 changed files
with
211 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
133 changes: 133 additions & 0 deletions
133
pkg/storage/stores/shipper/compactor/deletion/request_handler_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,133 @@ | ||
| package deletion | ||
|
|
||
| import ( | ||
| "net/http" | ||
| "net/http/httptest" | ||
| "path/filepath" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/prometheus/common/model" | ||
| "github.com/stretchr/testify/require" | ||
| "github.com/weaveworks/common/user" | ||
|
|
||
| "github.com/grafana/loki/pkg/storage/chunk/client/local" | ||
| "github.com/grafana/loki/pkg/storage/stores/shipper/storage" | ||
| "github.com/grafana/loki/pkg/validation" | ||
| ) | ||
|
|
||
| type retentionLimit struct { | ||
| compactorDeletionEnabled bool | ||
| retentionPeriod time.Duration | ||
| streamRetention []validation.StreamRetention | ||
| } | ||
|
|
||
| func (r retentionLimit) convertToValidationLimit() *validation.Limits { | ||
| return &validation.Limits{ | ||
| CompactorDeletionEnabled: r.compactorDeletionEnabled, | ||
| RetentionPeriod: model.Duration(r.retentionPeriod), | ||
| StreamRetention: r.streamRetention, | ||
| } | ||
| } | ||
|
|
||
| type fakeLimits struct { | ||
| defaultLimit retentionLimit | ||
| perTenant map[string]retentionLimit | ||
| } | ||
|
|
||
| func (f fakeLimits) RetentionPeriod(userID string) time.Duration { | ||
| return f.perTenant[userID].retentionPeriod | ||
| } | ||
|
|
||
| func (f fakeLimits) StreamRetention(userID string) []validation.StreamRetention { | ||
| return f.perTenant[userID].streamRetention | ||
| } | ||
|
|
||
| func (f fakeLimits) CompactorDeletionEnabled(userID string) bool { | ||
| return f.perTenant[userID].compactorDeletionEnabled | ||
| } | ||
|
|
||
| func (f fakeLimits) DefaultLimits() *validation.Limits { | ||
| return f.defaultLimit.convertToValidationLimit() | ||
| } | ||
|
|
||
| func (f fakeLimits) AllByUserID() map[string]*validation.Limits { | ||
| res := make(map[string]*validation.Limits) | ||
| for userID, ret := range f.perTenant { | ||
| res[userID] = ret.convertToValidationLimit() | ||
| } | ||
| return res | ||
| } | ||
|
|
||
| func TestDeleteRequestHandlerDeletionMiddleware(t *testing.T) { | ||
| // build the store | ||
| tempDir := t.TempDir() | ||
|
|
||
| workingDir := filepath.Join(tempDir, "working-dir") | ||
| objectStorePath := filepath.Join(tempDir, "object-store") | ||
|
|
||
| objectClient, err := local.NewFSObjectClient(local.FSConfig{ | ||
| Directory: objectStorePath, | ||
| }) | ||
| require.NoError(t, err) | ||
| testDeleteRequestsStore, err := NewDeleteStore(workingDir, storage.NewIndexStorageClient(objectClient, "")) | ||
| require.NoError(t, err) | ||
|
|
||
| // limits | ||
| fl := &fakeLimits{ | ||
| perTenant: map[string]retentionLimit{ | ||
| "1": {compactorDeletionEnabled: true}, | ||
| "2": {compactorDeletionEnabled: false}, | ||
| }, | ||
| } | ||
|
|
||
| // Setup handler | ||
| drh := NewDeleteRequestHandler(testDeleteRequestsStore, 10*time.Second, fl, nil) | ||
| middle := drh.deletionMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) | ||
|
|
||
| // User that has deletion enabled | ||
| req := httptest.NewRequest(http.MethodGet, "http://www.your-domain.com", nil) | ||
| req = req.WithContext(user.InjectOrgID(req.Context(), "1")) | ||
|
|
||
| res := httptest.NewRecorder() | ||
| middle.ServeHTTP(res, req) | ||
|
|
||
| require.Equal(t, http.StatusOK, res.Result().StatusCode) | ||
|
|
||
| // User that does not have deletion enabled | ||
| req = httptest.NewRequest(http.MethodGet, "http://www.your-domain.com", nil) | ||
| req = req.WithContext(user.InjectOrgID(req.Context(), "2")) | ||
|
|
||
| res = httptest.NewRecorder() | ||
| middle.ServeHTTP(res, req) | ||
|
|
||
| require.Equal(t, http.StatusForbidden, res.Result().StatusCode) | ||
|
|
||
| // User without override, this should use the default value which is false | ||
| req = httptest.NewRequest(http.MethodGet, "http://www.your-domain.com", nil) | ||
| req = req.WithContext(user.InjectOrgID(req.Context(), "3")) | ||
|
|
||
| res = httptest.NewRecorder() | ||
| middle.ServeHTTP(res, req) | ||
|
|
||
| require.Equal(t, http.StatusForbidden, res.Result().StatusCode) | ||
|
|
||
| // User without override, after the default value is set to true | ||
| fl.defaultLimit.compactorDeletionEnabled = true | ||
|
|
||
| req = httptest.NewRequest(http.MethodGet, "http://www.your-domain.com", nil) | ||
| req = req.WithContext(user.InjectOrgID(req.Context(), "3")) | ||
|
|
||
| res = httptest.NewRecorder() | ||
| middle.ServeHTTP(res, req) | ||
|
|
||
| require.Equal(t, http.StatusOK, res.Result().StatusCode) | ||
|
|
||
| // User header is not given | ||
| req = httptest.NewRequest(http.MethodGet, "http://www.your-domain.com", nil) | ||
|
|
||
| res = httptest.NewRecorder() | ||
| middle.ServeHTTP(res, req) | ||
|
|
||
| require.Equal(t, http.StatusBadRequest, res.Result().StatusCode) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters