From 560ca30ac47418ba0d5013e61449425d927af4ed Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Mon, 4 Dec 2023 18:22:35 +0530 Subject: [PATCH 01/29] Add AuditLogger --- .../overlord/http/OverlordResource.java | 45 +- .../overlord/http/OverlordResourceTest.java | 4 +- .../indexing/overlord/http/OverlordTest.java | 4 +- .../{AuditEntry.java => AuditEvent.java} | 124 ++--- .../org/apache/druid/audit/AuditManager.java | 58 ++- .../druid/audit/AuditManagerConfig.java | 12 +- .../druid/common/config/ConfigSerde.java | 9 - .../common/config/JacksonConfigManager.java | 35 +- .../org/apache/druid/audit/AuditInfoTest.java | 4 +- .../apache/druid/audit/NoopAuditManager.java | 71 --- .../common/config/ConfigManagerTest.java | 8 +- .../config/JacksonConfigManagerTest.java | 75 +-- .../java/util/metrics/StubServiceEmitter.java | 17 + .../guice/SQLMetadataStorageDruidModule.java | 62 ++- .../metadata/SQLMetadataRuleManager.java | 10 +- .../SQLMetadataRuleManagerProvider.java | 3 +- .../druid/server/audit/AuditLogger.java | 46 ++ .../druid/server/audit/AuditRecord.java | 114 +++++ .../druid/server/audit/AuditSerdeHelper.java | 129 +++++ .../server/audit/LoggingAuditManager.java | 65 +++ .../audit/LoggingAuditManagerConfig.java | 42 ++ .../druid/server/audit/SQLAuditManager.java | 156 +++--- .../server/audit/SQLAuditManagerConfig.java | 15 +- .../server/audit/SQLAuditManagerProvider.java | 86 ---- .../CoordinatorCompactionConfigsResource.java | 8 +- .../druid/server/http/RulesResource.java | 4 +- .../metadata/SQLMetadataRuleManagerTest.java | 19 +- .../server/audit/SQLAuditManagerTest.java | 450 +++++++----------- .../druid/server/http/RulesResourceTest.java | 26 +- .../org/apache/druid/cli/CliCoordinator.java | 6 - .../org/apache/druid/cli/CliOverlord.java | 6 - .../dialogs/history-dialog/history-dialog.tsx | 2 +- 32 files changed, 913 insertions(+), 802 deletions(-) rename processing/src/main/java/org/apache/druid/audit/{AuditEntry.java => AuditEvent.java} (60%) rename server/src/main/java/org/apache/druid/server/audit/AuditManagerProvider.java => processing/src/main/java/org/apache/druid/audit/AuditManagerConfig.java (78%) delete mode 100644 processing/src/test/java/org/apache/druid/audit/NoopAuditManager.java create mode 100644 server/src/main/java/org/apache/druid/server/audit/AuditLogger.java create mode 100644 server/src/main/java/org/apache/druid/server/audit/AuditRecord.java create mode 100644 server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java create mode 100644 server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java create mode 100644 server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java delete mode 100644 server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerProvider.java diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index 01af55eadb31..8b5fa52df9f6 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -28,7 +28,7 @@ import com.google.common.collect.Maps; import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; -import org.apache.druid.audit.AuditEntry; +import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.client.indexing.ClientTaskQuery; @@ -37,6 +37,7 @@ import org.apache.druid.error.DruidException; import org.apache.druid.error.ErrorResponse; import org.apache.druid.indexer.RunnerTaskState; +import org.apache.druid.indexer.TaskIdentifier; import org.apache.druid.indexer.TaskInfo; import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskStatus; @@ -193,7 +194,12 @@ public OverlordResource( @Path("/task") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - public Response taskPost(final Task task, @Context final HttpServletRequest req) + public Response taskPost( + final Task task, + @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, + @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, + @Context final HttpServletRequest req + ) { final Set resourceActions; try { @@ -201,13 +207,8 @@ public Response taskPost(final Task task, @Context final HttpServletRequest req) } catch (UOE e) { return Response.status(Response.Status.BAD_REQUEST) - .entity( - ImmutableMap.of( - "error", - e.getMessage() - ) - ) - .build(); + .entity(ImmutableMap.of("error", e.getMessage())) + .build(); } Access authResult = AuthorizationUtils.authorizeAllResourceActions( @@ -225,18 +226,30 @@ public Response taskPost(final Task task, @Context final HttpServletRequest req) taskQueue -> { try { taskQueue.add(task); + + // Do an audit only if this API was called by a user and not by internal services + if (author != null && !author.isEmpty()) { + auditManager.doAudit( + AuditEvent.builder() + .key(task.getDataSource()) + .type("submit.ingestion.task") + .payload(new TaskIdentifier(task.getId(), task.getGroupId(), task.getType())) + .auditInfo(new AuditInfo(author, comment, req.getRemoteAddr())) + ); + } + return Response.ok(ImmutableMap.of("task", task.getId())).build(); } catch (DruidException e) { return Response - .status(e.getStatusCode()) - .entity(new ErrorResponse(e)) - .build(); + .status(e.getStatusCode()) + .entity(new ErrorResponse(e)) + .build(); } catch (org.apache.druid.common.exception.DruidException e) { return Response.status(e.getResponseCode()) - .entity(ImmutableMap.of("error", e.getMessage())) - .build(); + .entity(ImmutableMap.of("error", e.getMessage())) + .build(); } } ); @@ -572,7 +585,7 @@ public Response getWorkerConfigHistory( Interval theInterval = interval == null ? null : Intervals.of(interval); if (theInterval == null && count != null) { try { - List workerEntryList = auditManager.fetchAuditHistory( + List workerEntryList = auditManager.fetchAuditHistory( WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.CONFIG_KEY, count @@ -585,7 +598,7 @@ public Response getWorkerConfigHistory( .build(); } } - List workerEntryList = auditManager.fetchAuditHistory( + List workerEntryList = auditManager.fetchAuditHistory( WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.CONFIG_KEY, theInterval diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java index 6eed9e32df38..ae537b85102c 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java @@ -877,7 +877,7 @@ public void testSecuredTaskPost() authConfig ); Task task = NoopTask.create(); - overlordResource.taskPost(task, req); + overlordResource.taskPost(task, "", "", req); } @Test @@ -900,7 +900,7 @@ public void testTaskPostDeniesDatasourceReadUser() Task task = NoopTask.forDatasource(Datasources.WIKIPEDIA); expectedException.expect(ForbiddenException.class); expectedException.expect(ForbiddenException.class); - overlordResource.taskPost(task, req); + overlordResource.taskPost(task, "", "", req); } @Test diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java index f9ce36df1815..1224a80965e6 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java @@ -282,12 +282,12 @@ public void testOverlordRun() throws Exception taskCompletionCountDownLatches.get(goodTaskId).countDown(); waitForTaskStatus(goodTaskId, TaskState.SUCCESS); - response = overlordResource.taskPost(task0, req); + response = overlordResource.taskPost(task0, "", "", req); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("task", taskId0), response.getEntity()); // Duplicate task - should fail - response = overlordResource.taskPost(task0, req); + response = overlordResource.taskPost(task0, "", "", req); Assert.assertEquals(400, response.getStatus()); // Task payload for task_0 should be present in taskStorage diff --git a/processing/src/main/java/org/apache/druid/audit/AuditEntry.java b/processing/src/main/java/org/apache/druid/audit/AuditEvent.java similarity index 60% rename from processing/src/main/java/org/apache/druid/audit/AuditEntry.java rename to processing/src/main/java/org/apache/druid/audit/AuditEvent.java index 2aeb77279953..aac12b050b53 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditEntry.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditEvent.java @@ -19,30 +19,54 @@ package org.apache.druid.audit; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import org.apache.druid.java.util.common.DateTimes; import org.joda.time.DateTime; +import java.util.Objects; + /** - * An Entry in Audit Table. + * Represents a single audit event with serialized payload. */ -public class AuditEntry +public class AuditEvent { private final String key; private final String type; private final AuditInfo auditInfo; - private final String payload; private final DateTime auditTime; - @JsonCreator - public AuditEntry( - @JsonProperty("key") String key, - @JsonProperty("type") String type, - @JsonProperty("auditInfo") AuditInfo authorInfo, - @JsonProperty("payload") String payload, - @JsonProperty("auditTime") DateTime auditTime + private final String serializedPayload; + private final Object payload; + + private AuditEvent( + String key, + String type, + AuditInfo authorInfo, + DateTime auditTime, + Object payload + ) + { + this(key, type, authorInfo, payload, null, auditTime); + } + + public AuditEvent( + String key, + String type, + AuditInfo authorInfo, + String serializedPayload, + DateTime auditTime + ) + { + this(key, type, authorInfo, null, serializedPayload, auditTime); + } + + public AuditEvent( + String key, + String type, + AuditInfo authorInfo, + Object payload, + String serializedPayload, + DateTime auditTime ) { Preconditions.checkNotNull(key, "key cannot be null"); @@ -52,40 +76,35 @@ public AuditEntry( this.type = type; this.auditInfo = authorInfo; this.auditTime = auditTime == null ? DateTimes.nowUtc() : auditTime; + this.serializedPayload = serializedPayload; this.payload = payload; } - @JsonProperty public String getKey() { return key; } - @JsonProperty public String getType() { return type; } - @JsonProperty public AuditInfo getAuditInfo() { return auditInfo; } - /** - * @return returns payload as String - */ - @JsonProperty - public String getPayload() + public Object getPayload() { return payload; } - /** - * @return audit time as DateTime - */ - @JsonProperty + public String getPayloadAsString() + { + return serializedPayload; + } + public DateTime getAuditTime() { return auditTime; @@ -106,36 +125,18 @@ public boolean equals(Object o) return false; } - AuditEntry entry = (AuditEntry) o; - - if (!auditTime.equals(entry.auditTime)) { - return false; - } - if (!auditInfo.equals(entry.auditInfo)) { - return false; - } - if (!key.equals(entry.key)) { - return false; - } - if (!payload.equals(entry.payload)) { - return false; - } - if (!type.equals(entry.type)) { - return false; - } - - return true; + AuditEvent that = (AuditEvent) o; + return Objects.equals(this.auditTime, that.auditTime) + && Objects.equals(this.key, that.key) + && Objects.equals(this.type, that.type) + && Objects.equals(this.auditInfo, that.auditInfo) + && Objects.equals(this.serializedPayload, that.serializedPayload); } @Override public int hashCode() { - int result = key.hashCode(); - result = 31 * result + type.hashCode(); - result = 31 * result + auditInfo.hashCode(); - result = 31 * result + payload.hashCode(); - result = 31 * result + auditTime.hashCode(); - return result; + return Objects.hash(key, type, auditInfo, serializedPayload, auditTime); } public static class Builder @@ -143,14 +144,17 @@ public static class Builder private String key; private String type; private AuditInfo auditInfo; - private String payload; + + private String serializedPayload; + private Object payload; + private DateTime auditTime; private Builder() { this.key = null; this.auditInfo = null; - this.payload = null; + this.serializedPayload = null; this.auditTime = DateTimes.nowUtc(); } @@ -172,7 +176,13 @@ public Builder auditInfo(AuditInfo auditInfo) return this; } - public Builder payload(String payload) + public Builder payloadAsString(String serializedPayload) + { + this.serializedPayload = serializedPayload; + return this; + } + + public Builder payload(Object payload) { this.payload = payload; return this; @@ -184,11 +194,13 @@ public Builder auditTime(DateTime auditTime) return this; } - public AuditEntry build() + public AuditEvent build() { - return new AuditEntry(key, type, auditInfo, payload, auditTime); + if (payload != null) { + return new AuditEvent(key, type, auditInfo, auditTime, payload); + } else { + return new AuditEvent(key, type, auditInfo, serializedPayload, auditTime); + } } - } - } diff --git a/processing/src/main/java/org/apache/druid/audit/AuditManager.java b/processing/src/main/java/org/apache/druid/audit/AuditManager.java index f403423c6321..487a9c9d3e76 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditManager.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditManager.java @@ -20,7 +20,6 @@ package org.apache.druid.audit; -import org.apache.druid.common.config.ConfigSerde; import org.joda.time.Interval; import org.skife.jdbi.v2.Handle; @@ -29,68 +28,65 @@ public interface AuditManager { - /** - * This String is the default message stored instead of the actual audit payload if the audit payload size - * exceeded the maximum size limit configuration - */ - String PAYLOAD_SKIP_MSG_FORMAT = "Payload was not stored as its size exceeds the limit [%d] configured by druid.audit.manager.maxPayloadSizeBytes"; String X_DRUID_AUTHOR = "X-Druid-Author"; String X_DRUID_COMMENT = "X-Druid-Comment"; - /** - * inserts an audit entry in the Audit Table - * @param key of the audit entry - * @param type of the audit entry - * @param auditInfo of the audit entry - * @param payload of the audit entry - * @param configSerde of the payload of the audit entry - */ - void doAudit(String key, String type, AuditInfo auditInfo, T payload, ConfigSerde configSerde); + void doAudit(AuditEvent event); /** - * inserts an audit Entry in audit table using the handler provided - * used to do the audit in same transaction as the config changes - * @param auditEntry - * @param handler - * @throws IOException + * Inserts an audit entry in audit table using the provided JDBI handle. + * This method can be used to perform the audit in the same transaction as the + * audited changes. Only SQL-based implementations need to implement this method, + * other implementations call {@link #doAudit} by default. + * + * @param AuditEvent + * @param handle JDBI Handle representing connection to the database */ - void doAudit(AuditEntry auditEntry, Handle handler) throws IOException; + default void doAudit(AuditEvent event, Handle handle) throws IOException + { + doAudit(event); + } /** - * provides audit history for given key, type and interval + * Fetches audit entries made for the given key, type and interval. Implementations + * that do not maintain an audit history should return an empty list. + * * @param key * @param type * @param interval * @return list of AuditEntries satisfying the passed parameters */ - List fetchAuditHistory(String key, String type, Interval interval); + List fetchAuditHistory(String key, String type, Interval interval); /** - * provides audit history for given type and interval - * @param type type of auditEntry - * @param interval interval for which to fetch auditHistory - * @return list of AuditEntries satisfying the passed parameters + * Fetches audit entries of a type whose audit time lies in the given interval. + * + * @param type Type of audit entry + * @param interval Eligible interval for audit time + * @return List of audit entries satisfying the passed parameters. */ - List fetchAuditHistory(String type, Interval interval); + List fetchAuditHistory(String type, Interval interval); /** * Provides last N entries of audit history for given key, type + * * @param key * @param type * @param limit * @return list of AuditEntries satisfying the passed parameters */ - List fetchAuditHistory(String key, String type, int limit); + List fetchAuditHistory(String key, String type, int limit); /** * Provides last N entries of audit history for given type - * @param type type of auditEntry + * + * @param type type of AuditEvent * @param limit * @return list of AuditEntries satisfying the passed parameters */ - List fetchAuditHistory(String type, int limit); + List fetchAuditHistory(String type, int limit); /** * Remove audit logs created older than the given timestamp. diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditManagerProvider.java b/processing/src/main/java/org/apache/druid/audit/AuditManagerConfig.java similarity index 78% rename from server/src/main/java/org/apache/druid/server/audit/AuditManagerProvider.java rename to processing/src/main/java/org/apache/druid/audit/AuditManagerConfig.java index 7607d90e3ebd..0858a466e058 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditManagerProvider.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditManagerConfig.java @@ -17,13 +17,11 @@ * under the License. */ -package org.apache.druid.server.audit; +package org.apache.druid.audit; -import com.google.inject.Provider; -import org.apache.druid.audit.AuditManager; - -public interface AuditManagerProvider extends Provider +public interface AuditManagerConfig { - @Override - AuditManager get(); + boolean isSkipNullField(); + + long getMaxPayloadSizeBytes(); } diff --git a/processing/src/main/java/org/apache/druid/common/config/ConfigSerde.java b/processing/src/main/java/org/apache/druid/common/config/ConfigSerde.java index 708d16d8b190..a4c06e1371e0 100644 --- a/processing/src/main/java/org/apache/druid/common/config/ConfigSerde.java +++ b/processing/src/main/java/org/apache/druid/common/config/ConfigSerde.java @@ -24,14 +24,5 @@ public interface ConfigSerde { byte[] serialize(T obj); - /** - * Serialize object to String - * - * @param obj to be serialize - * @param skipNull if true, then skip serialization of any field with null value. - * This can be used to reduce the size of the resulting String. - * @return String serialization of the input - */ - String serializeToString(T obj, boolean skipNull); T deserialize(byte[] bytes); } diff --git a/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java b/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java index 7cf78a5ca0bf..b18d0f0162cd 100644 --- a/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java +++ b/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java @@ -24,11 +24,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; +import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.common.config.ConfigManager.SetResult; import org.apache.druid.guice.annotations.Json; -import org.apache.druid.guice.annotations.JsonNonNull; import org.apache.druid.java.util.common.jackson.JacksonUtils; import javax.annotation.Nullable; @@ -41,21 +41,18 @@ public class JacksonConfigManager { private final ConfigManager configManager; private final ObjectMapper jsonMapper; - private final ObjectMapper jsonMapperSkipNull; private final AuditManager auditManager; @Inject public JacksonConfigManager( ConfigManager configManager, @Json ObjectMapper jsonMapper, - @JsonNonNull ObjectMapper jsonMapperOnlyNonNullValue, AuditManager auditManager ) { this.configManager = configManager; this.jsonMapper = jsonMapper; this.auditManager = auditManager; - this.jsonMapperSkipNull = jsonMapperOnlyNonNullValue; } public AtomicReference watch(String key, Class clazz) @@ -119,7 +116,13 @@ public SetResult set( ConfigSerde configSerde = create(newValue.getClass(), null); // Audit and actual config change are done in separate transactions // there can be phantom audits and reOrdering in audit changes as well. - auditManager.doAudit(key, key, auditInfo, newValue, configSerde); + auditManager.doAudit( + AuditEvent.builder() + .key(key) + .type(key) + .auditInfo(auditInfo) + .payload(newValue) + ); return configManager.set(key, configSerde, oldValue, newValue); } @@ -139,17 +142,6 @@ public byte[] serialize(T obj) } } - @Override - public String serializeToString(T obj, boolean skipNull) - { - try { - return skipNull ? jsonMapperSkipNull.writeValueAsString(obj) : jsonMapper.writeValueAsString(obj); - } - catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - @Override public T deserialize(byte[] bytes) { @@ -178,17 +170,6 @@ public byte[] serialize(T obj) } } - @Override - public String serializeToString(T obj, boolean skipNull) - { - try { - return skipNull ? jsonMapperSkipNull.writeValueAsString(obj) : jsonMapper.writeValueAsString(obj); - } - catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - @Override public T deserialize(byte[] bytes) { diff --git a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java index c61b50ec6a12..b5bc7d71fb46 100644 --- a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java +++ b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java @@ -41,7 +41,7 @@ public void testAuditInfoEquality() @Test(timeout = 60_000L) public void testAuditEntrySerde() throws IOException { - AuditEntry entry = new AuditEntry( + AuditEvent entry = new AuditEvent( "testKey", "testType", new AuditInfo( @@ -53,7 +53,7 @@ public void testAuditEntrySerde() throws IOException DateTimes.of("2013-01-01T00:00:00Z") ); ObjectMapper mapper = new DefaultObjectMapper(); - AuditEntry serde = mapper.readValue(mapper.writeValueAsString(entry), AuditEntry.class); + AuditEvent serde = mapper.readValue(mapper.writeValueAsString(entry), AuditEvent.class); Assert.assertEquals(entry, serde); } diff --git a/processing/src/test/java/org/apache/druid/audit/NoopAuditManager.java b/processing/src/test/java/org/apache/druid/audit/NoopAuditManager.java deleted file mode 100644 index 593a9c73eaca..000000000000 --- a/processing/src/test/java/org/apache/druid/audit/NoopAuditManager.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.audit; - -import org.apache.druid.common.config.ConfigSerde; -import org.joda.time.Interval; -import org.skife.jdbi.v2.Handle; - -import java.util.List; - -public class NoopAuditManager implements AuditManager -{ - @Override - public void doAudit(String key, String type, AuditInfo auditInfo, T payload, ConfigSerde configSerde) - { - throw new UnsupportedOperationException(); - } - - @Override - public void doAudit(AuditEntry auditEntry, Handle handler) - { - throw new UnsupportedOperationException(); - } - - @Override - public List fetchAuditHistory(String key, String type, Interval interval) - { - throw new UnsupportedOperationException(); - } - - @Override - public List fetchAuditHistory(String type, Interval interval) - { - throw new UnsupportedOperationException(); - } - - @Override - public List fetchAuditHistory(String key, String type, int limit) - { - throw new UnsupportedOperationException(); - } - - @Override - public List fetchAuditHistory(String type, int limit) - { - throw new UnsupportedOperationException(); - } - - @Override - public int removeAuditLogsOlderThan(long timestamp) - { - throw new UnsupportedOperationException(); - } -} diff --git a/processing/src/test/java/org/apache/druid/common/config/ConfigManagerTest.java b/processing/src/test/java/org/apache/druid/common/config/ConfigManagerTest.java index 69c319f1bcf2..0f6cb4e7c4e7 100644 --- a/processing/src/test/java/org/apache/druid/common/config/ConfigManagerTest.java +++ b/processing/src/test/java/org/apache/druid/common/config/ConfigManagerTest.java @@ -20,12 +20,9 @@ package org.apache.druid.common.config; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Suppliers; -import org.apache.druid.audit.AuditManager; -import org.apache.druid.audit.NoopAuditManager; import org.apache.druid.metadata.MetadataCASUpdate; import org.apache.druid.metadata.MetadataStorageConnector; import org.apache.druid.metadata.MetadataStorageTablesConfig; @@ -37,7 +34,6 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; -@SuppressWarnings("ALL") public class ConfigManagerTest { private static final String CONFIG_KEY = "configX"; @@ -47,7 +43,6 @@ public class ConfigManagerTest private MetadataStorageConnector dbConnector; private MetadataStorageTablesConfig metadataStorageTablesConfig; - private AuditManager mockAuditManager; private TestConfigManagerConfig configManagerConfig; private ConfigSerde configConfigSerdeFromClass; @@ -79,8 +74,7 @@ public String getConfigTable() jacksonConfigManager = new JacksonConfigManager( configManager, new ObjectMapper(), - new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL), - new NoopAuditManager() + null ); configConfigSerdeFromClass = jacksonConfigManager.create(TestConfig.class, null); } diff --git a/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java b/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java index 4a4aaf72cd4b..33e0e4cd324f 100644 --- a/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java +++ b/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java @@ -20,20 +20,17 @@ package org.apache.druid.common.config; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; -import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; @@ -51,68 +48,16 @@ public class JacksonConfigManagerTest private JacksonConfigManager jacksonConfigManager; - @Rule - public ExpectedException exception = ExpectedException.none(); - @Before public void setUp() { jacksonConfigManager = new JacksonConfigManager( mockConfigManager, new ObjectMapper(), - new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL), mockAuditManager ); } - @Test - public void testSerializeToStringWithSkipNullTrue() - { - ConfigSerde configConfigSerdeFromTypeReference = jacksonConfigManager.create(new TypeReference() - { - }, null); - ConfigSerde configConfigSerdeFromClass = jacksonConfigManager.create(TestConfig.class, null); - TestConfig config = new TestConfig("version", null, 3); - String actual = configConfigSerdeFromTypeReference.serializeToString(config, true); - Assert.assertEquals("{\"version\":\"version\",\"settingInt\":3}", actual); - actual = configConfigSerdeFromClass.serializeToString(config, true); - Assert.assertEquals("{\"version\":\"version\",\"settingInt\":3}", actual); - } - - @Test - public void testSerializeToStringWithSkipNullFalse() - { - ConfigSerde configConfigSerdeFromTypeReference = jacksonConfigManager.create(new TypeReference() - { - }, null); - ConfigSerde configConfigSerdeFromClass = jacksonConfigManager.create(TestConfig.class, null); - TestConfig config = new TestConfig("version", null, 3); - String actual = configConfigSerdeFromTypeReference.serializeToString(config, false); - Assert.assertEquals("{\"version\":\"version\",\"settingString\":null,\"settingInt\":3}", actual); - actual = configConfigSerdeFromClass.serializeToString(config, false); - Assert.assertEquals("{\"version\":\"version\",\"settingString\":null,\"settingInt\":3}", actual); - } - - @Test - public void testSerializeToStringWithInvalidConfigForConfigSerdeFromTypeReference() - { - ConfigSerde configConfigSerdeFromTypeReference = jacksonConfigManager.create(new TypeReference() - { - }, null); - exception.expect(RuntimeException.class); - exception.expectMessage("InvalidDefinitionException"); - configConfigSerdeFromTypeReference.serializeToString(new ClassThatJacksonCannotSerialize(), false); - } - - @Test - public void testSerializeToStringWithInvalidConfigForConfigSerdeFromClass() - { - ConfigSerde configConfigSerdeFromClass = jacksonConfigManager.create(ClassThatJacksonCannotSerialize.class, null); - exception.expect(RuntimeException.class); - exception.expectMessage("InvalidDefinitionException"); - configConfigSerdeFromClass.serializeToString(new ClassThatJacksonCannotSerialize(), false); - } - @Test public void testSet() { @@ -126,16 +71,9 @@ public void testSet() jacksonConfigManager.set(key, val, auditInfo); - ArgumentCaptor configSerdeCapture = ArgumentCaptor.forClass( - ConfigSerde.class); - Mockito.verify(mockAuditManager).doAudit( - ArgumentMatchers.eq(key), - ArgumentMatchers.eq(key), - ArgumentMatchers.eq(auditInfo), - ArgumentMatchers.eq(val), - configSerdeCapture.capture() - ); - Assert.assertNotNull(configSerdeCapture.getValue()); + ArgumentCaptor auditCapture = ArgumentCaptor.forClass(AuditEvent.Builder.class); + Mockito.verify(mockAuditManager).doAudit(auditCapture.capture()); + Assert.assertNotNull(auditCapture.getValue()); } @Test @@ -214,9 +152,4 @@ public int hashCode() return Objects.hash(version, settingString, settingInt); } } - - static class ClassThatJacksonCannotSerialize - { - - } } diff --git a/processing/src/test/java/org/apache/druid/java/util/metrics/StubServiceEmitter.java b/processing/src/test/java/org/apache/druid/java/util/metrics/StubServiceEmitter.java index 52049c6956f6..40826af80473 100644 --- a/processing/src/test/java/org/apache/druid/java/util/metrics/StubServiceEmitter.java +++ b/processing/src/test/java/org/apache/druid/java/util/metrics/StubServiceEmitter.java @@ -30,6 +30,10 @@ import java.util.List; import java.util.Map; +/** + * Test implementation of {@link ServiceEmitter} that collects emitted metrics + * and alerts in lists. + */ public class StubServiceEmitter extends ServiceEmitter implements MetricsVerifier { private final List events = new ArrayList<>(); @@ -62,6 +66,19 @@ public List getEvents() return events; } + /** + * Gets all the metric events emitted since the previous {@link #flush()}. + * + * @return Map from metric name to list of events emitted for that metric. + */ + public Map> getMetricEvents() + { + return metricEvents; + } + + /** + * Gets all the alerts emitted since the previous {@link #flush()}. + */ public List getAlerts() { return alertEvents; diff --git a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java index 6bfe1e25eedc..7dfe63eb9c5f 100644 --- a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java +++ b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java @@ -22,7 +22,10 @@ import com.google.inject.Binder; import com.google.inject.Key; import com.google.inject.Module; +import com.google.inject.multibindings.MapBinder; import org.apache.druid.audit.AuditManager; +import org.apache.druid.audit.AuditManagerConfig; +import org.apache.druid.server.audit.AuditSerdeHelper; import org.apache.druid.indexer.MetadataStorageUpdaterJobHandler; import org.apache.druid.indexer.SQLMetadataStorageUpdaterJobHandler; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; @@ -45,10 +48,10 @@ import org.apache.druid.metadata.SegmentsMetadataManagerProvider; import org.apache.druid.metadata.SqlSegmentsMetadataManager; import org.apache.druid.metadata.SqlSegmentsMetadataManagerProvider; -import org.apache.druid.server.audit.AuditManagerProvider; +import org.apache.druid.server.audit.LoggingAuditManager; +import org.apache.druid.server.audit.LoggingAuditManagerConfig; import org.apache.druid.server.audit.SQLAuditManager; import org.apache.druid.server.audit.SQLAuditManagerConfig; -import org.apache.druid.server.audit.SQLAuditManagerProvider; public class SQLMetadataStorageDruidModule implements Module { @@ -82,9 +85,10 @@ public void createBindingChoices(Binder binder, String defaultValue) PolyBind.createChoiceWithDefault(binder, prop, Key.get(IndexerMetadataStorageCoordinator.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageActionHandlerFactory.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageUpdaterJobHandler.class), defaultValue); - PolyBind.createChoiceWithDefault(binder, prop, Key.get(AuditManager.class), defaultValue); - PolyBind.createChoiceWithDefault(binder, prop, Key.get(AuditManagerProvider.class), defaultValue); + // PolyBind.createChoiceWithDefault(binder, prop, Key.get(AuditManager.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataSupervisorManager.class), defaultValue); + + configureAuditDestination(binder); } @Override @@ -130,21 +134,55 @@ public void configure(Binder binder) .to(SQLMetadataStorageUpdaterJobHandler.class) .in(LazySingleton.class); - JsonConfigProvider.bind(binder, "druid.audit.manager", SQLAuditManagerConfig.class); + // JsonConfigProvider.bind(binder, "druid.audit.manager", SQLAuditManagerConfig.class); - PolyBind.optionBinder(binder, Key.get(AuditManager.class)) + /*PolyBind.optionBinder(binder, Key.get(AuditManager.class)) .addBinding(type) .to(SQLAuditManager.class) - .in(LazySingleton.class); - - PolyBind.optionBinder(binder, Key.get(AuditManagerProvider.class)) - .addBinding(type) - .to(SQLAuditManagerProvider.class) - .in(LazySingleton.class); + .in(LazySingleton.class);*/ PolyBind.optionBinder(binder, Key.get(MetadataSupervisorManager.class)) .addBinding(type) .to(SQLMetadataSupervisorManager.class) .in(LazySingleton.class); } + + private void configureAuditDestination(Binder binder) + { + PolyBind.createChoice( + binder, + "druid.audit.destination", + Key.get(AuditManager.class), + Key.get(SQLAuditManager.class) + ); + final MapBinder auditManagerBinder + = PolyBind.optionBinder(binder, Key.get(AuditManager.class)); + auditManagerBinder + .addBinding("log") + .to(LoggingAuditManager.class) + .in(LazySingleton.class); + auditManagerBinder + .addBinding("sql") + .to(SQLAuditManager.class) + .in(LazySingleton.class); + + PolyBind.createChoice( + binder, + "druid.audit.destination", + Key.get(AuditManagerConfig.class), + Key.get(SQLAuditManagerConfig.class) + ); + final MapBinder auditManagerConfigBinder + = PolyBind.optionBinder(binder, Key.get(AuditManagerConfig.class)); + auditManagerConfigBinder + .addBinding("log") + .to(LoggingAuditManagerConfig.class) + .in(LazySingleton.class); + auditManagerConfigBinder + .addBinding("sql") + .to(SQLAuditManagerConfig.class) + .in(LazySingleton.class); + + binder.bind(AuditSerdeHelper.class); + } } diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java b/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java index d0f9799e6aab..a438b2cc6302 100644 --- a/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java +++ b/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java @@ -25,7 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; -import org.apache.druid.audit.AuditEntry; +import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.client.DruidServer; @@ -336,10 +336,10 @@ public boolean overrideRule(final String dataSource, final List newRules, final String ruleString; try { ruleString = jsonMapper.writeValueAsString(newRules); - log.info("Updating [%s] with rules [%s] as per [%s]", dataSource, ruleString, auditInfo); + log.info("Updating datasource[%s] with rules[%s] as per [%s]", dataSource, ruleString, auditInfo); } catch (JsonProcessingException e) { - log.error(e, "Unable to write rules as string for [%s]", dataSource); + log.error(e, "Unable to write rules as string for datasource[%s]", dataSource); return false; } synchronized (lock) { @@ -348,11 +348,11 @@ public boolean overrideRule(final String dataSource, final List newRules, (handle, transactionStatus) -> { final DateTime auditTime = DateTimes.nowUtc(); auditManager.doAudit( - AuditEntry.builder() + AuditEvent.builder() .key(dataSource) .type("rules") .auditInfo(auditInfo) - .payload(ruleString) + .payloadAsString(ruleString) .auditTime(auditTime) .build(), handle diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManagerProvider.java b/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManagerProvider.java index ba3d05c131dc..d11d33ba5b17 100644 --- a/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManagerProvider.java +++ b/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManagerProvider.java @@ -23,7 +23,6 @@ import com.google.inject.Inject; import org.apache.druid.audit.AuditManager; import org.apache.druid.java.util.common.lifecycle.Lifecycle; -import org.apache.druid.server.audit.SQLAuditManager; import org.skife.jdbi.v2.IDBI; /** @@ -45,7 +44,7 @@ public SQLMetadataRuleManagerProvider( MetadataStorageTablesConfig dbTables, SQLMetadataConnector connector, Lifecycle lifecycle, - SQLAuditManager auditManager + AuditManager auditManager ) { this.jsonMapper = jsonMapper; diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java new file mode 100644 index 000000000000..12913da7a874 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java @@ -0,0 +1,46 @@ +package org.apache.druid.server.audit; + +import org.apache.druid.java.util.common.logger.Logger; + +public class AuditLogger +{ + public enum Level + { + DEBUG, INFO, WARN + } + + private static final String MSG_FORMAT + = "[%s] User[%s], ip[%s] performed action[%s] on key[%s] with comment[%s]. Payload[%s]."; + + private final Level level; + private final Logger logger = new Logger(AuditLogger.class); + + public AuditLogger(Level level) + { + this.level = level; + } + + public void log(AuditRecord record) + { + Object[] args = { + record.getAuditTime(), + record.getAuditInfo().getAuthor(), + record.getAuditInfo().getIp(), + record.getType(), + record.getKey(), + record.getAuditInfo().getComment(), + record.getPayload() + }; + switch (level) { + case DEBUG: + logger.debug(MSG_FORMAT, args); + break; + case INFO: + logger.info(MSG_FORMAT, args); + break; + case WARN: + logger.warn(MSG_FORMAT, args); + break; + } + } +} diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditRecord.java b/server/src/main/java/org/apache/druid/server/audit/AuditRecord.java new file mode 100644 index 000000000000..814d405e7dcb --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/audit/AuditRecord.java @@ -0,0 +1,114 @@ +package org.apache.druid.server.audit; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditInfo; +import org.apache.druid.java.util.common.DateTimes; +import org.joda.time.DateTime; + +import java.util.Objects; + +/** + * Record of an audit event persisted in the metadata store. + */ +public class AuditRecord +{ + private final String key; + private final String type; + private final AuditInfo auditInfo; + private final String payload; + private final DateTime auditTime; + + public static AuditRecord fromAuditEvent(AuditEvent event) + { + return new AuditRecord( + event.getKey(), + event.getType(), + event.getAuditInfo(), + event.getPayloadAsString(), + event.getAuditTime() + ); + } + + @JsonCreator + public AuditRecord( + @JsonProperty("key") String key, + @JsonProperty("type") String type, + @JsonProperty("auditInfo") AuditInfo authorInfo, + @JsonProperty("payload") String payload, + @JsonProperty("auditTime") DateTime auditTime + ) + { + Preconditions.checkNotNull(key, "key cannot be null"); + Preconditions.checkNotNull(type, "type cannot be null"); + Preconditions.checkNotNull(authorInfo, "author cannot be null"); + this.key = key; + this.type = type; + this.auditInfo = authorInfo; + this.auditTime = auditTime == null ? DateTimes.nowUtc() : auditTime; + this.payload = payload == null ? "" : payload; + } + + @JsonProperty + public String getKey() + { + return key; + } + + @JsonProperty + public String getType() + { + return type; + } + + @JsonProperty + public AuditInfo getAuditInfo() + { + return auditInfo; + } + + /** + * @return Payload as a non-null String. + */ + @JsonProperty + public String getPayload() + { + return payload; + } + + /** + * @return audit time as DateTime + */ + @JsonProperty + public DateTime getAuditTime() + { + return auditTime; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + AuditRecord that = (AuditRecord) o; + return Objects.equals(this.auditTime, that.auditTime) + && Objects.equals(this.key, that.key) + && Objects.equals(this.type, that.type) + && Objects.equals(this.auditInfo, that.auditInfo) + && Objects.equals(this.payload, that.payload); + } + + @Override + public int hashCode() + { + return Objects.hash(key, type, auditInfo, payload, auditTime); + } + +} diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java new file mode 100644 index 000000000000..728c1d5d3ecf --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.audit; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditManagerConfig; +import org.apache.druid.error.InvalidInput; +import org.apache.druid.guice.annotations.Json; +import org.apache.druid.guice.annotations.JsonNonNull; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.StringUtils; + +import java.io.IOException; + +public class AuditSerdeHelper implements AuditEvent.PayloadDeserializer +{ + /** + * Default message stored instead of the actual audit payload if the audit + * payload size exceeds the maximum size limit. + */ + private static final String PAYLOAD_TRUNCATED_MSG = + "Payload truncated as it exceeds 'druid.audit.manager.maxPayloadSizeBytes'"; + + private final ObjectMapper jsonMapper; + private final ObjectMapper jsonMapperSkipNulls; + + private final AuditManagerConfig config; + + @Inject + public AuditSerdeHelper( + AuditManagerConfig config, + @Json ObjectMapper jsonMapper, + @JsonNonNull ObjectMapper jsonMapperSkipNulls + ) + { + this.config = config; + this.jsonMapper = jsonMapper; + this.jsonMapperSkipNulls = jsonMapperSkipNulls; + } + + public AuditRecord processAuditEvent(AuditEvent event) + { + final String serialized = event.getPayloadAsString() == null + ? serializePayloadToString(event.getPayload()) + : event.getPayloadAsString(); + + return new AuditRecord( + event.getKey(), + event.getType(), + event.getAuditInfo(), + truncateSerializedAuditPayload(serialized), + event.getAuditTime() + ); + } + + @Override + public T deserializePayloadFromString(String serializedPayload, Class clazz) + { + if (serializedPayload == null || serializedPayload.isEmpty()) { + return null; + } else if (serializedPayload.contains(PAYLOAD_TRUNCATED_MSG)) { + throw InvalidInput.exception("Cannot deserialize audit payload[%s].", serializedPayload); + } + + try { + return jsonMapper.readValue(serializedPayload, clazz); + } + catch (IOException e) { + throw InvalidInput.exception( + e, + "Could not deserialize audit payload[%s] into class[%s]", + serializedPayload, clazz + ); + } + } + + private String serializePayloadToString(Object payload) + { + if (payload == null) { + return ""; + } + + try { + return config.isSkipNullField() + ? jsonMapperSkipNulls.writeValueAsString(payload) + : jsonMapper.writeValueAsString(payload); + } + catch (IOException e) { + throw new ISE(e, "Could not serialize audit payload[%s]", payload); + } + } + + /** + * Truncates the audit payload string if it exceeds + * {@link AuditManagerConfig#getMaxPayloadSizeBytes()}. + */ + private String truncateSerializedAuditPayload(String serializedPayload) + { + if (serializedPayload == null || config.getMaxPayloadSizeBytes() < 0) { + return serializedPayload; + } + + int payloadSize = serializedPayload.getBytes().length; + if (payloadSize > config.getMaxPayloadSizeBytes()) { + return PAYLOAD_TRUNCATED_MSG + StringUtils.format("[%s].", config.getMaxPayloadSizeBytes()); + } else { + return serializedPayload; + } + } +} diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java new file mode 100644 index 000000000000..19be98c0f558 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java @@ -0,0 +1,65 @@ +package org.apache.druid.server.audit; + +import com.google.inject.Inject; +import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditManager; +import org.joda.time.Interval; + +import java.util.Collections; +import java.util.List; + +/** + * Audit manager that logs audited events at the level specified in + * {@link LoggingAuditManagerConfig}. + */ +public class LoggingAuditManager implements AuditManager +{ + private final AuditLogger auditLogger; + private final AuditSerdeHelper serdeHelper; + + @Inject + public LoggingAuditManager( + LoggingAuditManagerConfig config, + AuditSerdeHelper serdeHelper + ) + { + this.serdeHelper = serdeHelper; + this.auditLogger = new AuditLogger(config.getLogLevel()); + } + + @Override + public void doAudit(AuditEvent event) + { + auditLogger.log(serdeHelper.processAuditEvent(event)); + } + + @Override + public List fetchAuditHistory(String key, String type, Interval interval) + { + return Collections.emptyList(); + } + + @Override + public List fetchAuditHistory(String type, Interval interval) + { + return Collections.emptyList(); + } + + @Override + public List fetchAuditHistory(String key, String type, int limit) + { + return Collections.emptyList(); + } + + @Override + public List fetchAuditHistory(String type, int limit) + { + return Collections.emptyList(); + } + + @Override + public int removeAuditLogsOlderThan(long timestamp) + { + return 0; + } +} diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java new file mode 100644 index 000000000000..b3b350c9a634 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java @@ -0,0 +1,42 @@ +package org.apache.druid.server.audit; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.audit.AuditManagerConfig; +import org.apache.druid.java.util.common.HumanReadableBytes; +import org.apache.druid.java.util.common.HumanReadableBytesRange; + +public class LoggingAuditManagerConfig implements AuditManagerConfig +{ + @JsonProperty + private final boolean includePayloadAsDimensionInMetric = false; + + @JsonProperty + private final AuditLogger.Level logLevel = AuditLogger.Level.INFO; + + @JsonProperty + @HumanReadableBytesRange( + min = -1, + message = "maxPayloadSizeBytes must either be -1 (for disabling the check) or a non negative number" + ) + private final HumanReadableBytes maxPayloadSizeBytes = HumanReadableBytes.valueOf(-1); + + @JsonProperty + private final boolean skipNullField = false; + + @Override + public boolean isSkipNullField() + { + return skipNullField; + } + + @Override + public long getMaxPayloadSizeBytes() + { + return maxPayloadSizeBytes.getBytes(); + } + + public AuditLogger.Level getLogLevel() + { + return logLevel; + } +} diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java index 37163f5d7a27..f9a4a7c8cc02 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java @@ -20,18 +20,17 @@ package org.apache.druid.server.audit; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Supplier; import com.google.inject.Inject; -import org.apache.druid.audit.AuditEntry; -import org.apache.druid.audit.AuditInfo; +import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditManager; -import org.apache.druid.common.config.ConfigSerde; import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.guice.annotations.Json; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.jackson.JacksonUtils; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.lifecycle.LifecycleStop; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.metadata.MetadataStorageTablesConfig; @@ -41,10 +40,13 @@ import org.skife.jdbi.v2.Handle; import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.Query; +import org.skife.jdbi.v2.StatementContext; import org.skife.jdbi.v2.Update; -import org.skife.jdbi.v2.tweak.HandleCallback; +import org.skife.jdbi.v2.tweak.ResultSetMapper; import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; import java.util.Map; @@ -52,111 +54,106 @@ public class SQLAuditManager implements AuditManager { private final IDBI dbi; + private final SQLMetadataConnector connector; private final Supplier dbTables; private final ServiceEmitter emitter; private final ObjectMapper jsonMapper; private final SQLAuditManagerConfig config; + private final AuditSerdeHelper serdeHelper; + + private final ResultSetMapper resultMapper; @Inject public SQLAuditManager( + SQLAuditManagerConfig config, + AuditSerdeHelper serdeHelper, SQLMetadataConnector connector, Supplier dbTables, ServiceEmitter emitter, - @Json ObjectMapper jsonMapper, - SQLAuditManagerConfig config + @Json ObjectMapper jsonMapper ) { this.dbi = connector.getDBI(); + this.connector = connector; this.dbTables = dbTables; this.emitter = emitter; this.jsonMapper = jsonMapper; + this.serdeHelper = serdeHelper; this.config = config; + this.resultMapper = new AuditEventMapper(); + } + + @LifecycleStart + public void start() + { + connector.createAuditTable(); } - public String getAuditTable() + @LifecycleStop + public void stop() + { + // Do nothing + } + + private String getAuditTable() { return dbTables.get().getAuditTable(); } @Override - public void doAudit(String key, String type, AuditInfo auditInfo, T payload, ConfigSerde configSerde) + public void doAudit(AuditEvent event) { - AuditEntry auditEntry = AuditEntry.builder() - .key(key) - .type(type) - .auditInfo(auditInfo) - .payload(configSerde.serializeToString(payload, config.isSkipNullField())) - .build(); - dbi.withHandle( - new HandleCallback() - { - @Override - public Void withHandle(Handle handle) throws Exception - { - doAudit(auditEntry, handle); - return null; - } + handle -> { + doAudit(event, handle); + return 0; } ); } - @VisibleForTesting - ServiceMetricEvent.Builder getAuditMetricEventBuilder(AuditEntry auditEntry) + private ServiceMetricEvent.Builder createMetricEventBuilder(AuditEvent auditEvent) { ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder() - .setDimension("key", auditEntry.getKey()) - .setDimension("type", auditEntry.getType()) - .setDimension("author", auditEntry.getAuditInfo().getAuthor()) - .setDimension("comment", auditEntry.getAuditInfo().getComment()) - .setDimension("remote_address", auditEntry.getAuditInfo().getIp()) - .setDimension("created_date", auditEntry.getAuditTime().toString()); + .setDimension("key", auditEvent.getKey()) + .setDimension("type", auditEvent.getType()) + .setDimension("author", auditEvent.getAuditInfo().getAuthor()) + .setDimension("comment", auditEvent.getAuditInfo().getComment()) + .setDimension("remote_address", auditEvent.getAuditInfo().getIp()) + .setDimension("created_date", auditEvent.getAuditTime().toString()); - if (config.getIncludePayloadAsDimensionInMetric()) { - builder.setDimension("payload", auditEntry.getPayload()); + if (config.isIncludePayloadAsDimensionInMetric()) { + builder.setDimension("payload", auditEvent.getPayloadAsString()); } return builder; } @Override - public void doAudit(AuditEntry auditEntry, Handle handle) throws IOException + public void doAudit(AuditEvent event, Handle handle) throws IOException { - emitter.emit(getAuditMetricEventBuilder(auditEntry).setMetric("config/audit", 1)); - - AuditEntry auditEntryToStore = auditEntry; - if (config.getMaxPayloadSizeBytes() >= 0) { - int payloadSize = jsonMapper.writeValueAsBytes(auditEntry.getPayload()).length; - if (payloadSize > config.getMaxPayloadSizeBytes()) { - auditEntryToStore = AuditEntry.builder() - .key(auditEntry.getKey()) - .type(auditEntry.getType()) - .auditInfo(auditEntry.getAuditInfo()) - .payload(StringUtils.format(PAYLOAD_SKIP_MSG_FORMAT, config.getMaxPayloadSizeBytes())) - .auditTime(auditEntry.getAuditTime()) - .build(); - } - } + emitter.emit(createMetricEventBuilder(event).setMetric("config/audit", 1)); + final AuditRecord record = serdeHelper.processAuditEvent(event); handle.createStatement( StringUtils.format( - "INSERT INTO %s ( audit_key, type, author, comment, created_date, payload) VALUES (:audit_key, :type, :author, :comment, :created_date, :payload)", + "INSERT INTO %s (audit_key, type, author, comment, created_date, payload)" + + " VALUES (:audit_key, :type, :author, :comment, :created_date, :payload)", getAuditTable() ) ) - .bind("audit_key", auditEntry.getKey()) - .bind("type", auditEntry.getType()) - .bind("author", auditEntry.getAuditInfo().getAuthor()) - .bind("comment", auditEntry.getAuditInfo().getComment()) - .bind("created_date", auditEntry.getAuditTime().toString()) - .bind("payload", jsonMapper.writeValueAsBytes(auditEntryToStore)) + .bind("audit_key", record.getKey()) + .bind("type", record.getType()) + .bind("author", record.getAuditInfo().getAuthor()) + .bind("comment", record.getAuditInfo().getComment()) + .bind("created_date", record.getAuditTime().toString()) + .bind("payload", jsonMapper.writeValueAsBytes(record)) .execute(); } @Override - public List fetchAuditHistory(final String key, final String type, Interval interval) + public List fetchAuditHistory(final String key, final String type, Interval interval) { - final Interval theInterval = getIntervalOrDefault(interval); + final Interval theInterval = createAuditHistoryIntervalIfNull(interval); return dbi.withHandle( (Handle handle) -> handle .createQuery( @@ -170,21 +167,19 @@ public List fetchAuditHistory(final String key, final String type, I .bind("type", type) .bind("start_date", theInterval.getStart().toString()) .bind("end_date", theInterval.getEnd().toString()) - .map((index, r, ctx) -> JacksonUtils.readValue(jsonMapper, r.getBytes("payload"), AuditEntry.class)) + .map(resultMapper) .list() ); } - private Interval getIntervalOrDefault(Interval interval) + private Interval createAuditHistoryIntervalIfNull(Interval interval) { - final Interval theInterval; if (interval == null) { DateTime now = DateTimes.nowUtc(); - theInterval = new Interval(now.minus(config.getAuditHistoryMillis()), now); + return new Interval(now.minus(config.getAuditHistoryMillis()), now); } else { - theInterval = interval; + return interval; } - return theInterval; } private int getLimit(int limit) throws IllegalArgumentException @@ -196,9 +191,9 @@ private int getLimit(int limit) throws IllegalArgumentException } @Override - public List fetchAuditHistory(final String type, Interval interval) + public List fetchAuditHistory(final String type, Interval interval) { - final Interval theInterval = getIntervalOrDefault(interval); + final Interval theInterval = createAuditHistoryIntervalIfNull(interval); return dbi.withHandle( (Handle handle) -> handle .createQuery( @@ -211,20 +206,20 @@ public List fetchAuditHistory(final String type, Interval interval) .bind("type", type) .bind("start_date", theInterval.getStart().toString()) .bind("end_date", theInterval.getEnd().toString()) - .map((index, r, ctx) -> JacksonUtils.readValue(jsonMapper, r.getBytes("payload"), AuditEntry.class)) + .map(resultMapper) .list() ); } @Override - public List fetchAuditHistory(final String key, final String type, int limit) + public List fetchAuditHistory(final String key, final String type, int limit) throws IllegalArgumentException { return fetchAuditHistoryLastEntries(key, type, limit); } @Override - public List fetchAuditHistory(final String type, int limit) + public List fetchAuditHistory(final String type, int limit) throws IllegalArgumentException { return fetchAuditHistoryLastEntries(null, type, limit); @@ -248,7 +243,7 @@ public int removeAuditLogsOlderThan(final long timestamp) ); } - private List fetchAuditHistoryLastEntries(final String key, final String type, int limit) + private List fetchAuditHistoryLastEntries(final String key, final String type, int limit) throws IllegalArgumentException { final int theLimit = getLimit(limit); @@ -268,10 +263,27 @@ private List fetchAuditHistoryLastEntries(final String key, final St return query .bind("type", type) .setMaxRows(theLimit) - .map((index, r, ctx) -> JacksonUtils.readValue(jsonMapper, r.getBytes("payload"), AuditEntry.class)) + .map(resultMapper) .list(); } ); } + private class AuditEventMapper implements ResultSetMapper + { + @Override + public AuditEvent map(int index, ResultSet r, StatementContext ctx) throws SQLException + { + // Read the record and convert to an AuditEvent that can deserialize the payload on-demand + AuditRecord record = JacksonUtils.readValue(jsonMapper, r.getBytes("payload"), AuditRecord.class); + return new AuditEvent( + record.getKey(), + record.getType(), + record.getAuditInfo(), + record.getPayload(), + record.getAuditTime() + ); + } + } + } diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java index 8509e06c1029..2392d5f66e9a 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java @@ -20,44 +20,47 @@ package org.apache.druid.server.audit; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.audit.AuditManagerConfig; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.HumanReadableBytesRange; /** */ -public class SQLAuditManagerConfig +public class SQLAuditManagerConfig implements AuditManagerConfig { @JsonProperty - private long auditHistoryMillis = 7 * 24 * 60 * 60 * 1000L; // 1 WEEK + private final long auditHistoryMillis = 7 * 24 * 60 * 60 * 1000L; // 1 WEEK @JsonProperty - private boolean includePayloadAsDimensionInMetric = false; + private final boolean includePayloadAsDimensionInMetric = false; @JsonProperty @HumanReadableBytesRange( min = -1, message = "maxPayloadSizeBytes must either be -1 (for disabling the check) or a non negative number" ) - private HumanReadableBytes maxPayloadSizeBytes = HumanReadableBytes.valueOf(-1); + private final HumanReadableBytes maxPayloadSizeBytes = HumanReadableBytes.valueOf(-1); @JsonProperty - private boolean skipNullField = false; + private final boolean skipNullField = false; public long getAuditHistoryMillis() { return auditHistoryMillis; } - public boolean getIncludePayloadAsDimensionInMetric() + public boolean isIncludePayloadAsDimensionInMetric() { return includePayloadAsDimensionInMetric; } + @Override public long getMaxPayloadSizeBytes() { return maxPayloadSizeBytes.getBytes(); } + @Override public boolean isSkipNullField() { return skipNullField; diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerProvider.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerProvider.java deleted file mode 100644 index c1d87399f591..000000000000 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerProvider.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.server.audit; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Supplier; -import com.google.inject.Inject; -import org.apache.druid.audit.AuditManager; -import org.apache.druid.guice.annotations.Json; -import org.apache.druid.java.util.common.lifecycle.Lifecycle; -import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.metadata.MetadataStorageTablesConfig; -import org.apache.druid.metadata.SQLMetadataConnector; - -public class SQLAuditManagerProvider implements AuditManagerProvider -{ - private final Supplier dbTables; - private final SQLMetadataConnector connector; - private final Lifecycle lifecycle; - private final ServiceEmitter emitter; - private final ObjectMapper mapper; - private final SQLAuditManagerConfig config; - - @Inject - public SQLAuditManagerProvider( - Supplier dbTables, - SQLMetadataConnector connector, - Lifecycle lifecycle, - ServiceEmitter emitter, - @Json ObjectMapper mapper, - SQLAuditManagerConfig config - ) - { - this.dbTables = dbTables; - this.connector = connector; - this.lifecycle = lifecycle; - this.emitter = emitter; - this.mapper = mapper; - this.config = config; - } - - @Override - public AuditManager get() - { - try { - lifecycle.addMaybeStartHandler( - new Lifecycle.Handler() - { - @Override - public void start() - { - connector.createAuditTable(); - } - - @Override - public void stop() - { - - } - } - ); - } - catch (Exception e) { - throw new RuntimeException(e); - } - - return new SQLAuditManager(connector, dbTables, emitter, mapper, config); - } -} diff --git a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java index d8ae8e3aaa3d..910ee8bd8508 100644 --- a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java +++ b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; -import org.apache.druid.audit.AuditEntry; +import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.common.config.ConfigManager.SetResult; @@ -166,7 +166,7 @@ public Response getCompactionConfigHistory( { Interval theInterval = interval == null ? null : Intervals.of(interval); try { - List auditEntries; + List auditEntries; if (theInterval == null && count != null) { auditEntries = auditManager.fetchAuditHistory( CoordinatorCompactionConfig.CONFIG_KEY, @@ -181,9 +181,9 @@ public Response getCompactionConfigHistory( ); } DataSourceCompactionConfigHistory history = new DataSourceCompactionConfigHistory(dataSource); - for (AuditEntry audit : auditEntries) { + for (AuditEvent audit : auditEntries) { CoordinatorCompactionConfig coordinatorCompactionConfig = configManager.convertBytesToCompactionConfig( - audit.getPayload().getBytes(StandardCharsets.UTF_8) + audit.getPayloadAsString().getBytes(StandardCharsets.UTF_8) ); history.add(coordinatorCompactionConfig, audit.getAuditInfo(), audit.getAuditTime()); } diff --git a/server/src/main/java/org/apache/druid/server/http/RulesResource.java b/server/src/main/java/org/apache/druid/server/http/RulesResource.java index beb223a6cc65..aafaa8471d88 100644 --- a/server/src/main/java/org/apache/druid/server/http/RulesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/RulesResource.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; -import org.apache.druid.audit.AuditEntry; +import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.java.util.common.Intervals; @@ -161,7 +161,7 @@ public Response getDatasourceRuleHistory( } } - private List getRuleHistory( + private List getRuleHistory( final String dataSourceName, final String interval, final Integer count diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java index 20ffdb81188a..84332e42c6e0 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java @@ -25,7 +25,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.apache.druid.audit.AuditEntry; +import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.client.DruidServer; @@ -74,11 +74,10 @@ public void setUp() tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); connector.createAuditTable(); auditManager = new SQLAuditManager( - connector, + new SQLAuditManagerConfig(), null, connector, Suppliers.ofInstance(tablesConfig), new NoopServiceEmitter(), - mapper, - new SQLAuditManagerConfig() + mapper ); connector.createRulesTable(); @@ -184,13 +183,13 @@ public void testAuditEntryCreated() throws Exception Assert.assertEquals(rules, ruleManager.getRules(DATASOURCE)); // verify audit entry is created - List auditEntries = auditManager.fetchAuditHistory(DATASOURCE, "rules", null); + List auditEntries = auditManager.fetchAuditHistory(DATASOURCE, "rules", null); Assert.assertEquals(1, auditEntries.size()); - AuditEntry entry = auditEntries.get(0); + AuditEvent entry = auditEntries.get(0); Assert.assertEquals( rules, - mapper.readValue(entry.getPayload(), new TypeReference>() {}) + mapper.readValue(entry.getPayloadAsString(), new TypeReference>() {}) ); Assert.assertEquals(auditInfo, entry.getAuditInfo()); Assert.assertEquals(DATASOURCE, entry.getKey()); @@ -218,12 +217,12 @@ public void testFetchAuditEntriesForAllDataSources() throws Exception Assert.assertEquals(rules, ruleManager.getRules("test_dataSource2")); // test fetch audit entries - List auditEntries = auditManager.fetchAuditHistory("rules", null); + List auditEntries = auditManager.fetchAuditHistory("rules", null); Assert.assertEquals(2, auditEntries.size()); - for (AuditEntry entry : auditEntries) { + for (AuditEvent entry : auditEntries) { Assert.assertEquals( rules, - mapper.readValue(entry.getPayload(), new TypeReference>() {}) + mapper.readValue(entry.getPayloadAsString(), new TypeReference>() {}) ); Assert.assertEquals(auditInfo, entry.getAuditInfo()); } diff --git a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java index 7e003c346211..d4c9510e1627 100644 --- a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java @@ -19,299 +19,208 @@ package org.apache.druid.server.audit; -import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.druid.audit.AuditEntry; +import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditInfo; -import org.apache.druid.audit.AuditManager; import org.apache.druid.common.config.ConfigSerde; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.metadata.TestDerbyConnector; -import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.joda.time.DateTime; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; @RunWith(MockitoJUnitRunner.class) public class SQLAuditManagerTest { @Rule - public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule = new TestDerbyConnector.DerbyConnectorRule(); + public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule + = new TestDerbyConnector.DerbyConnectorRule(); private TestDerbyConnector connector; - private AuditManager auditManager; - private ConfigSerde stringConfigSerde; + private SQLAuditManager auditManager; + private StubServiceEmitter serviceEmitter; private final ObjectMapper mapper = new DefaultObjectMapper(); + private final ObjectMapper mapperSkipNull + = new DefaultObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL); @Before public void setUp() { + serviceEmitter = new StubServiceEmitter("audit-test", "localhost"); connector = derbyConnectorRule.getConnector(); connector.createAuditTable(); - auditManager = new SQLAuditManager( + auditManager = createAuditManager(new SQLAuditManagerConfig()); + } + + private SQLAuditManager createAuditManager(SQLAuditManagerConfig config) + { + return new SQLAuditManager( + config, + new AuditSerdeHelper(config, mapper, mapperSkipNull), connector, derbyConnectorRule.metadataTablesConfigSupplier(), - new NoopServiceEmitter(), - mapper, - new SQLAuditManagerConfig() + serviceEmitter, + mapper ); - stringConfigSerde = new ConfigSerde() - { - @Override - public byte[] serialize(String obj) - { - try { - return mapper.writeValueAsBytes(obj); - } - catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - @Override - public String serializeToString(String obj, boolean skipNull) - { - // In our test, payload Object is already a String - // So to serialize to String, we just return a String - return obj; - } - - @Override - public String deserialize(byte[] bytes) - { - return JacksonUtils.readValue(mapper, bytes, String.class); - } - }; } @Test - public void testAuditMetricEventBuilderConfig() + public void testAuditMetricEventWithPayload() throws IOException { - AuditEntry entry = new AuditEntry( - "testKey", - "testType", - new AuditInfo("testAuthor", "testComment", "127.0.0.1"), - "testPayload", - DateTimes.of("2013-01-01T00:00:00Z") - ); - - SQLAuditManager auditManagerWithPayloadAsDimension = new SQLAuditManager( - connector, - derbyConnectorRule.metadataTablesConfigSupplier(), - new NoopServiceEmitter(), - mapper, + SQLAuditManager auditManager = createAuditManager( new SQLAuditManagerConfig() { @Override - public boolean getIncludePayloadAsDimensionInMetric() + public boolean isIncludePayloadAsDimensionInMetric() { return true; } } ); - ServiceMetricEvent.Builder auditEntryBuilder = ((SQLAuditManager) auditManager).getAuditMetricEventBuilder(entry); - final String payloadDimensionKey = "payload"; - Assert.assertNull(auditEntryBuilder.getDimension(payloadDimensionKey)); + final AuditEvent auditEvent = createAuditEvent("testKey", "testType", DateTimes.nowUtc()); + auditManager.doAudit(auditEvent); + + Map> metricEvents = serviceEmitter.getMetricEvents(); + Assert.assertEquals(1, metricEvents.size()); + + List auditMetricEvents = metricEvents.get("config/audit"); + Assert.assertNotNull(auditMetricEvents); + Assert.assertEquals(1, auditMetricEvents.size()); - ServiceMetricEvent.Builder auditEntryBuilderWithPayload = auditManagerWithPayloadAsDimension.getAuditMetricEventBuilder(entry); - Assert.assertEquals("testPayload", auditEntryBuilderWithPayload.getDimension(payloadDimensionKey)); + ServiceMetricEvent metric = auditMetricEvents.get(0); + + final AuditEvent entry = lookupAuditEntryForKey("testKey"); + Assert.assertNotNull(entry); + Assert.assertEquals(entry.getKey(), metric.getUserDims().get("key")); + Assert.assertEquals(entry.getType(), metric.getUserDims().get("type")); + Assert.assertEquals(entry.getPayloadAsString(), metric.getUserDims().get("payload")); + Assert.assertEquals(entry.getAuditInfo().getAuthor(), metric.getUserDims().get("author")); + Assert.assertEquals(entry.getAuditInfo().getComment(), metric.getUserDims().get("comment")); + Assert.assertEquals(entry.getAuditInfo().getIp(), metric.getUserDims().get("remote_address")); } @Test(timeout = 60_000L) public void testCreateAuditEntry() throws IOException { - String entry1Key = "testKey"; - String entry1Type = "testType"; - AuditInfo entry1AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - String entry1Payload = "testPayload"; - - auditManager.doAudit(entry1Key, entry1Type, entry1AuditInfo, entry1Payload, stringConfigSerde); - - byte[] payload = connector.lookup( - derbyConnectorRule.metadataTablesConfigSupplier().get().getAuditTable(), - "audit_key", - "payload", - "testKey" - ); - AuditEntry dbEntry = mapper.readValue(payload, AuditEntry.class); - Assert.assertEquals(entry1Key, dbEntry.getKey()); - Assert.assertEquals(entry1Payload, dbEntry.getPayload()); - Assert.assertEquals(entry1Type, dbEntry.getType()); - Assert.assertEquals(entry1AuditInfo, dbEntry.getAuditInfo()); + final AuditEvent auditEvent = createAuditEvent("key1", "type1", DateTimes.nowUtc()); + auditManager.doAudit(auditEvent); + + AuditEvent dbEvent = lookupAuditEntryForKey(auditEvent.getKey()); + Assert.assertEquals(auditEvent, dbEvent); + + // Verify emitted metrics + Map> metricEvents = serviceEmitter.getMetricEvents(); + Assert.assertEquals(1, metricEvents.size()); + + List auditMetricEvents = metricEvents.get("config/audit"); + Assert.assertNotNull(auditMetricEvents); + Assert.assertEquals(1, auditMetricEvents.size()); + + ServiceMetricEvent metric = auditMetricEvents.get(0); + Assert.assertEquals(dbEvent.getKey(), metric.getUserDims().get("key")); + Assert.assertEquals(dbEvent.getType(), metric.getUserDims().get("type")); + Assert.assertNull(metric.getUserDims().get("payload")); + Assert.assertEquals(dbEvent.getAuditInfo().getAuthor(), metric.getUserDims().get("author")); + Assert.assertEquals(dbEvent.getAuditInfo().getComment(), metric.getUserDims().get("comment")); + Assert.assertEquals(dbEvent.getAuditInfo().getIp(), metric.getUserDims().get("remote_address")); } @Test(timeout = 60_000L) public void testFetchAuditHistory() { - String entry1Key = "testKey"; - String entry1Type = "testType"; - AuditInfo entry1AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - String entry1Payload = "testPayload"; + final AuditEvent event = createAuditEvent("testKey", "testType", DateTimes.nowUtc()); + auditManager.doAudit(event); + auditManager.doAudit(event); - auditManager.doAudit(entry1Key, entry1Type, entry1AuditInfo, entry1Payload, stringConfigSerde); - auditManager.doAudit(entry1Key, entry1Type, entry1AuditInfo, entry1Payload, stringConfigSerde); - - List auditEntries = auditManager.fetchAuditHistory( + List auditEntries = auditManager.fetchAuditHistory( "testKey", "testType", Intervals.of("2000-01-01T00:00:00Z/2100-01-03T00:00:00Z") ); - Assert.assertEquals(2, auditEntries.size()); - - Assert.assertEquals(entry1Key, auditEntries.get(0).getKey()); - Assert.assertEquals(entry1Payload, auditEntries.get(0).getPayload()); - Assert.assertEquals(entry1Type, auditEntries.get(0).getType()); - Assert.assertEquals(entry1AuditInfo, auditEntries.get(0).getAuditInfo()); - Assert.assertEquals(entry1Key, auditEntries.get(1).getKey()); - Assert.assertEquals(entry1Payload, auditEntries.get(1).getPayload()); - Assert.assertEquals(entry1Type, auditEntries.get(1).getType()); - Assert.assertEquals(entry1AuditInfo, auditEntries.get(1).getAuditInfo()); + Assert.assertEquals(2, auditEntries.size()); + Assert.assertEquals(event, auditEntries.get(0)); + Assert.assertEquals(event, auditEntries.get(1)); } @Test(timeout = 60_000L) public void testFetchAuditHistoryByKeyAndTypeWithLimit() { - String entry1Key = "testKey1"; - String entry1Type = "testType"; - AuditInfo entry1AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - String entry1Payload = "testPayload"; - - String entry2Key = "testKey2"; - String entry2Type = "testType"; - AuditInfo entry2AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - String entry2Payload = "testPayload"; - - auditManager.doAudit(entry1Key, entry1Type, entry1AuditInfo, entry1Payload, stringConfigSerde); - auditManager.doAudit(entry2Key, entry2Type, entry2AuditInfo, entry2Payload, stringConfigSerde); - List auditEntries = auditManager.fetchAuditHistory("testKey1", "testType", 1); + final AuditEvent entry1 = createAuditEvent("key1", "type1", DateTimes.nowUtc()); + final AuditEvent entry2 = createAuditEvent("key2", "type2", DateTimes.nowUtc()); + + auditManager.doAudit(entry1); + auditManager.doAudit(entry2); + + List auditEntries = auditManager.fetchAuditHistory(entry1.getKey(), entry1.getType(), 1); Assert.assertEquals(1, auditEntries.size()); - Assert.assertEquals(entry1Key, auditEntries.get(0).getKey()); - Assert.assertEquals(entry1Payload, auditEntries.get(0).getPayload()); - Assert.assertEquals(entry1Type, auditEntries.get(0).getType()); - Assert.assertEquals(entry1AuditInfo, auditEntries.get(0).getAuditInfo()); + Assert.assertEquals(entry1, auditEntries.get(0)); } @Test(timeout = 60_000L) public void testRemoveAuditLogsOlderThanWithEntryOlderThanTime() throws IOException { - String entry1Key = "testKey"; - String entry1Type = "testType"; - AuditInfo entry1AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - String entry1Payload = "testPayload"; + final AuditEvent entry = createAuditEvent("key1", "type1", DateTimes.nowUtc()); + auditManager.doAudit(entry); - auditManager.doAudit(entry1Key, entry1Type, entry1AuditInfo, entry1Payload, stringConfigSerde); - byte[] payload = connector.lookup( - derbyConnectorRule.metadataTablesConfigSupplier().get().getAuditTable(), - "audit_key", - "payload", - "testKey" - ); - AuditEntry dbEntry = mapper.readValue(payload, AuditEntry.class); - Assert.assertEquals(entry1Key, dbEntry.getKey()); - Assert.assertEquals(entry1Payload, dbEntry.getPayload()); - Assert.assertEquals(entry1Type, dbEntry.getType()); - Assert.assertEquals(entry1AuditInfo, dbEntry.getAuditInfo()); + AuditEvent dbEntry = lookupAuditEntryForKey(entry.getKey()); + Assert.assertEquals(entry, dbEntry); - // Do delete + // Verify that the audit entry gets deleted auditManager.removeAuditLogsOlderThan(System.currentTimeMillis()); - // Verify the delete - payload = connector.lookup( - derbyConnectorRule.metadataTablesConfigSupplier().get().getAuditTable(), - "audit_key", - "payload", - "testKey" - ); - Assert.assertNull(payload); + Assert.assertNull(lookupAuditEntryForKey(entry.getKey())); } @Test(timeout = 60_000L) public void testRemoveAuditLogsOlderThanWithEntryNotOlderThanTime() throws IOException { - String entry1Key = "testKey"; - String entry1Type = "testType"; - AuditInfo entry1AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - String entry1Payload = "testPayload"; + AuditEvent entry = createAuditEvent("key", "type", DateTimes.nowUtc()); + auditManager.doAudit(entry); - auditManager.doAudit(entry1Key, entry1Type, entry1AuditInfo, entry1Payload, stringConfigSerde); - byte[] payload = connector.lookup( - derbyConnectorRule.metadataTablesConfigSupplier().get().getAuditTable(), - "audit_key", - "payload", - "testKey" - ); - AuditEntry dbEntry = mapper.readValue(payload, AuditEntry.class); - Assert.assertEquals(entry1Key, dbEntry.getKey()); - Assert.assertEquals(entry1Payload, dbEntry.getPayload()); - Assert.assertEquals(entry1Type, dbEntry.getType()); - Assert.assertEquals(entry1AuditInfo, dbEntry.getAuditInfo()); - // Do delete + AuditEvent dbEntry = lookupAuditEntryForKey(entry.getKey()); + Assert.assertEquals(entry, dbEntry); + + // Delete old audit logs auditManager.removeAuditLogsOlderThan(DateTimes.of("2012-01-01T00:00:00Z").getMillis()); - // Verify that entry was not delete - payload = connector.lookup( - derbyConnectorRule.metadataTablesConfigSupplier().get().getAuditTable(), - "audit_key", - "payload", - "testKey" - ); - dbEntry = mapper.readValue(payload, AuditEntry.class); - Assert.assertEquals(entry1Key, dbEntry.getKey()); - Assert.assertEquals(entry1Payload, dbEntry.getPayload()); - Assert.assertEquals(entry1Type, dbEntry.getType()); - Assert.assertEquals(entry1AuditInfo, dbEntry.getAuditInfo()); + + dbEntry = lookupAuditEntryForKey(entry.getKey()); + Assert.assertEquals(entry, dbEntry); } @Test(timeout = 60_000L) public void testFetchAuditHistoryByTypeWithLimit() { - String entry1Key = "testKey"; - String entry1Type = "testType"; - AuditInfo entry1AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - String entry1Payload = "testPayload1"; - - String entry2Key = "testKey"; - String entry2Type = "testType"; - AuditInfo entry2AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - String entry2Payload = "testPayload2"; - - String entry3Key = "testKey"; - String entry3Type = "testType"; - AuditInfo entry3AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - String entry3Payload = "testPayload3"; - - auditManager.doAudit(entry1Key, entry1Type, entry1AuditInfo, entry1Payload, stringConfigSerde); - auditManager.doAudit(entry2Key, entry2Type, entry2AuditInfo, entry2Payload, stringConfigSerde); - auditManager.doAudit(entry3Key, entry3Type, entry3AuditInfo, entry3Payload, stringConfigSerde); - - List auditEntries = auditManager.fetchAuditHistory("testType", 2); + final AuditEvent entry1 = createAuditEvent("testKey", "testType", DateTimes.nowUtc()); + final AuditEvent entry2 = createAuditEvent("testKey", "testType", DateTimes.nowUtc()); + final AuditEvent entry3 = createAuditEvent("testKey", "testType", DateTimes.nowUtc()); + + auditManager.doAudit(entry1); + auditManager.doAudit(entry2); + auditManager.doAudit(entry3); + + List auditEntries = auditManager.fetchAuditHistory("testType", 2); Assert.assertEquals(2, auditEntries.size()); - Assert.assertEquals(entry3Key, auditEntries.get(0).getKey()); - Assert.assertEquals(entry3Payload, auditEntries.get(0).getPayload()); - Assert.assertEquals(entry3Type, auditEntries.get(0).getType()); - Assert.assertEquals(entry3AuditInfo, auditEntries.get(0).getAuditInfo()); - - Assert.assertEquals(entry2Key, auditEntries.get(1).getKey()); - Assert.assertEquals(entry2Payload, auditEntries.get(1).getPayload()); - Assert.assertEquals(entry2Type, auditEntries.get(1).getType()); - Assert.assertEquals(entry2AuditInfo, auditEntries.get(1).getAuditInfo()); + Assert.assertEquals(entry1, auditEntries.get(0)); + Assert.assertEquals(entry2, auditEntries.get(1)); } @Test(expected = IllegalArgumentException.class, timeout = 10_000L) @@ -329,12 +238,8 @@ public void testFetchAuditHistoryLimitZero() @Test(timeout = 60_000L) public void testCreateAuditEntryWithPayloadOverSkipPayloadLimit() throws IOException { - int maxPayloadSize = 10; - SQLAuditManager auditManagerWithMaxPayloadSizeBytes = new SQLAuditManager( - connector, - derbyConnectorRule.metadataTablesConfigSupplier(), - new NoopServiceEmitter(), - mapper, + final int maxPayloadSize = 10; + final SQLAuditManager auditManager = createAuditManager( new SQLAuditManagerConfig() { @Override @@ -345,42 +250,25 @@ public long getMaxPayloadSizeBytes() } ); - String entry1Key = "testKey"; - String entry1Type = "testType"; - AuditInfo entry1AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - String entry1Payload = "payload audit to store"; - - auditManagerWithMaxPayloadSizeBytes.doAudit( - entry1Key, - entry1Type, - entry1AuditInfo, - entry1Payload, - stringConfigSerde - ); + final AuditEvent entry = createAuditEvent("key", "type", DateTimes.nowUtc()); + auditManager.doAudit(entry); - byte[] payload = connector.lookup( - derbyConnectorRule.metadataTablesConfigSupplier().get().getAuditTable(), - "audit_key", - "payload", - "testKey" + // Verify that all the fields are the same except for the payload + AuditEvent dbEntry = lookupAuditEntryForKey(entry.getKey()); + Assert.assertEquals(entry.getKey(), dbEntry.getKey()); + // Assert.assertNotEquals(entry.getPayload(), dbEntry.getPayload()); + Assert.assertEquals( + "Payload truncated as it exceeds 'druid.audit.manager.maxPayloadSizeBytes'[10].", + dbEntry.getPayloadAsString() ); - - AuditEntry dbEntry = mapper.readValue(payload, AuditEntry.class); - Assert.assertEquals(entry1Key, dbEntry.getKey()); - Assert.assertNotEquals(entry1Payload, dbEntry.getPayload()); - Assert.assertEquals(StringUtils.format(AuditManager.PAYLOAD_SKIP_MSG_FORMAT, maxPayloadSize), dbEntry.getPayload()); - Assert.assertEquals(entry1Type, dbEntry.getType()); - Assert.assertEquals(entry1AuditInfo, dbEntry.getAuditInfo()); + Assert.assertEquals(entry.getType(), dbEntry.getType()); + Assert.assertEquals(entry.getAuditInfo(), dbEntry.getAuditInfo()); } @Test(timeout = 60_000L) public void testCreateAuditEntryWithPayloadUnderSkipPayloadLimit() throws IOException { - SQLAuditManager auditManagerWithMaxPayloadSizeBytes = new SQLAuditManager( - connector, - derbyConnectorRule.metadataTablesConfigSupplier(), - new NoopServiceEmitter(), - mapper, + SQLAuditManager auditManager = createAuditManager( new SQLAuditManagerConfig() { @Override @@ -390,42 +278,20 @@ public long getMaxPayloadSizeBytes() } } ); - String entry1Key = "testKey"; - String entry1Type = "testType"; - AuditInfo entry1AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - String entry1Payload = "payload audit to store"; - - auditManagerWithMaxPayloadSizeBytes.doAudit( - entry1Key, - entry1Type, - entry1AuditInfo, - entry1Payload, - stringConfigSerde - ); - byte[] payload = connector.lookup( - derbyConnectorRule.metadataTablesConfigSupplier().get().getAuditTable(), - "audit_key", - "payload", - "testKey" - ); + final AuditEvent entry = createAuditEvent("key", "type", DateTimes.nowUtc()); + auditManager.doAudit(entry); - AuditEntry dbEntry = mapper.readValue(payload, AuditEntry.class); - Assert.assertEquals(entry1Key, dbEntry.getKey()); - Assert.assertEquals(entry1Payload, dbEntry.getPayload()); - Assert.assertEquals(entry1Type, dbEntry.getType()); - Assert.assertEquals(entry1AuditInfo, dbEntry.getAuditInfo()); + // Verify that the actual payload has been persisted + AuditEvent dbEntry = lookupAuditEntryForKey(entry.getKey()); + Assert.assertEquals(entry, dbEntry); } @Test(timeout = 60_000L) - public void testCreateAuditEntryWithSkipNullConfigTrue() + public void testCreateAuditEntryWithSkipNullsInPayload() throws IOException { - ConfigSerde> mockConfigSerde = Mockito.mock(ConfigSerde.class); - SQLAuditManager auditManagerWithSkipNull = new SQLAuditManager( - connector, - derbyConnectorRule.metadataTablesConfigSupplier(), - new NoopServiceEmitter(), - mapper, + final ConfigSerde> mockConfigSerde = Mockito.mock(ConfigSerde.class); + final SQLAuditManager auditManagerSkipNull = createAuditManager( new SQLAuditManagerConfig() { @Override @@ -436,16 +302,23 @@ public boolean isSkipNullField() } ); - String entry1Key = "test1Key"; - String entry1Type = "test1Type"; - AuditInfo entry1AuditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); - // Entry 1 payload has a null field for one of the property - Map entryPayload1WithNull = new HashMap<>(); - entryPayload1WithNull.put("version", "x"); - entryPayload1WithNull.put("something", null); + AuditInfo auditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); + + final Map payloadMap = new TreeMap<>(); + payloadMap.put("version", "x"); + payloadMap.put("something", null); - auditManagerWithSkipNull.doAudit(entry1Key, entry1Type, entry1AuditInfo, entryPayload1WithNull, mockConfigSerde); - Mockito.verify(mockConfigSerde).serializeToString(ArgumentMatchers.eq(entryPayload1WithNull), ArgumentMatchers.eq(true)); + auditManager.doAudit( + AuditEvent.builder().key("key1").type("type1").auditInfo(auditInfo).payload(payloadMap).build() + ); + AuditEvent entryWithNulls = lookupAuditEntryForKey("key1"); + Assert.assertEquals("{\"something\":null,\"version\":\"x\"}", entryWithNulls.getPayloadAsString()); + + auditManagerSkipNull.doAudit( + AuditEvent.builder().key("key2").type("type2").auditInfo(auditInfo).payload(payloadMap).build() + ); + AuditEvent entryWithoutNulls = lookupAuditEntryForKey("key2"); + Assert.assertEquals("{\"version\":\"x\"}", entryWithoutNulls.getPayloadAsString()); } @After @@ -456,12 +329,37 @@ public void cleanup() private void dropTable(final String tableName) { - Assert.assertEquals( - 0, - connector.getDBI().withHandle( - handle -> handle.createStatement(StringUtils.format("DROP TABLE %s", tableName)) + int rowsAffected = connector.getDBI().withHandle( + handle -> handle.createStatement(StringUtils.format("DROP TABLE %s", tableName)) .execute() - ).intValue() ); + Assert.assertEquals(0, rowsAffected); + } + + private AuditEvent lookupAuditEntryForKey(String key) throws IOException + { + byte[] payload = connector.lookup( + derbyConnectorRule.metadataTablesConfigSupplier().get().getAuditTable(), + "audit_key", + "payload", + key + ); + + if (payload == null) { + return null; + } else { + return mapper.readValue(payload, AuditEvent.class); + } + } + + private AuditEvent createAuditEvent(String key, String type, DateTime auditTime) + { + return AuditEvent.builder() + .key(key) + .type(type) + .payloadAsString(StringUtils.format("Test payload for key[%s], type[%s]", key, type)) + .auditInfo(new AuditInfo("author", "comment", "127.0.0.1")) + .auditTime(auditTime) + .build(); } } diff --git a/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java index 5b52e3d30ee9..c2cb5ed2c571 100644 --- a/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java @@ -20,7 +20,7 @@ package org.apache.druid.server.http; import com.google.common.collect.ImmutableList; -import org.apache.druid.audit.AuditEntry; +import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.java.util.common.DateTimes; @@ -51,7 +51,7 @@ public void setUp() @Test public void testGetDatasourceRuleHistoryWithCount() { - AuditEntry entry1 = new AuditEntry( + AuditEvent entry1 = new AuditEvent( "testKey", "testType", new AuditInfo( @@ -62,7 +62,7 @@ public void testGetDatasourceRuleHistoryWithCount() "testPayload", DateTimes.of("2013-01-02T00:00:00Z") ); - AuditEntry entry2 = new AuditEntry( + AuditEvent entry2 = new AuditEvent( "testKey", "testType", new AuditInfo( @@ -81,7 +81,7 @@ public void testGetDatasourceRuleHistoryWithCount() RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); Response response = rulesResource.getDatasourceRuleHistory("datasource1", null, 2); - List rulesHistory = (List) response.getEntity(); + List rulesHistory = (List) response.getEntity(); Assert.assertEquals(2, rulesHistory.size()); Assert.assertEquals(entry1, rulesHistory.get(0)); Assert.assertEquals(entry2, rulesHistory.get(1)); @@ -94,7 +94,7 @@ public void testGetDatasourceRuleHistoryWithInterval() { String interval = "P2D/2013-01-02T00:00:00Z"; Interval theInterval = Intervals.of(interval); - AuditEntry entry1 = new AuditEntry( + AuditEvent entry1 = new AuditEvent( "testKey", "testType", new AuditInfo( @@ -105,7 +105,7 @@ public void testGetDatasourceRuleHistoryWithInterval() "testPayload", DateTimes.of("2013-01-02T00:00:00Z") ); - AuditEntry entry2 = new AuditEntry( + AuditEvent entry2 = new AuditEvent( "testKey", "testType", new AuditInfo( @@ -124,7 +124,7 @@ public void testGetDatasourceRuleHistoryWithInterval() RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); Response response = rulesResource.getDatasourceRuleHistory("datasource1", interval, null); - List rulesHistory = (List) response.getEntity(); + List rulesHistory = (List) response.getEntity(); Assert.assertEquals(2, rulesHistory.size()); Assert.assertEquals(entry1, rulesHistory.get(0)); Assert.assertEquals(entry2, rulesHistory.get(1)); @@ -154,7 +154,7 @@ public void testGetDatasourceRuleHistoryWithWrongCount() @Test public void testGetAllDatasourcesRuleHistoryWithCount() { - AuditEntry entry1 = new AuditEntry( + AuditEvent entry1 = new AuditEvent( "testKey", "testType", new AuditInfo( @@ -165,7 +165,7 @@ public void testGetAllDatasourcesRuleHistoryWithCount() "testPayload", DateTimes.of("2013-01-02T00:00:00Z") ); - AuditEntry entry2 = new AuditEntry( + AuditEvent entry2 = new AuditEvent( "testKey", "testType", new AuditInfo( @@ -184,7 +184,7 @@ public void testGetAllDatasourcesRuleHistoryWithCount() RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); Response response = rulesResource.getDatasourceRuleHistory(null, 2); - List rulesHistory = (List) response.getEntity(); + List rulesHistory = (List) response.getEntity(); Assert.assertEquals(2, rulesHistory.size()); Assert.assertEquals(entry1, rulesHistory.get(0)); Assert.assertEquals(entry2, rulesHistory.get(1)); @@ -197,7 +197,7 @@ public void testGetAllDatasourcesRuleHistoryWithInterval() { String interval = "P2D/2013-01-02T00:00:00Z"; Interval theInterval = Intervals.of(interval); - AuditEntry entry1 = new AuditEntry( + AuditEvent entry1 = new AuditEvent( "testKey", "testType", new AuditInfo( @@ -208,7 +208,7 @@ public void testGetAllDatasourcesRuleHistoryWithInterval() "testPayload", DateTimes.of("2013-01-02T00:00:00Z") ); - AuditEntry entry2 = new AuditEntry( + AuditEvent entry2 = new AuditEvent( "testKey", "testType", new AuditInfo( @@ -227,7 +227,7 @@ public void testGetAllDatasourcesRuleHistoryWithInterval() RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); Response response = rulesResource.getDatasourceRuleHistory(interval, null); - List rulesHistory = (List) response.getEntity(); + List rulesHistory = (List) response.getEntity(); Assert.assertEquals(2, rulesHistory.size()); Assert.assertEquals(entry1, rulesHistory.get(0)); Assert.assertEquals(entry2, rulesHistory.get(1)); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index db7c3bfdfd48..c7a127e98cd9 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -36,7 +36,6 @@ import com.google.inject.name.Names; import com.google.inject.util.Providers; import org.apache.curator.framework.CuratorFramework; -import org.apache.druid.audit.AuditManager; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DirectDruidClientFactory; @@ -97,7 +96,6 @@ import org.apache.druid.segment.metadata.SegmentMetadataQuerySegmentWalker; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QuerySchedulerProvider; -import org.apache.druid.server.audit.AuditManagerProvider; import org.apache.druid.server.coordinator.CoordinatorConfigManager; import org.apache.druid.server.coordinator.DruidCoordinator; import org.apache.druid.server.coordinator.DruidCoordinatorConfig; @@ -248,10 +246,6 @@ public void configure(Binder binder) .toProvider(MetadataRuleManagerProvider.class) .in(ManageLifecycle.class); - binder.bind(AuditManager.class) - .toProvider(AuditManagerProvider.class) - .in(ManageLifecycle.class); - binder.bind(LookupCoordinatorManager.class).in(LazySingleton.class); binder.bind(CoordinatorConfigManager.class); diff --git a/services/src/main/java/org/apache/druid/cli/CliOverlord.java b/services/src/main/java/org/apache/druid/cli/CliOverlord.java index d72d5eb24737..f48c9be01b61 100644 --- a/services/src/main/java/org/apache/druid/cli/CliOverlord.java +++ b/services/src/main/java/org/apache/druid/cli/CliOverlord.java @@ -37,7 +37,6 @@ import com.google.inject.name.Names; import com.google.inject.servlet.GuiceFilter; import com.google.inject.util.Providers; -import org.apache.druid.audit.AuditManager; import org.apache.druid.client.indexing.IndexingService; import org.apache.druid.discovery.NodeRole; import org.apache.druid.guice.IndexingServiceFirehoseModule; @@ -110,7 +109,6 @@ import org.apache.druid.segment.realtime.appenderator.DummyForInjectionAppenderatorsManager; import org.apache.druid.segment.realtime.firehose.ChatHandlerProvider; import org.apache.druid.segment.realtime.firehose.NoopChatHandlerProvider; -import org.apache.druid.server.audit.AuditManagerProvider; import org.apache.druid.server.coordinator.CoordinatorOverlordServiceConfig; import org.apache.druid.server.http.RedirectFilter; import org.apache.druid.server.http.RedirectInfo; @@ -246,10 +244,6 @@ public void configure(Binder binder) binder.install(runnerConfigModule()); configureOverlordHelpers(binder); - binder.bind(AuditManager.class) - .toProvider(AuditManagerProvider.class) - .in(ManageLifecycle.class); - if (standalone) { binder.bind(RedirectFilter.class).in(LazySingleton.class); binder.bind(RedirectInfo.class).to(OverlordRedirectInfo.class).in(LazySingleton.class); diff --git a/web-console/src/dialogs/history-dialog/history-dialog.tsx b/web-console/src/dialogs/history-dialog/history-dialog.tsx index d204f316d3ed..66656c7d948f 100644 --- a/web-console/src/dialogs/history-dialog/history-dialog.tsx +++ b/web-console/src/dialogs/history-dialog/history-dialog.tsx @@ -60,7 +60,7 @@ export const HistoryDialog = React.memo(function HistoryDialog(props: HistoryDia let content: JSX.Element; if (historyRecords.length === 0) { - content =
No history records available
; + content =
No history records available. Try setting 'druid.audit.destiation=sql' to record config changes in metadata store.
; } else { content = ( From 9cb9dc693f73d1194fd789db329c69653ebaee55 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Mon, 4 Dec 2023 18:35:29 +0530 Subject: [PATCH 02/29] Fix compile and checkstyle --- .../overlord/http/OverlordResource.java | 1 + .../common/config/JacksonConfigManager.java | 1 + .../guice/SQLMetadataStorageDruidModule.java | 10 +-------- .../druid/server/audit/AuditLogger.java | 19 +++++++++++++++++ .../druid/server/audit/AuditRecord.java | 19 +++++++++++++++++ .../server/audit/LoggingAuditManager.java | 19 +++++++++++++++++ .../audit/LoggingAuditManagerConfig.java | 21 +++++++++++++++++-- 7 files changed, 79 insertions(+), 11 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index 8b5fa52df9f6..c3dd6ac52a65 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -235,6 +235,7 @@ public Response taskPost( .type("submit.ingestion.task") .payload(new TaskIdentifier(task.getId(), task.getGroupId(), task.getType())) .auditInfo(new AuditInfo(author, comment, req.getRemoteAddr())) + .build() ); } diff --git a/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java b/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java index b18d0f0162cd..c24177c160ea 100644 --- a/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java +++ b/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java @@ -122,6 +122,7 @@ public SetResult set( .type(key) .auditInfo(auditInfo) .payload(newValue) + .build() ); return configManager.set(key, configSerde, oldValue, newValue); } diff --git a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java index 7dfe63eb9c5f..1e396013f461 100644 --- a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java +++ b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java @@ -25,7 +25,6 @@ import com.google.inject.multibindings.MapBinder; import org.apache.druid.audit.AuditManager; import org.apache.druid.audit.AuditManagerConfig; -import org.apache.druid.server.audit.AuditSerdeHelper; import org.apache.druid.indexer.MetadataStorageUpdaterJobHandler; import org.apache.druid.indexer.SQLMetadataStorageUpdaterJobHandler; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; @@ -48,6 +47,7 @@ import org.apache.druid.metadata.SegmentsMetadataManagerProvider; import org.apache.druid.metadata.SqlSegmentsMetadataManager; import org.apache.druid.metadata.SqlSegmentsMetadataManagerProvider; +import org.apache.druid.server.audit.AuditSerdeHelper; import org.apache.druid.server.audit.LoggingAuditManager; import org.apache.druid.server.audit.LoggingAuditManagerConfig; import org.apache.druid.server.audit.SQLAuditManager; @@ -85,7 +85,6 @@ public void createBindingChoices(Binder binder, String defaultValue) PolyBind.createChoiceWithDefault(binder, prop, Key.get(IndexerMetadataStorageCoordinator.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageActionHandlerFactory.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageUpdaterJobHandler.class), defaultValue); - // PolyBind.createChoiceWithDefault(binder, prop, Key.get(AuditManager.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataSupervisorManager.class), defaultValue); configureAuditDestination(binder); @@ -134,13 +133,6 @@ public void configure(Binder binder) .to(SQLMetadataStorageUpdaterJobHandler.class) .in(LazySingleton.class); - // JsonConfigProvider.bind(binder, "druid.audit.manager", SQLAuditManagerConfig.class); - - /*PolyBind.optionBinder(binder, Key.get(AuditManager.class)) - .addBinding(type) - .to(SQLAuditManager.class) - .in(LazySingleton.class);*/ - PolyBind.optionBinder(binder, Key.get(MetadataSupervisorManager.class)) .addBinding(type) .to(SQLMetadataSupervisorManager.class) diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java index 12913da7a874..b5316250c217 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.server.audit; import org.apache.druid.java.util.common.logger.Logger; diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditRecord.java b/server/src/main/java/org/apache/druid/server/audit/AuditRecord.java index 814d405e7dcb..d0829209a145 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditRecord.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditRecord.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.server.audit; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java index 19be98c0f558..a65102bf9183 100644 --- a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.server.audit; import com.google.inject.Inject; diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java index b3b350c9a634..eba8d412c507 100644 --- a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.server.audit; import com.fasterxml.jackson.annotation.JsonProperty; @@ -7,8 +26,6 @@ public class LoggingAuditManagerConfig implements AuditManagerConfig { - @JsonProperty - private final boolean includePayloadAsDimensionInMetric = false; @JsonProperty private final AuditLogger.Level logLevel = AuditLogger.Level.INFO; From 74860c7c39181183765deb97934588b162983be5 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Mon, 4 Dec 2023 19:03:00 +0530 Subject: [PATCH 03/29] Audit more endpoints --- .../overlord/http/OverlordResource.java | 2 +- .../org/apache/druid/audit/AuditManager.java | 22 ++-- .../config/JacksonConfigManagerTest.java | 2 +- .../druid/server/audit/AuditSerdeHelper.java | 12 +- .../server/http/DataSourcesResource.java | 57 +++++++-- .../server/http/DataSourcesResourceTest.java | 119 ++++++++++-------- 6 files changed, 134 insertions(+), 80 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index c3dd6ac52a65..e0280945bf74 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -232,7 +232,7 @@ public Response taskPost( auditManager.doAudit( AuditEvent.builder() .key(task.getDataSource()) - .type("submit.ingestion.task") + .type("ingestion.batch") .payload(new TaskIdentifier(task.getId(), task.getGroupId(), task.getType())) .auditInfo(new AuditInfo(author, comment, req.getRemoteAddr())) .build() diff --git a/processing/src/main/java/org/apache/druid/audit/AuditManager.java b/processing/src/main/java/org/apache/druid/audit/AuditManager.java index 487a9c9d3e76..abc232346519 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditManager.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditManager.java @@ -41,8 +41,8 @@ public interface AuditManager * audited changes. Only SQL-based implementations need to implement this method, * other implementations call {@link #doAudit} by default. * - * @param AuditEvent - * @param handle JDBI Handle representing connection to the database + * @param event Event to audit + * @param handle JDBI Handle representing connection to the database */ default void doAudit(AuditEvent event, Handle handle) throws IOException { @@ -53,10 +53,7 @@ default void doAudit(AuditEvent event, Handle handle) throws IOException * Fetches audit entries made for the given key, type and interval. Implementations * that do not maintain an audit history should return an empty list. * - * @param key - * @param type - * @param interval - * @return list of AuditEntries satisfying the passed parameters + * @return List of recorded audit events satisfying the passed parameters. */ List fetchAuditHistory(String key, String type, Interval interval); @@ -65,26 +62,21 @@ default void doAudit(AuditEvent event, Handle handle) throws IOException * * @param type Type of audit entry * @param interval Eligible interval for audit time - * @return List of audit entries satisfying the passed parameters. + * @return List of recorded audit events satisfying the passed parameters. */ List fetchAuditHistory(String type, Interval interval); /** * Provides last N entries of audit history for given key, type * - * @param key - * @param type - * @param limit - * @return list of AuditEntries satisfying the passed parameters + * @return list of recorded audit events satisfying the passed parameters */ List fetchAuditHistory(String key, String type, int limit); /** * Provides last N entries of audit history for given type * - * @param type type of AuditEvent - * @param limit - * @return list of AuditEntries satisfying the passed parameters + * @return List of recorded audit events satisfying the passed parameters. */ List fetchAuditHistory(String type, int limit); @@ -92,7 +84,7 @@ default void doAudit(AuditEvent event, Handle handle) throws IOException * Remove audit logs created older than the given timestamp. * * @param timestamp timestamp in milliseconds - * @return number of audit logs removed + * @return Number of audit logs removed */ int removeAuditLogsOlderThan(long timestamp); } diff --git a/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java b/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java index 33e0e4cd324f..876d2e89a3e4 100644 --- a/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java +++ b/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java @@ -71,7 +71,7 @@ public void testSet() jacksonConfigManager.set(key, val, auditInfo); - ArgumentCaptor auditCapture = ArgumentCaptor.forClass(AuditEvent.Builder.class); + ArgumentCaptor auditCapture = ArgumentCaptor.forClass(AuditEvent.class); Mockito.verify(mockAuditManager).doAudit(auditCapture.capture()); Assert.assertNotNull(auditCapture.getValue()); } diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java index 728c1d5d3ecf..8e4c414c412a 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java @@ -26,12 +26,12 @@ import org.apache.druid.error.InvalidInput; import org.apache.druid.guice.annotations.Json; import org.apache.druid.guice.annotations.JsonNonNull; -import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; import java.io.IOException; -public class AuditSerdeHelper implements AuditEvent.PayloadDeserializer +public class AuditSerdeHelper { /** * Default message stored instead of the actual audit payload if the audit @@ -39,6 +39,9 @@ public class AuditSerdeHelper implements AuditEvent.PayloadDeserializer */ private static final String PAYLOAD_TRUNCATED_MSG = "Payload truncated as it exceeds 'druid.audit.manager.maxPayloadSizeBytes'"; + private static final String SERIALIZE_ERROR_MSG = + "Error serializing payload"; + private static final Logger log = new Logger(AuditSerdeHelper.class); private final ObjectMapper jsonMapper; private final ObjectMapper jsonMapperSkipNulls; @@ -72,7 +75,6 @@ public AuditRecord processAuditEvent(AuditEvent event) ); } - @Override public T deserializePayloadFromString(String serializedPayload, Class clazz) { if (serializedPayload == null || serializedPayload.isEmpty()) { @@ -105,7 +107,9 @@ private String serializePayloadToString(Object payload) : jsonMapper.writeValueAsString(payload); } catch (IOException e) { - throw new ISE(e, "Could not serialize audit payload[%s]", payload); + // Do not throw exception, only log error + log.error(e, "Could not serialize audit payload[%s]", payload); + return SERIALIZE_ERROR_MSG; } } diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 79237f507a54..46c521200a83 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -30,6 +30,9 @@ import com.sun.jersey.spi.container.ResourceFilters; import it.unimi.dsi.fastutil.objects.Object2LongMap; import org.apache.commons.lang.StringUtils; +import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditInfo; +import org.apache.druid.audit.AuditManager; import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; @@ -70,7 +73,9 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; +import javax.ws.rs.HeaderParam; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -110,6 +115,7 @@ public class DataSourcesResource private final OverlordClient overlordClient; private final AuthorizerMapper authorizerMapper; private final DruidCoordinator coordinator; + private final AuditManager auditManager; @Inject public DataSourcesResource( @@ -118,7 +124,8 @@ public DataSourcesResource( MetadataRuleManager metadataRuleManager, @Nullable OverlordClient overlordClient, AuthorizerMapper authorizerMapper, - DruidCoordinator coordinator + DruidCoordinator coordinator, + AuditManager auditManager ) { this.serverInventoryView = serverInventoryView; @@ -127,6 +134,7 @@ public DataSourcesResource( this.overlordClient = overlordClient; this.authorizerMapper = authorizerMapper; this.coordinator = coordinator; + this.auditManager = auditManager; } @GET @@ -220,13 +228,19 @@ public Response markAsUsedNonOvershadowedSegments( @Consumes(MediaType.APPLICATION_JSON) public Response markSegmentsAsUnused( @PathParam("dataSourceName") final String dataSourceName, - final MarkDataSourceSegmentsPayload payload + final MarkDataSourceSegmentsPayload payload, + @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, + @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, + @Context final HttpServletRequest req ) { MarkSegments markSegments = () -> { final Interval interval = payload.getInterval(); + final int numUpdatedSegments; + final Object auditPayload; if (interval != null) { - return segmentsMetadataManager.markAsUnusedSegmentsInInterval(dataSourceName, interval); + numUpdatedSegments = segmentsMetadataManager.markAsUnusedSegmentsInInterval(dataSourceName, interval); + auditPayload = Collections.singletonMap("interval", interval); } else { final Set segmentIds = payload.getSegmentIds() @@ -236,12 +250,24 @@ public Response markSegmentsAsUnused( .collect(Collectors.toSet()); // Note: segments for the "wrong" datasource are ignored. - return segmentsMetadataManager.markSegmentsAsUnused( + numUpdatedSegments = segmentsMetadataManager.markSegmentsAsUnused( segmentIds.stream() .filter(segmentId -> segmentId.getDataSource().equals(dataSourceName)) .collect(Collectors.toSet()) ); + auditPayload = Collections.singletonMap("segmentIds", segmentIds); + } + if (author != null && !author.isEmpty()) { + auditManager.doAudit( + AuditEvent.builder() + .key(dataSourceName) + .type("segments.markUnused") + .payload(auditPayload) + .auditInfo(new AuditInfo(author, comment, req.getRemoteAddr())) + .build() + ); } + return numUpdatedSegments; }; return doMarkSegmentsWithPayload("markSegmentsAsUnused", dataSourceName, payload, markSegments); } @@ -312,7 +338,8 @@ private static Response doMarkSegments(String method, String dataSourceName, Mar public Response markAsUnusedAllSegmentsOrKillUnusedSegmentsInInterval( @PathParam("dataSourceName") final String dataSourceName, @QueryParam("kill") final String kill, - @QueryParam("interval") final String interval + @QueryParam("interval") final String interval, + @Context HttpServletRequest req ) { if (overlordClient == null) { @@ -321,7 +348,7 @@ public Response markAsUnusedAllSegmentsOrKillUnusedSegmentsInInterval( boolean killSegments = kill != null && Boolean.valueOf(kill); if (killSegments) { - return killUnusedSegmentsInInterval(dataSourceName, interval); + return killUnusedSegmentsInInterval(dataSourceName, interval, null, null, req); } else { MarkSegments markSegments = () -> segmentsMetadataManager.markAsUnusedAllSegmentsInDataSource(dataSourceName); return doMarkSegments("markAsUnusedAllSegments", dataSourceName, markSegments); @@ -334,7 +361,10 @@ public Response markAsUnusedAllSegmentsOrKillUnusedSegmentsInInterval( @Produces(MediaType.APPLICATION_JSON) public Response killUnusedSegmentsInInterval( @PathParam("dataSourceName") final String dataSourceName, - @PathParam("interval") final String interval + @PathParam("interval") final String interval, + @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, + @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, + @Context final HttpServletRequest req ) { if (overlordClient == null) { @@ -345,7 +375,18 @@ public Response killUnusedSegmentsInInterval( } final Interval theInterval = Intervals.of(interval.replace('_', '/')); try { - FutureUtils.getUnchecked(overlordClient.runKillTask("api-issued", dataSourceName, theInterval, null), true); + final String killTaskId = FutureUtils.getUnchecked( + overlordClient.runKillTask("api-issued", dataSourceName, theInterval, null), + true + ); + auditManager.doAudit( + AuditEvent.builder() + .key(dataSourceName) + .type("segments.killTask") + .payload(ImmutableMap.of("killTaskId", killTaskId, "interval", theInterval)) + .auditInfo(new AuditInfo(author, comment, req.getRemoteAddr())) + .build() + ); return Response.ok().build(); } catch (Exception e) { diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index c33fffc80830..2413f96577ad 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -181,7 +181,7 @@ public void testGetFullQueryableDataSources() EasyMock.replay(inventoryView, server, request); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, null); + new DataSourcesResource(inventoryView, null, null, null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, null, null); Response response = dataSourcesResource.getQueryableDataSources("full", null, request); Set result = (Set) response.getEntity(); Assert.assertEquals(200, response.getStatus()); @@ -255,7 +255,9 @@ public Access authorize(AuthenticationResult authenticationResult1, Resource res } }; - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, null, authMapper, null); + DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, null, authMapper, null, + null + ); Response response = dataSourcesResource.getQueryableDataSources("full", null, request); Set result = (Set) response.getEntity(); @@ -294,7 +296,7 @@ public void testGetSimpleQueryableDataSources() EasyMock.replay(inventoryView, server, request); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, null); + new DataSourcesResource(inventoryView, null, null, null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, null, null); Response response = dataSourcesResource.getQueryableDataSources(null, "simple", request); Assert.assertEquals(200, response.getStatus()); List> results = (List>) response.getEntity(); @@ -318,7 +320,7 @@ public void testFullGetTheDataSource() EasyMock.replay(inventoryView, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, null, null); + new DataSourcesResource(inventoryView, null, null, null, null, null, null); Response response = dataSourcesResource.getDataSource("datasource1", "full"); ImmutableDruidDataSource result = (ImmutableDruidDataSource) response.getEntity(); Assert.assertEquals(200, response.getStatus()); @@ -334,7 +336,7 @@ public void testNullGetTheDataSource() EasyMock.replay(inventoryView, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, null, null); + new DataSourcesResource(inventoryView, null, null, null, null, null, null); Assert.assertEquals(204, dataSourcesResource.getDataSource("none", null).getStatus()); EasyMock.verify(inventoryView, server); } @@ -352,7 +354,7 @@ public void testSimpleGetTheDataSource() EasyMock.replay(inventoryView, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, null, null); + new DataSourcesResource(inventoryView, null, null, null, null, null, null); Response response = dataSourcesResource.getDataSource("datasource1", null); Assert.assertEquals(200, response.getStatus()); Map> result = (Map>) response.getEntity(); @@ -385,7 +387,7 @@ public void testSimpleGetTheDataSourceManyTiers() EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server, server2, server3)).atLeastOnce(); EasyMock.replay(inventoryView, server, server2, server3); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, null, null, null); + DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, null, null, null, null); Response response = dataSourcesResource.getDataSource("datasource1", null); Assert.assertEquals(200, response.getStatus()); Map> result = (Map>) response.getEntity(); @@ -423,7 +425,7 @@ public void testSimpleGetTheDataSourceWithReplicatedSegments() EasyMock.replay(inventoryView); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, null, null, null); + DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, null, null, null, null); Response response = dataSourcesResource.getDataSource("datasource1", null); Assert.assertEquals(200, response.getStatus()); Map> result1 = (Map>) response.getEntity(); @@ -468,7 +470,7 @@ public void testGetSegmentDataSourceIntervals() expectedIntervals.add(Intervals.of("2010-01-22T00:00:00.000Z/2010-01-23T00:00:00.000Z")); expectedIntervals.add(Intervals.of("2010-01-01T00:00:00.000Z/2010-01-02T00:00:00.000Z")); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, null, null); + new DataSourcesResource(inventoryView, null, null, null, null, null, null); Response response = dataSourcesResource.getIntervalsWithServedSegmentsOrAllServedSegmentsPerIntervals( "invalidDataSource", @@ -528,7 +530,7 @@ public void testGetServedSegmentsInIntervalInDataSource() EasyMock.replay(inventoryView); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, null, null); + new DataSourcesResource(inventoryView, null, null, null, null, null, null); Response response = dataSourcesResource.getServedSegmentsInInterval( "invalidDataSource", "2010-01-01/P1D", @@ -598,8 +600,8 @@ public void testKillSegmentsInIntervalInDataSource() EasyMock.replay(overlordClient, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, overlordClient, null, null); - Response response = dataSourcesResource.killUnusedSegmentsInInterval("datasource1", interval); + new DataSourcesResource(inventoryView, null, null, overlordClient, null, null, null); + Response response = dataSourcesResource.killUnusedSegmentsInInterval("datasource1", interval, null, null, request); Assert.assertEquals(200, response.getStatus()); Assert.assertNull(response.getEntity()); @@ -612,10 +614,10 @@ public void testMarkAsUnusedAllSegmentsInDataSource() OverlordClient overlordClient = EasyMock.createStrictMock(OverlordClient.class); EasyMock.replay(overlordClient, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, overlordClient, null, null); + new DataSourcesResource(inventoryView, null, null, overlordClient, null, null, null); try { Response response = - dataSourcesResource.markAsUnusedAllSegmentsOrKillUnusedSegmentsInInterval("datasource", "true", "???"); + dataSourcesResource.markAsUnusedAllSegmentsOrKillUnusedSegmentsInInterval("datasource", "true", "???", request); // 400 (Bad Request) or an IllegalArgumentException is expected. Assert.assertEquals(400, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -635,7 +637,7 @@ public void testIsHandOffComplete() Rule loadRule = new IntervalLoadRule(Intervals.of("2013-01-02T00:00:00Z/2013-01-03T00:00:00Z"), null, null); Rule dropRule = new IntervalDropRule(Intervals.of("2013-01-01T00:00:00Z/2013-01-02T00:00:00Z")); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, databaseRuleManager, null, null, null); + new DataSourcesResource(inventoryView, null, databaseRuleManager, null, null, null, null); // test dropped EasyMock.expect(databaseRuleManager.getRulesWithDefault("dataSource1")) @@ -704,7 +706,9 @@ public void testMarkSegmentAsUsed() EasyMock.expect(segmentsMetadataManager.markSegmentAsUsed(segment.getId().toString())).andReturn(true).once(); EasyMock.replay(segmentsMetadataManager); - DataSourcesResource dataSourcesResource = new DataSourcesResource(null, segmentsMetadataManager, null, null, null, null); + DataSourcesResource dataSourcesResource = new DataSourcesResource(null, segmentsMetadataManager, null, null, null, null, + null + ); Response response = dataSourcesResource.markSegmentAsUsed(segment.getDataSource(), segment.getId().toString()); Assert.assertEquals(200, response.getStatus()); @@ -718,7 +722,9 @@ public void testMarkSegmentAsUsedNoChange() EasyMock.expect(segmentsMetadataManager.markSegmentAsUsed(segment.getId().toString())).andReturn(false).once(); EasyMock.replay(segmentsMetadataManager); - DataSourcesResource dataSourcesResource = new DataSourcesResource(null, segmentsMetadataManager, null, null, null, null); + DataSourcesResource dataSourcesResource = new DataSourcesResource(null, segmentsMetadataManager, null, null, null, null, + null + ); Response response = dataSourcesResource.markSegmentAsUsed(segment.getDataSource(), segment.getId().toString()); Assert.assertEquals(200, response.getStatus()); @@ -739,7 +745,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsInterval() EasyMock.replay(segmentsMetadataManager, inventoryView, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -762,7 +768,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsIntervalNoneUpdated() EasyMock.replay(segmentsMetadataManager, inventoryView, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -785,7 +791,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsSet() throws UnknownSegmentIdsE EasyMock.replay(segmentsMetadataManager, inventoryView, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -808,7 +814,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsIntervalException() EasyMock.replay(segmentsMetadataManager, inventoryView, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -826,7 +832,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsNoDataSource() EasyMock.replay(segmentsMetadataManager, inventoryView, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -840,7 +846,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsNoDataSource() public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadNoArguments() { DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -853,7 +859,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadNoArguments() public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadBothArguments() { DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -866,7 +872,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadBothArguments() public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadEmptyArray() { DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -879,7 +885,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadEmptyArray() public void testMarkAsUsedNonOvershadowedSegmentsNoPayload() { DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments("datasource1", null); Assert.assertEquals(400, response.getStatus()); @@ -1039,8 +1045,8 @@ public void testMarkSegmentsAsUnused() ); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 1), response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1070,8 +1076,8 @@ public void testMarkSegmentsAsUnusedNoChanges() ); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 0), response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1103,8 +1109,8 @@ public void testMarkSegmentsAsUnusedException() ); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); Assert.assertEquals(500, response.getStatus()); Assert.assertNotNull(response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1125,8 +1131,8 @@ public void testMarkAsUnusedSegmentsInInterval() new DataSourcesResource.MarkDataSourceSegmentsPayload(theInterval, null); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 1), response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1148,8 +1154,8 @@ public void testMarkAsUnusedSegmentsInIntervalNoChanges() new DataSourcesResource.MarkDataSourceSegmentsPayload(theInterval, null); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 0), response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1172,8 +1178,8 @@ public void testMarkAsUnusedSegmentsInIntervalException() new DataSourcesResource.MarkDataSourceSegmentsPayload(theInterval, null); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); Assert.assertEquals(500, response.getStatus()); Assert.assertNotNull(response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1183,9 +1189,9 @@ public void testMarkAsUnusedSegmentsInIntervalException() public void testMarkSegmentsAsUnusedNullPayload() { DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", null); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", null, "", "", request); Assert.assertEquals(400, response.getStatus()); Assert.assertNotNull(response.getEntity()); Assert.assertEquals( @@ -1198,12 +1204,12 @@ public void testMarkSegmentsAsUnusedNullPayload() public void testMarkSegmentsAsUnusedInvalidPayload() { DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); final DataSourcesResource.MarkDataSourceSegmentsPayload payload = new DataSourcesResource.MarkDataSourceSegmentsPayload(null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); Assert.assertEquals(400, response.getStatus()); Assert.assertNotNull(response.getEntity()); } @@ -1212,12 +1218,12 @@ public void testMarkSegmentsAsUnusedInvalidPayload() public void testMarkSegmentsAsUnusedInvalidPayloadBothArguments() { DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); final DataSourcesResource.MarkDataSourceSegmentsPayload payload = new DataSourcesResource.MarkDataSourceSegmentsPayload(Intervals.of("2010-01-01/P1D"), ImmutableSet.of()); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); Assert.assertEquals(400, response.getStatus()); Assert.assertNotNull(response.getEntity()); } @@ -1225,7 +1231,9 @@ public void testMarkSegmentsAsUnusedInvalidPayloadBothArguments() @Test public void testGetDatasourceLoadstatusForceMetadataRefreshNull() { - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, + null + ); Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", null, null, null, null, null); Assert.assertEquals(400, response.getStatus()); } @@ -1246,6 +1254,7 @@ public void testGetDatasourceLoadstatusNoSegmentForInterval() null, null, null, + null, null ); Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, null, null, null); @@ -1305,7 +1314,9 @@ public void testGetDatasourceLoadstatusDefault() EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, + null + ); Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, null, null, null); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1321,7 +1332,7 @@ public void testGetDatasourceLoadstatusDefault() EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); - dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, null, null, null); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1384,7 +1395,9 @@ public void testGetDatasourceLoadstatusSimple() EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, + null + ); Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, "simple", null, null); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1400,7 +1413,7 @@ public void testGetDatasourceLoadstatusSimple() EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); - dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); + dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, "simple", null, null); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1455,7 +1468,9 @@ public void testGetDatasourceLoadstatusFull() EasyMock.replay(segmentsMetadataManager, druidCoordinator); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, druidCoordinator); + DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, druidCoordinator, + null + ); Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, null, "full", null); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1512,7 +1527,9 @@ public void testGetDatasourceLoadstatusFullAndComputeUsingClusterView() EasyMock.replay(segmentsMetadataManager, druidCoordinator); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, druidCoordinator); + DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, druidCoordinator, + null + ); Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, null, "full", "computeUsingClusterView"); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); From f418435223195a60b29b28744e7cab021c999040 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Mon, 4 Dec 2023 19:08:17 +0530 Subject: [PATCH 04/29] Fix codeql --- .../druid/server/audit/AuditSerdeHelper.java | 24 ++----------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java index 8e4c414c412a..9bb2913bf8ca 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java @@ -20,10 +20,10 @@ package org.apache.druid.server.audit; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; import com.google.inject.Inject; import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditManagerConfig; -import org.apache.druid.error.InvalidInput; import org.apache.druid.guice.annotations.Json; import org.apache.druid.guice.annotations.JsonNonNull; import org.apache.druid.java.util.common.StringUtils; @@ -75,26 +75,6 @@ public AuditRecord processAuditEvent(AuditEvent event) ); } - public T deserializePayloadFromString(String serializedPayload, Class clazz) - { - if (serializedPayload == null || serializedPayload.isEmpty()) { - return null; - } else if (serializedPayload.contains(PAYLOAD_TRUNCATED_MSG)) { - throw InvalidInput.exception("Cannot deserialize audit payload[%s].", serializedPayload); - } - - try { - return jsonMapper.readValue(serializedPayload, clazz); - } - catch (IOException e) { - throw InvalidInput.exception( - e, - "Could not deserialize audit payload[%s] into class[%s]", - serializedPayload, clazz - ); - } - } - private String serializePayloadToString(Object payload) { if (payload == null) { @@ -123,7 +103,7 @@ private String truncateSerializedAuditPayload(String serializedPayload) return serializedPayload; } - int payloadSize = serializedPayload.getBytes().length; + int payloadSize = serializedPayload.getBytes(Charsets.UTF_8).length; if (payloadSize > config.getMaxPayloadSizeBytes()) { return PAYLOAD_TRUNCATED_MSG + StringUtils.format("[%s].", config.getMaxPayloadSizeBytes()); } else { From 5adc89a5c5df96b92e762019e32779c1002adfbc Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Mon, 4 Dec 2023 19:27:47 +0530 Subject: [PATCH 05/29] Audit some more endpoints --- .../endpoint/BasicAuthenticatorResource.java | 78 +++++++++++++++++-- ...dinatorBasicAuthenticatorResourceTest.java | 41 ++++++---- .../BasicAuthenticatorResourceTest.java | 14 ++-- .../server/http/DataSourcesResource.java | 20 ++--- 4 files changed, 113 insertions(+), 40 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java index 23d8316771c4..6798c9875182 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java @@ -22,6 +22,9 @@ import com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes; import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; +import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditInfo; +import org.apache.druid.audit.AuditManager; import org.apache.druid.guice.LazySingleton; import org.apache.druid.security.basic.BasicSecurityResourceFilter; import org.apache.druid.security.basic.authentication.entity.BasicAuthenticatorCredentialUpdate; @@ -30,7 +33,9 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; +import javax.ws.rs.HeaderParam; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -38,6 +43,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.util.Collections; @Path("/druid-ext/basic-security/authentication") @LazySingleton @@ -45,15 +51,18 @@ public class BasicAuthenticatorResource { private final BasicAuthenticatorResourceHandler handler; private final AuthValidator authValidator; + private final AuditManager auditManager; @Inject public BasicAuthenticatorResource( BasicAuthenticatorResourceHandler handler, - AuthValidator authValidator + AuthValidator authValidator, + AuditManager auditManager ) { this.handler = handler; this.authValidator = authValidator; + this.auditManager = auditManager; } /** @@ -137,6 +146,8 @@ public Response getUser( * @param req HTTP request * @param userName Name to assign the new user * + * @param author + * @param comment * @return OK response, or 400 error response if user already exists */ @POST @@ -147,11 +158,21 @@ public Response getUser( public Response createUser( @Context HttpServletRequest req, @PathParam("authenticatorName") final String authenticatorName, - @PathParam("userName") String userName + @PathParam("userName") String userName, + @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, + @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment ) { authValidator.validateAuthenticatorName(authenticatorName); - return handler.createUser(authenticatorName, userName); + + final Response response = handler.createUser(authenticatorName, userName); + + if (isSuccess(response)) { + final AuditInfo auditInfo = new AuditInfo(author, comment, req.getRemoteAddr()); + performAudit(authenticatorName, "users.create", Collections.singletonMap("username", userName), auditInfo); + } + + return response; } /** @@ -160,6 +181,8 @@ public Response createUser( * @param req HTTP request * @param userName Name of user to delete * + * @param author + * @param comment * @return OK response, or 400 error response if user doesn't exist */ @DELETE @@ -170,11 +193,20 @@ public Response createUser( public Response deleteUser( @Context HttpServletRequest req, @PathParam("authenticatorName") final String authenticatorName, - @PathParam("userName") String userName + @PathParam("userName") String userName, + @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, + @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment ) { authValidator.validateAuthenticatorName(authenticatorName); - return handler.deleteUser(authenticatorName, userName); + final Response response = handler.deleteUser(authenticatorName, userName); + + if (isSuccess(response)) { + final AuditInfo auditInfo = new AuditInfo(author, comment, req.getRemoteAddr()); + performAudit(authenticatorName, "users.delete", Collections.singletonMap("username", userName), auditInfo); + } + + return response; } /** @@ -183,6 +215,8 @@ public Response deleteUser( * @param req HTTP request * @param userName Name of user * + * @param author + * @param comment * @return OK response, 400 error if user doesn't exist */ @POST @@ -194,11 +228,20 @@ public Response updateUserCredentials( @Context HttpServletRequest req, @PathParam("authenticatorName") final String authenticatorName, @PathParam("userName") String userName, - BasicAuthenticatorCredentialUpdate update + BasicAuthenticatorCredentialUpdate update, + @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, + @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment ) { authValidator.validateAuthenticatorName(authenticatorName); - return handler.updateUserCredentials(authenticatorName, userName, update); + final Response response = handler.updateUserCredentials(authenticatorName, userName, update); + + if (isSuccess(response)) { + final AuditInfo auditInfo = new AuditInfo(author, comment, req.getRemoteAddr()); + performAudit(authenticatorName, "users.update", Collections.singletonMap("username", userName), auditInfo); + } + + return response; } /** @@ -237,4 +280,25 @@ public Response authenticatorUpdateListener( authValidator.validateAuthenticatorName(authenticatorName); return handler.authenticatorUserUpdateListener(authenticatorName, serializedUserMap); } + + private boolean isSuccess(Response response) { + if (response == null) { + return false; + } + + int responseCode = response.getStatus(); + return responseCode >= 200 && responseCode < 300; + } + + private void performAudit(String authenticatorName, String action, Object payload, AuditInfo auditInfo) + { + auditManager.doAudit( + AuditEvent.builder() + .key(authenticatorName) + .type(action) + .auditInfo(auditInfo) + .payload(payload) + .build() + ); + } } diff --git a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java index b383a6d3d92f..a696286f5f83 100644 --- a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java +++ b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.apache.druid.audit.AuditManager; import org.apache.druid.metadata.DefaultPasswordProvider; import org.apache.druid.metadata.MetadataStorageTablesConfig; import org.apache.druid.metadata.TestDerbyConnector; @@ -68,6 +69,9 @@ public class CoordinatorBasicAuthenticatorResourceTest @Mock private AuthValidator authValidator; + + @Mock + private AuditManager auditManager; private BasicAuthenticatorResource resource; private CoordinatorBasicAuthenticatorMetadataStorageUpdater storageUpdater; private HttpServletRequest req; @@ -145,7 +149,8 @@ public void setUp() authenticatorMapper, objectMapper ), - authValidator + authValidator, + auditManager ); storageUpdater.start(); @@ -175,9 +180,9 @@ public void testGetAllUsers() Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableSet.of(BasicAuthUtils.ADMIN_NAME, BasicAuthUtils.INTERNAL_USER_NAME), response.getEntity()); - resource.createUser(req, AUTHENTICATOR_NAME, "druid"); - resource.createUser(req, AUTHENTICATOR_NAME, "druid2"); - resource.createUser(req, AUTHENTICATOR_NAME, "druid3"); + resource.createUser(req, AUTHENTICATOR_NAME, "druid", null, null); + resource.createUser(req, AUTHENTICATOR_NAME, "druid2", null, null); + resource.createUser(req, AUTHENTICATOR_NAME, "druid3", null, null); Set expectedUsers = ImmutableSet.of( BasicAuthUtils.ADMIN_NAME, @@ -215,13 +220,13 @@ public void testGetAllUsersSeparateDatabaseTables() Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableSet.of(BasicAuthUtils.ADMIN_NAME, BasicAuthUtils.INTERNAL_USER_NAME), response.getEntity()); - resource.createUser(req, AUTHENTICATOR_NAME, "druid"); - resource.createUser(req, AUTHENTICATOR_NAME, "druid2"); - resource.createUser(req, AUTHENTICATOR_NAME, "druid3"); + resource.createUser(req, AUTHENTICATOR_NAME, "druid", null, null); + resource.createUser(req, AUTHENTICATOR_NAME, "druid2", null, null); + resource.createUser(req, AUTHENTICATOR_NAME, "druid3", null, null); - resource.createUser(req, AUTHENTICATOR_NAME2, "druid4"); - resource.createUser(req, AUTHENTICATOR_NAME2, "druid5"); - resource.createUser(req, AUTHENTICATOR_NAME2, "druid6"); + resource.createUser(req, AUTHENTICATOR_NAME2, "druid4", null, null); + resource.createUser(req, AUTHENTICATOR_NAME2, "druid5", null, null); + resource.createUser(req, AUTHENTICATOR_NAME2, "druid6", null, null); Set expectedUsers = ImmutableSet.of( BasicAuthUtils.ADMIN_NAME, @@ -285,7 +290,7 @@ public void testGetAllUsersSeparateDatabaseTables() @Test public void testCreateDeleteUser() { - Response response = resource.createUser(req, AUTHENTICATOR_NAME, "druid"); + Response response = resource.createUser(req, AUTHENTICATOR_NAME, "druid", null, null); Assert.assertEquals(200, response.getStatus()); response = resource.getUser(req, AUTHENTICATOR_NAME, "druid"); @@ -293,7 +298,7 @@ public void testCreateDeleteUser() BasicAuthenticatorUser expectedUser = new BasicAuthenticatorUser("druid", null); Assert.assertEquals(expectedUser, response.getEntity()); - response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid"); + response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid", null, null); Assert.assertEquals(200, response.getStatus()); response = resource.getCachedSerializedUserMap(req, AUTHENTICATOR_NAME); @@ -303,7 +308,7 @@ public void testCreateDeleteUser() Assert.assertNotNull(cachedUserMap); Assert.assertNull(cachedUserMap.get("druid")); - response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid"); + response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid", null, null); Assert.assertEquals(400, response.getStatus()); Assert.assertEquals(errorMapWithMsg("User [druid] does not exist."), response.getEntity()); @@ -315,14 +320,15 @@ public void testCreateDeleteUser() @Test public void testUserCredentials() { - Response response = resource.createUser(req, AUTHENTICATOR_NAME, "druid"); + Response response = resource.createUser(req, AUTHENTICATOR_NAME, "druid", null, null); Assert.assertEquals(200, response.getStatus()); response = resource.updateUserCredentials( req, AUTHENTICATOR_NAME, "druid", - new BasicAuthenticatorCredentialUpdate("helloworld", null) + new BasicAuthenticatorCredentialUpdate("helloworld", null), + null, null ); Assert.assertEquals(200, response.getStatus()); @@ -369,7 +375,7 @@ public void testUserCredentials() ); Assert.assertArrayEquals(recalculatedHash, hash); - response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid"); + response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid", null, null); Assert.assertEquals(200, response.getStatus()); response = resource.getUser(req, AUTHENTICATOR_NAME, "druid"); @@ -380,7 +386,8 @@ public void testUserCredentials() req, AUTHENTICATOR_NAME, "druid", - new BasicAuthenticatorCredentialUpdate("helloworld", null) + new BasicAuthenticatorCredentialUpdate("helloworld", null), + null, null ); Assert.assertEquals(400, response.getStatus()); Assert.assertEquals(errorMapWithMsg("User [druid] does not exist."), response.getEntity()); diff --git a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResourceTest.java b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResourceTest.java index fc403b08ca30..69ac7ef7ca1f 100644 --- a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResourceTest.java +++ b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResourceTest.java @@ -58,7 +58,7 @@ public void setUp() .when(authValidator) .validateAuthenticatorName(INVALID_AUTHENTICATOR_NAME); - target = new BasicAuthenticatorResource(handler, authValidator); + target = new BasicAuthenticatorResource(handler, authValidator, null); } @Test @@ -88,37 +88,37 @@ public void getCachedSerializedUserMapWithInvalidAuthenticatorNameShouldReturnEx @Test public void updateUserCredentialsShouldReturnExpectedResponse() { - Assert.assertNotNull(target.updateUserCredentials(req, AUTHENTICATOR_NAME, USER_NAME, update)); + Assert.assertNotNull(target.updateUserCredentials(req, AUTHENTICATOR_NAME, USER_NAME, update, null, null)); } @Test(expected = IllegalArgumentException.class) public void updateUserCredentialsWithInvalidAuthenticatorNameShouldReturnExpectedResponse() { - target.updateUserCredentials(req, INVALID_AUTHENTICATOR_NAME, USER_NAME, update); + target.updateUserCredentials(req, INVALID_AUTHENTICATOR_NAME, USER_NAME, update, null, null); } @Test public void deleteUserShouldReturnExpectedResponse() { - Assert.assertNotNull(target.deleteUser(req, AUTHENTICATOR_NAME, USER_NAME)); + Assert.assertNotNull(target.deleteUser(req, AUTHENTICATOR_NAME, USER_NAME, null, null)); } @Test(expected = IllegalArgumentException.class) public void deleteUserWithInvalidAuthenticatorNameShouldReturnExpectedResponse() { - target.deleteUser(req, INVALID_AUTHENTICATOR_NAME, USER_NAME); + target.deleteUser(req, INVALID_AUTHENTICATOR_NAME, USER_NAME, null, null); } @Test public void createUserShouldReturnExpectedResponse() { - Assert.assertNotNull(target.createUser(req, AUTHENTICATOR_NAME, USER_NAME)); + Assert.assertNotNull(target.createUser(req, AUTHENTICATOR_NAME, USER_NAME, null, null)); } @Test(expected = IllegalArgumentException.class) public void createUserWithInvalidAuthenticatorNameShouldReturnExpectedResponse() { - target.createUser(req, INVALID_AUTHENTICATOR_NAME, USER_NAME); + target.createUser(req, INVALID_AUTHENTICATOR_NAME, USER_NAME, null, null); } @Test diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 46c521200a83..5ebdaf0bf471 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -257,7 +257,7 @@ public Response markSegmentsAsUnused( ); auditPayload = Collections.singletonMap("segmentIds", segmentIds); } - if (author != null && !author.isEmpty()) { + if (auditManager != null && author != null && !author.isEmpty()) { auditManager.doAudit( AuditEvent.builder() .key(dataSourceName) @@ -379,14 +379,16 @@ public Response killUnusedSegmentsInInterval( overlordClient.runKillTask("api-issued", dataSourceName, theInterval, null), true ); - auditManager.doAudit( - AuditEvent.builder() - .key(dataSourceName) - .type("segments.killTask") - .payload(ImmutableMap.of("killTaskId", killTaskId, "interval", theInterval)) - .auditInfo(new AuditInfo(author, comment, req.getRemoteAddr())) - .build() - ); + if (auditManager != null && author != null && !author.isEmpty()) { + auditManager.doAudit( + AuditEvent.builder() + .key(dataSourceName) + .type("segments.killTask") + .payload(ImmutableMap.of("killTaskId", killTaskId, "interval", theInterval)) + .auditInfo(new AuditInfo(author, comment, req.getRemoteAddr())) + .build() + ); + } return Response.ok().build(); } catch (Exception e) { From 383eb1d3de9572d0b31bce8a2ea045403e05919a Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Mon, 4 Dec 2023 19:32:11 +0530 Subject: [PATCH 06/29] Fix codeQL check --- .../endpoint/BasicAuthenticatorResource.java | 9 ++------- .../org/apache/druid/server/audit/AuditSerdeHelper.java | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java index 6798c9875182..70168ed16382 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java @@ -146,8 +146,6 @@ public Response getUser( * @param req HTTP request * @param userName Name to assign the new user * - * @param author - * @param comment * @return OK response, or 400 error response if user already exists */ @POST @@ -181,8 +179,6 @@ public Response createUser( * @param req HTTP request * @param userName Name of user to delete * - * @param author - * @param comment * @return OK response, or 400 error response if user doesn't exist */ @DELETE @@ -215,8 +211,6 @@ public Response deleteUser( * @param req HTTP request * @param userName Name of user * - * @param author - * @param comment * @return OK response, 400 error if user doesn't exist */ @POST @@ -281,7 +275,8 @@ public Response authenticatorUpdateListener( return handler.authenticatorUserUpdateListener(authenticatorName, serializedUserMap); } - private boolean isSuccess(Response response) { + private boolean isSuccess(Response response) + { if (response == null) { return false; } diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java index 9bb2913bf8ca..41f7a628a5a5 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java @@ -20,7 +20,6 @@ package org.apache.druid.server.audit; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; import com.google.inject.Inject; import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditManagerConfig; @@ -30,6 +29,7 @@ import org.apache.druid.java.util.common.logger.Logger; import java.io.IOException; +import java.nio.charset.StandardCharsets; public class AuditSerdeHelper { @@ -103,7 +103,7 @@ private String truncateSerializedAuditPayload(String serializedPayload) return serializedPayload; } - int payloadSize = serializedPayload.getBytes(Charsets.UTF_8).length; + int payloadSize = serializedPayload.getBytes(StandardCharsets.UTF_8).length; if (payloadSize > config.getMaxPayloadSizeBytes()) { return PAYLOAD_TRUNCATED_MSG + StringUtils.format("[%s].", config.getMaxPayloadSizeBytes()); } else { From 011319ee9572de2552de97ed71f477dcb98b2aac Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Mon, 4 Dec 2023 19:42:11 +0530 Subject: [PATCH 07/29] Fix web checks --- web-console/src/dialogs/history-dialog/history-dialog.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console/src/dialogs/history-dialog/history-dialog.tsx b/web-console/src/dialogs/history-dialog/history-dialog.tsx index 66656c7d948f..6b7f219e6471 100644 --- a/web-console/src/dialogs/history-dialog/history-dialog.tsx +++ b/web-console/src/dialogs/history-dialog/history-dialog.tsx @@ -60,7 +60,7 @@ export const HistoryDialog = React.memo(function HistoryDialog(props: HistoryDia let content: JSX.Element; if (historyRecords.length === 0) { - content =
No history records available. Try setting 'druid.audit.destiation=sql' to record config changes in metadata store.
; + content =
No history records available. Try setting 'druid.audit.destiation=sql' to record config changes in metadata store.
; } else { content = ( From 0d88e648843cf3f63478d0ef4050a4f0b8c3de94 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Mon, 4 Dec 2023 23:08:28 +0530 Subject: [PATCH 08/29] Fix tests --- ...dinatorBasicAuthenticatorResourceTest.java | 2 ++ .../org/apache/druid/audit/AuditEvent.java | 3 ++- .../org/apache/druid/audit/AuditInfoTest.java | 23 +++++++++++-------- .../server/audit/SQLAuditManagerTest.java | 9 +++++++- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java index a696286f5f83..defcf488cb61 100644 --- a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java +++ b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java @@ -81,6 +81,8 @@ public class CoordinatorBasicAuthenticatorResourceTest public void setUp() { req = EasyMock.createStrictMock(HttpServletRequest.class); + EasyMock.expect(req.getRemoteAddr()).andReturn("127.0.0.1").anyTimes(); + EasyMock.replay(req); objectMapper = new ObjectMapper(new SmileFactory()); TestDerbyConnector connector = derbyConnectorRule.getConnector(); diff --git a/processing/src/main/java/org/apache/druid/audit/AuditEvent.java b/processing/src/main/java/org/apache/druid/audit/AuditEvent.java index aac12b050b53..170628e86ff7 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditEvent.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditEvent.java @@ -130,13 +130,14 @@ public boolean equals(Object o) && Objects.equals(this.key, that.key) && Objects.equals(this.type, that.type) && Objects.equals(this.auditInfo, that.auditInfo) + && Objects.equals(this.payload, that.payload) && Objects.equals(this.serializedPayload, that.serializedPayload); } @Override public int hashCode() { - return Objects.hash(key, type, auditInfo, serializedPayload, auditTime); + return Objects.hash(key, type, auditInfo, payload, serializedPayload, auditTime); } public static class Builder diff --git a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java index b5bc7d71fb46..0a00b3a53ee3 100644 --- a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java +++ b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java @@ -19,14 +19,10 @@ package org.apache.druid.audit; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; import org.junit.Assert; import org.junit.Test; -import java.io.IOException; - public class AuditInfoTest { @Test @@ -39,9 +35,20 @@ public void testAuditInfoEquality() } @Test(timeout = 60_000L) - public void testAuditEntrySerde() throws IOException + public void testAuditEventEquality() { - AuditEvent entry = new AuditEvent( + final AuditEvent event1 = new AuditEvent( + "testKey", + "testType", + new AuditInfo( + "testAuthor", + "testComment", + "127.0.0.1" + ), + "testPayload", + DateTimes.of("2013-01-01T00:00:00Z") + ); + final AuditEvent event2 = new AuditEvent( "testKey", "testType", new AuditInfo( @@ -52,9 +59,7 @@ public void testAuditEntrySerde() throws IOException "testPayload", DateTimes.of("2013-01-01T00:00:00Z") ); - ObjectMapper mapper = new DefaultObjectMapper(); - AuditEvent serde = mapper.readValue(mapper.writeValueAsString(entry), AuditEvent.class); - Assert.assertEquals(entry, serde); + Assert.assertEquals(event1, event2); } } diff --git a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java index d4c9510e1627..4959538dcadb 100644 --- a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java @@ -348,7 +348,14 @@ private AuditEvent lookupAuditEntryForKey(String key) throws IOException if (payload == null) { return null; } else { - return mapper.readValue(payload, AuditEvent.class); + AuditRecord record = mapper.readValue(payload, AuditRecord.class); + return new AuditEvent( + record.getKey(), + record.getType(), + record.getAuditInfo(), + record.getPayload(), + record.getAuditTime() + ); } } From 23c29394ce8fddebdb62523c3c7de44ed893bc8b Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Mon, 4 Dec 2023 23:10:19 +0530 Subject: [PATCH 09/29] Remove unused variable --- .../org/apache/druid/server/audit/SQLAuditManagerTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java index 4959538dcadb..211f15f5d2f7 100644 --- a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.druid.audit.AuditEvent; import org.apache.druid.audit.AuditInfo; -import org.apache.druid.common.config.ConfigSerde; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; @@ -38,7 +37,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import java.io.IOException; @@ -290,7 +288,6 @@ public long getMaxPayloadSizeBytes() @Test(timeout = 60_000L) public void testCreateAuditEntryWithSkipNullsInPayload() throws IOException { - final ConfigSerde> mockConfigSerde = Mockito.mock(ConfigSerde.class); final SQLAuditManager auditManagerSkipNull = createAuditManager( new SQLAuditManagerConfig() { From ce41276c11823201902520b8528b386362d17d34 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 5 Dec 2023 08:04:12 +0530 Subject: [PATCH 10/29] Fix test --- .../apache/druid/server/audit/SQLAuditManagerTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java index 211f15f5d2f7..e596f2428f32 100644 --- a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java @@ -207,9 +207,9 @@ public void testRemoveAuditLogsOlderThanWithEntryNotOlderThanTime() throws IOExc @Test(timeout = 60_000L) public void testFetchAuditHistoryByTypeWithLimit() { - final AuditEvent entry1 = createAuditEvent("testKey", "testType", DateTimes.nowUtc()); - final AuditEvent entry2 = createAuditEvent("testKey", "testType", DateTimes.nowUtc()); - final AuditEvent entry3 = createAuditEvent("testKey", "testType", DateTimes.nowUtc()); + final AuditEvent entry1 = createAuditEvent("testKey", "testType", DateTimes.of("2022-01")); + final AuditEvent entry2 = createAuditEvent("testKey", "testType", DateTimes.of("2022-03")); + final AuditEvent entry3 = createAuditEvent("testKey", "testType", DateTimes.of("2022-02")); auditManager.doAudit(entry1); auditManager.doAudit(entry2); @@ -217,8 +217,8 @@ public void testFetchAuditHistoryByTypeWithLimit() List auditEntries = auditManager.fetchAuditHistory("testType", 2); Assert.assertEquals(2, auditEntries.size()); - Assert.assertEquals(entry1, auditEntries.get(0)); - Assert.assertEquals(entry2, auditEntries.get(1)); + Assert.assertEquals(entry2, auditEntries.get(0)); + Assert.assertEquals(entry3, auditEntries.get(1)); } @Test(expected = IllegalArgumentException.class, timeout = 10_000L) From b466857e9054e356c7a8acf0e4d6b916d49e5cc8 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 5 Dec 2023 15:12:57 +0530 Subject: [PATCH 11/29] Remove AuditRecord, use AuditEntry --- .../endpoint/BasicAuthenticatorResource.java | 4 +- .../overlord/http/OverlordResource.java | 8 +- .../{AuditEvent.java => AuditEntry.java} | 150 +++++++++++------- .../org/apache/druid/audit/AuditManager.java | 12 +- .../common/config/JacksonConfigManager.java | 4 +- .../org/apache/druid/audit/AuditInfoTest.java | 10 +- .../config/JacksonConfigManagerTest.java | 4 +- .../metadata/SQLMetadataRuleManager.java | 6 +- .../druid/server/audit/AuditLogger.java | 17 +- .../druid/server/audit/AuditRecord.java | 133 ---------------- .../druid/server/audit/AuditSerdeHelper.java | 36 +++-- .../server/audit/LoggingAuditManager.java | 14 +- .../druid/server/audit/SQLAuditManager.java | 52 +++--- .../CoordinatorCompactionConfigsResource.java | 8 +- .../server/http/DataSourcesResource.java | 6 +- .../druid/server/http/RulesResource.java | 4 +- .../metadata/SQLMetadataRuleManagerTest.java | 14 +- .../server/audit/SQLAuditManagerTest.java | 100 ++++++------ .../druid/server/http/RulesResourceTest.java | 101 +++--------- 19 files changed, 265 insertions(+), 418 deletions(-) rename processing/src/main/java/org/apache/druid/audit/{AuditEvent.java => AuditEntry.java} (59%) delete mode 100644 server/src/main/java/org/apache/druid/server/audit/AuditRecord.java diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java index 70168ed16382..1bd90e976d56 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes; import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.guice.LazySingleton; @@ -288,7 +288,7 @@ private boolean isSuccess(Response response) private void performAudit(String authenticatorName, String action, Object payload, AuditInfo auditInfo) { auditManager.doAudit( - AuditEvent.builder() + AuditEntry.builder() .key(authenticatorName) .type(action) .auditInfo(auditInfo) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index e0280945bf74..d514dc5621bb 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -28,7 +28,7 @@ import com.google.common.collect.Maps; import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.client.indexing.ClientTaskQuery; @@ -230,7 +230,7 @@ public Response taskPost( // Do an audit only if this API was called by a user and not by internal services if (author != null && !author.isEmpty()) { auditManager.doAudit( - AuditEvent.builder() + AuditEntry.builder() .key(task.getDataSource()) .type("ingestion.batch") .payload(new TaskIdentifier(task.getId(), task.getGroupId(), task.getType())) @@ -586,7 +586,7 @@ public Response getWorkerConfigHistory( Interval theInterval = interval == null ? null : Intervals.of(interval); if (theInterval == null && count != null) { try { - List workerEntryList = auditManager.fetchAuditHistory( + List workerEntryList = auditManager.fetchAuditHistory( WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.CONFIG_KEY, count @@ -599,7 +599,7 @@ public Response getWorkerConfigHistory( .build(); } } - List workerEntryList = auditManager.fetchAuditHistory( + List workerEntryList = auditManager.fetchAuditHistory( WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.CONFIG_KEY, theInterval diff --git a/processing/src/main/java/org/apache/druid/audit/AuditEvent.java b/processing/src/main/java/org/apache/druid/audit/AuditEntry.java similarity index 59% rename from processing/src/main/java/org/apache/druid/audit/AuditEvent.java rename to processing/src/main/java/org/apache/druid/audit/AuditEntry.java index 170628e86ff7..470905b5a014 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditEvent.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditEntry.java @@ -19,6 +19,9 @@ package org.apache.druid.audit; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.base.Preconditions; import org.apache.druid.java.util.common.DateTimes; import org.joda.time.DateTime; @@ -26,47 +29,24 @@ import java.util.Objects; /** - * Represents a single audit event with serialized payload. + * Serializable record of an audit event that can be persisted, logged or sent + * over REST APIs. */ -public class AuditEvent +public class AuditEntry { private final String key; private final String type; private final AuditInfo auditInfo; + private final Payload payload; private final DateTime auditTime; - private final String serializedPayload; - private final Object payload; - - private AuditEvent( - String key, - String type, - AuditInfo authorInfo, - DateTime auditTime, - Object payload - ) - { - this(key, type, authorInfo, payload, null, auditTime); - } - - public AuditEvent( - String key, - String type, - AuditInfo authorInfo, - String serializedPayload, - DateTime auditTime - ) - { - this(key, type, authorInfo, null, serializedPayload, auditTime); - } - - public AuditEvent( - String key, - String type, - AuditInfo authorInfo, - Object payload, - String serializedPayload, - DateTime auditTime + @JsonCreator + public AuditEntry( + @JsonProperty("key") String key, + @JsonProperty("type") String type, + @JsonProperty("auditInfo") AuditInfo authorInfo, + @JsonProperty("payload") Payload payload, + @JsonProperty("auditTime") DateTime auditTime ) { Preconditions.checkNotNull(key, "key cannot be null"); @@ -76,45 +56,45 @@ public AuditEvent( this.type = type; this.auditInfo = authorInfo; this.auditTime = auditTime == null ? DateTimes.nowUtc() : auditTime; - this.serializedPayload = serializedPayload; - this.payload = payload; + this.payload = payload == null ? Payload.fromString("") : payload; } + @JsonProperty public String getKey() { return key; } + @JsonProperty public String getType() { return type; } + @JsonProperty public AuditInfo getAuditInfo() { return auditInfo; } - public Object getPayload() + /** + * Non-null payload of the audit event. + */ + @JsonProperty + public Payload getPayload() { return payload; } - public String getPayloadAsString() - { - return serializedPayload; - } - + /** + * @return audit time as DateTime + */ + @JsonProperty public DateTime getAuditTime() { return auditTime; } - public static Builder builder() - { - return new Builder(); - } - @Override public boolean equals(Object o) { @@ -125,19 +105,23 @@ public boolean equals(Object o) return false; } - AuditEvent that = (AuditEvent) o; + AuditEntry that = (AuditEntry) o; return Objects.equals(this.auditTime, that.auditTime) && Objects.equals(this.key, that.key) && Objects.equals(this.type, that.type) && Objects.equals(this.auditInfo, that.auditInfo) - && Objects.equals(this.payload, that.payload) - && Objects.equals(this.serializedPayload, that.serializedPayload); + && Objects.equals(this.payload, that.payload); } @Override public int hashCode() { - return Objects.hash(key, type, auditInfo, payload, serializedPayload, auditTime); + return Objects.hash(key, type, auditInfo, payload, auditTime); + } + + public static Builder builder() + { + return new Builder(); } public static class Builder @@ -177,7 +161,7 @@ public Builder auditInfo(AuditInfo auditInfo) return this; } - public Builder payloadAsString(String serializedPayload) + public Builder serializedPayload(String serializedPayload) { this.serializedPayload = serializedPayload; return this; @@ -195,13 +179,65 @@ public Builder auditTime(DateTime auditTime) return this; } - public AuditEvent build() + public AuditEntry build() + { + return new AuditEntry(key, type, auditInfo, new Payload(serializedPayload, payload), auditTime); + } + } + + public static class Payload + { + private final String serialized; + private final Object raw; + + @JsonCreator + public static Payload fromString(String serialized) + { + return new Payload(serialized, null); + } + + @JsonValue + @Override + public String toString() + { + return serialized; + } + + private Payload(String serialized, Object raw) + { + this.serialized = serialized; + this.raw = raw; + } + + public String asString() + { + return serialized; + } + + public Object raw() + { + return raw; + } + + @Override + public boolean equals(Object o) { - if (payload != null) { - return new AuditEvent(key, type, auditInfo, auditTime, payload); - } else { - return new AuditEvent(key, type, auditInfo, serializedPayload, auditTime); + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; } + + Payload that = (Payload) o; + return Objects.equals(this.serialized, that.serialized) + && Objects.equals(this.raw, that.raw); + } + + @Override + public int hashCode() + { + return Objects.hash(serialized, raw); } } } diff --git a/processing/src/main/java/org/apache/druid/audit/AuditManager.java b/processing/src/main/java/org/apache/druid/audit/AuditManager.java index abc232346519..039fc3d5bc9c 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditManager.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditManager.java @@ -33,7 +33,7 @@ public interface AuditManager String X_DRUID_COMMENT = "X-Druid-Comment"; - void doAudit(AuditEvent event); + void doAudit(AuditEntry event); /** * Inserts an audit entry in audit table using the provided JDBI handle. @@ -44,7 +44,7 @@ public interface AuditManager * @param event Event to audit * @param handle JDBI Handle representing connection to the database */ - default void doAudit(AuditEvent event, Handle handle) throws IOException + default void doAudit(AuditEntry event, Handle handle) throws IOException { doAudit(event); } @@ -55,7 +55,7 @@ default void doAudit(AuditEvent event, Handle handle) throws IOException * * @return List of recorded audit events satisfying the passed parameters. */ - List fetchAuditHistory(String key, String type, Interval interval); + List fetchAuditHistory(String key, String type, Interval interval); /** * Fetches audit entries of a type whose audit time lies in the given interval. @@ -64,21 +64,21 @@ default void doAudit(AuditEvent event, Handle handle) throws IOException * @param interval Eligible interval for audit time * @return List of recorded audit events satisfying the passed parameters. */ - List fetchAuditHistory(String type, Interval interval); + List fetchAuditHistory(String type, Interval interval); /** * Provides last N entries of audit history for given key, type * * @return list of recorded audit events satisfying the passed parameters */ - List fetchAuditHistory(String key, String type, int limit); + List fetchAuditHistory(String key, String type, int limit); /** * Provides last N entries of audit history for given type * * @return List of recorded audit events satisfying the passed parameters. */ - List fetchAuditHistory(String type, int limit); + List fetchAuditHistory(String type, int limit); /** * Remove audit logs created older than the given timestamp. diff --git a/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java b/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java index c24177c160ea..92a9ea0ebee7 100644 --- a/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java +++ b/processing/src/main/java/org/apache/druid/common/config/JacksonConfigManager.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.common.config.ConfigManager.SetResult; @@ -117,7 +117,7 @@ public SetResult set( // Audit and actual config change are done in separate transactions // there can be phantom audits and reOrdering in audit changes as well. auditManager.doAudit( - AuditEvent.builder() + AuditEntry.builder() .key(key) .type(key) .auditInfo(auditInfo) diff --git a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java index 0a00b3a53ee3..888a4d11f074 100644 --- a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java +++ b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java @@ -35,9 +35,9 @@ public void testAuditInfoEquality() } @Test(timeout = 60_000L) - public void testAuditEventEquality() + public void testAuditEntryEquality() { - final AuditEvent event1 = new AuditEvent( + final AuditEntry event1 = new AuditEntry( "testKey", "testType", new AuditInfo( @@ -45,10 +45,10 @@ public void testAuditEventEquality() "testComment", "127.0.0.1" ), - "testPayload", + AuditEntry.Payload.fromString("testPayload"), DateTimes.of("2013-01-01T00:00:00Z") ); - final AuditEvent event2 = new AuditEvent( + final AuditEntry event2 = new AuditEntry( "testKey", "testType", new AuditInfo( @@ -56,7 +56,7 @@ public void testAuditEventEquality() "testComment", "127.0.0.1" ), - "testPayload", + AuditEntry.Payload.fromString("testPayload"), DateTimes.of("2013-01-01T00:00:00Z") ); Assert.assertEquals(event1, event2); diff --git a/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java b/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java index 876d2e89a3e4..85c548157c26 100644 --- a/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java +++ b/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.junit.Assert; @@ -71,7 +71,7 @@ public void testSet() jacksonConfigManager.set(key, val, auditInfo); - ArgumentCaptor auditCapture = ArgumentCaptor.forClass(AuditEvent.class); + ArgumentCaptor auditCapture = ArgumentCaptor.forClass(AuditEntry.class); Mockito.verify(mockAuditManager).doAudit(auditCapture.capture()); Assert.assertNotNull(auditCapture.getValue()); } diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java b/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java index a438b2cc6302..b83f77183aa5 100644 --- a/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java +++ b/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java @@ -25,7 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.client.DruidServer; @@ -348,11 +348,11 @@ public boolean overrideRule(final String dataSource, final List newRules, (handle, transactionStatus) -> { final DateTime auditTime = DateTimes.nowUtc(); auditManager.doAudit( - AuditEvent.builder() + AuditEntry.builder() .key(dataSource) .type("rules") .auditInfo(auditInfo) - .payloadAsString(ruleString) + .serializedPayload(ruleString) .auditTime(auditTime) .build(), handle diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java index b5316250c217..ca162baf232a 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java @@ -19,6 +19,7 @@ package org.apache.druid.server.audit; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.java.util.common.logger.Logger; public class AuditLogger @@ -39,16 +40,16 @@ public AuditLogger(Level level) this.level = level; } - public void log(AuditRecord record) + public void log(AuditEntry entry) { Object[] args = { - record.getAuditTime(), - record.getAuditInfo().getAuthor(), - record.getAuditInfo().getIp(), - record.getType(), - record.getKey(), - record.getAuditInfo().getComment(), - record.getPayload() + entry.getAuditTime(), + entry.getAuditInfo().getAuthor(), + entry.getAuditInfo().getIp(), + entry.getType(), + entry.getKey(), + entry.getAuditInfo().getComment(), + entry.getPayload() }; switch (level) { case DEBUG: diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditRecord.java b/server/src/main/java/org/apache/druid/server/audit/AuditRecord.java deleted file mode 100644 index d0829209a145..000000000000 --- a/server/src/main/java/org/apache/druid/server/audit/AuditRecord.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.server.audit; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; -import org.apache.druid.audit.AuditEvent; -import org.apache.druid.audit.AuditInfo; -import org.apache.druid.java.util.common.DateTimes; -import org.joda.time.DateTime; - -import java.util.Objects; - -/** - * Record of an audit event persisted in the metadata store. - */ -public class AuditRecord -{ - private final String key; - private final String type; - private final AuditInfo auditInfo; - private final String payload; - private final DateTime auditTime; - - public static AuditRecord fromAuditEvent(AuditEvent event) - { - return new AuditRecord( - event.getKey(), - event.getType(), - event.getAuditInfo(), - event.getPayloadAsString(), - event.getAuditTime() - ); - } - - @JsonCreator - public AuditRecord( - @JsonProperty("key") String key, - @JsonProperty("type") String type, - @JsonProperty("auditInfo") AuditInfo authorInfo, - @JsonProperty("payload") String payload, - @JsonProperty("auditTime") DateTime auditTime - ) - { - Preconditions.checkNotNull(key, "key cannot be null"); - Preconditions.checkNotNull(type, "type cannot be null"); - Preconditions.checkNotNull(authorInfo, "author cannot be null"); - this.key = key; - this.type = type; - this.auditInfo = authorInfo; - this.auditTime = auditTime == null ? DateTimes.nowUtc() : auditTime; - this.payload = payload == null ? "" : payload; - } - - @JsonProperty - public String getKey() - { - return key; - } - - @JsonProperty - public String getType() - { - return type; - } - - @JsonProperty - public AuditInfo getAuditInfo() - { - return auditInfo; - } - - /** - * @return Payload as a non-null String. - */ - @JsonProperty - public String getPayload() - { - return payload; - } - - /** - * @return audit time as DateTime - */ - @JsonProperty - public DateTime getAuditTime() - { - return auditTime; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - AuditRecord that = (AuditRecord) o; - return Objects.equals(this.auditTime, that.auditTime) - && Objects.equals(this.key, that.key) - && Objects.equals(this.type, that.type) - && Objects.equals(this.auditInfo, that.auditInfo) - && Objects.equals(this.payload, that.payload); - } - - @Override - public int hashCode() - { - return Objects.hash(key, type, auditInfo, payload, auditTime); - } - -} diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java index 41f7a628a5a5..85b6f527e0b0 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditManagerConfig; import org.apache.druid.guice.annotations.Json; import org.apache.druid.guice.annotations.JsonNonNull; @@ -60,35 +60,39 @@ public AuditSerdeHelper( this.jsonMapperSkipNulls = jsonMapperSkipNulls; } - public AuditRecord processAuditEvent(AuditEvent event) + public AuditEntry processAuditEntry(AuditEntry entry) { - final String serialized = event.getPayloadAsString() == null - ? serializePayloadToString(event.getPayload()) - : event.getPayloadAsString(); + final AuditEntry.Payload payload = entry.getPayload(); + final String serialized = payload.asString() == null + ? serializePayloadToString(payload.raw()) + : payload.asString(); - return new AuditRecord( - event.getKey(), - event.getType(), - event.getAuditInfo(), - truncateSerializedAuditPayload(serialized), - event.getAuditTime() + final AuditEntry.Payload processedPayload = AuditEntry.Payload.fromString( + truncateSerializedAuditPayload(serialized) + ); + return new AuditEntry( + entry.getKey(), + entry.getType(), + entry.getAuditInfo(), + processedPayload, + entry.getAuditTime() ); } - private String serializePayloadToString(Object payload) + private String serializePayloadToString(Object rawPayload) { - if (payload == null) { + if (rawPayload == null) { return ""; } try { return config.isSkipNullField() - ? jsonMapperSkipNulls.writeValueAsString(payload) - : jsonMapper.writeValueAsString(payload); + ? jsonMapperSkipNulls.writeValueAsString(rawPayload) + : jsonMapper.writeValueAsString(rawPayload); } catch (IOException e) { // Do not throw exception, only log error - log.error(e, "Could not serialize audit payload[%s]", payload); + log.error(e, "Could not serialize audit payload[%s]", rawPayload); return SERIALIZE_ERROR_MSG; } } diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java index a65102bf9183..c83155207bb0 100644 --- a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java @@ -20,7 +20,7 @@ package org.apache.druid.server.audit; import com.google.inject.Inject; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditManager; import org.joda.time.Interval; @@ -47,31 +47,31 @@ public LoggingAuditManager( } @Override - public void doAudit(AuditEvent event) + public void doAudit(AuditEntry entry) { - auditLogger.log(serdeHelper.processAuditEvent(event)); + auditLogger.log(serdeHelper.processAuditEntry(entry)); } @Override - public List fetchAuditHistory(String key, String type, Interval interval) + public List fetchAuditHistory(String key, String type, Interval interval) { return Collections.emptyList(); } @Override - public List fetchAuditHistory(String type, Interval interval) + public List fetchAuditHistory(String type, Interval interval) { return Collections.emptyList(); } @Override - public List fetchAuditHistory(String key, String type, int limit) + public List fetchAuditHistory(String key, String type, int limit) { return Collections.emptyList(); } @Override - public List fetchAuditHistory(String type, int limit) + public List fetchAuditHistory(String type, int limit) { return Collections.emptyList(); } diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java index f9a4a7c8cc02..d5b7b6de87dc 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Supplier; import com.google.inject.Inject; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditManager; import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.guice.annotations.Json; @@ -61,7 +61,7 @@ public class SQLAuditManager implements AuditManager private final SQLAuditManagerConfig config; private final AuditSerdeHelper serdeHelper; - private final ResultSetMapper resultMapper; + private final ResultSetMapper resultMapper; @Inject public SQLAuditManager( @@ -80,7 +80,7 @@ public SQLAuditManager( this.jsonMapper = jsonMapper; this.serdeHelper = serdeHelper; this.config = config; - this.resultMapper = new AuditEventMapper(); + this.resultMapper = new AuditEntryMapper(); } @LifecycleStart @@ -101,7 +101,7 @@ private String getAuditTable() } @Override - public void doAudit(AuditEvent event) + public void doAudit(AuditEntry event) { dbi.withHandle( handle -> { @@ -111,29 +111,29 @@ public void doAudit(AuditEvent event) ); } - private ServiceMetricEvent.Builder createMetricEventBuilder(AuditEvent auditEvent) + private ServiceMetricEvent.Builder createMetricEventBuilder(AuditEntry entry) { ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder() - .setDimension("key", auditEvent.getKey()) - .setDimension("type", auditEvent.getType()) - .setDimension("author", auditEvent.getAuditInfo().getAuthor()) - .setDimension("comment", auditEvent.getAuditInfo().getComment()) - .setDimension("remote_address", auditEvent.getAuditInfo().getIp()) - .setDimension("created_date", auditEvent.getAuditTime().toString()); + .setDimension("key", entry.getKey()) + .setDimension("type", entry.getType()) + .setDimension("author", entry.getAuditInfo().getAuthor()) + .setDimension("comment", entry.getAuditInfo().getComment()) + .setDimension("remote_address", entry.getAuditInfo().getIp()) + .setDimension("created_date", entry.getAuditTime().toString()); if (config.isIncludePayloadAsDimensionInMetric()) { - builder.setDimension("payload", auditEvent.getPayloadAsString()); + builder.setDimension("payload", entry.getPayload().asString()); } return builder; } @Override - public void doAudit(AuditEvent event, Handle handle) throws IOException + public void doAudit(AuditEntry event, Handle handle) throws IOException { emitter.emit(createMetricEventBuilder(event).setMetric("config/audit", 1)); - final AuditRecord record = serdeHelper.processAuditEvent(event); + final AuditEntry record = serdeHelper.processAuditEntry(event); handle.createStatement( StringUtils.format( "INSERT INTO %s (audit_key, type, author, comment, created_date, payload)" @@ -151,7 +151,7 @@ public void doAudit(AuditEvent event, Handle handle) throws IOException } @Override - public List fetchAuditHistory(final String key, final String type, Interval interval) + public List fetchAuditHistory(final String key, final String type, Interval interval) { final Interval theInterval = createAuditHistoryIntervalIfNull(interval); return dbi.withHandle( @@ -191,7 +191,7 @@ private int getLimit(int limit) throws IllegalArgumentException } @Override - public List fetchAuditHistory(final String type, Interval interval) + public List fetchAuditHistory(final String type, Interval interval) { final Interval theInterval = createAuditHistoryIntervalIfNull(interval); return dbi.withHandle( @@ -212,14 +212,14 @@ public List fetchAuditHistory(final String type, Interval interval) } @Override - public List fetchAuditHistory(final String key, final String type, int limit) + public List fetchAuditHistory(final String key, final String type, int limit) throws IllegalArgumentException { return fetchAuditHistoryLastEntries(key, type, limit); } @Override - public List fetchAuditHistory(final String type, int limit) + public List fetchAuditHistory(final String type, int limit) throws IllegalArgumentException { return fetchAuditHistoryLastEntries(null, type, limit); @@ -243,7 +243,7 @@ public int removeAuditLogsOlderThan(final long timestamp) ); } - private List fetchAuditHistoryLastEntries(final String key, final String type, int limit) + private List fetchAuditHistoryLastEntries(final String key, final String type, int limit) throws IllegalArgumentException { final int theLimit = getLimit(limit); @@ -269,20 +269,12 @@ private List fetchAuditHistoryLastEntries(final String key, final St ); } - private class AuditEventMapper implements ResultSetMapper + private class AuditEntryMapper implements ResultSetMapper { @Override - public AuditEvent map(int index, ResultSet r, StatementContext ctx) throws SQLException + public AuditEntry map(int index, ResultSet r, StatementContext ctx) throws SQLException { - // Read the record and convert to an AuditEvent that can deserialize the payload on-demand - AuditRecord record = JacksonUtils.readValue(jsonMapper, r.getBytes("payload"), AuditRecord.class); - return new AuditEvent( - record.getKey(), - record.getType(), - record.getAuditInfo(), - record.getPayload(), - record.getAuditTime() - ); + return JacksonUtils.readValue(jsonMapper, r.getBytes("payload"), AuditEntry.class); } } diff --git a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java index 910ee8bd8508..e76a7401e2e3 100644 --- a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java +++ b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.common.config.ConfigManager.SetResult; @@ -166,7 +166,7 @@ public Response getCompactionConfigHistory( { Interval theInterval = interval == null ? null : Intervals.of(interval); try { - List auditEntries; + List auditEntries; if (theInterval == null && count != null) { auditEntries = auditManager.fetchAuditHistory( CoordinatorCompactionConfig.CONFIG_KEY, @@ -181,9 +181,9 @@ public Response getCompactionConfigHistory( ); } DataSourceCompactionConfigHistory history = new DataSourceCompactionConfigHistory(dataSource); - for (AuditEvent audit : auditEntries) { + for (AuditEntry audit : auditEntries) { CoordinatorCompactionConfig coordinatorCompactionConfig = configManager.convertBytesToCompactionConfig( - audit.getPayloadAsString().getBytes(StandardCharsets.UTF_8) + audit.getPayload().asString().getBytes(StandardCharsets.UTF_8) ); history.add(coordinatorCompactionConfig, audit.getAuditInfo(), audit.getAuditTime()); } diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 5ebdaf0bf471..89de20a9f5cb 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -30,7 +30,7 @@ import com.sun.jersey.spi.container.ResourceFilters; import it.unimi.dsi.fastutil.objects.Object2LongMap; import org.apache.commons.lang.StringUtils; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.client.CoordinatorServerView; @@ -259,7 +259,7 @@ public Response markSegmentsAsUnused( } if (auditManager != null && author != null && !author.isEmpty()) { auditManager.doAudit( - AuditEvent.builder() + AuditEntry.builder() .key(dataSourceName) .type("segments.markUnused") .payload(auditPayload) @@ -381,7 +381,7 @@ public Response killUnusedSegmentsInInterval( ); if (auditManager != null && author != null && !author.isEmpty()) { auditManager.doAudit( - AuditEvent.builder() + AuditEntry.builder() .key(dataSourceName) .type("segments.killTask") .payload(ImmutableMap.of("killTaskId", killTaskId, "interval", theInterval)) diff --git a/server/src/main/java/org/apache/druid/server/http/RulesResource.java b/server/src/main/java/org/apache/druid/server/http/RulesResource.java index aafaa8471d88..beb223a6cc65 100644 --- a/server/src/main/java/org/apache/druid/server/http/RulesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/RulesResource.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.java.util.common.Intervals; @@ -161,7 +161,7 @@ public Response getDatasourceRuleHistory( } } - private List getRuleHistory( + private List getRuleHistory( final String dataSourceName, final String interval, final Integer count diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java index 84332e42c6e0..6368ab08fb2c 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java @@ -25,7 +25,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.client.DruidServer; @@ -183,13 +183,13 @@ public void testAuditEntryCreated() throws Exception Assert.assertEquals(rules, ruleManager.getRules(DATASOURCE)); // verify audit entry is created - List auditEntries = auditManager.fetchAuditHistory(DATASOURCE, "rules", null); + List auditEntries = auditManager.fetchAuditHistory(DATASOURCE, "rules", null); Assert.assertEquals(1, auditEntries.size()); - AuditEvent entry = auditEntries.get(0); + AuditEntry entry = auditEntries.get(0); Assert.assertEquals( rules, - mapper.readValue(entry.getPayloadAsString(), new TypeReference>() {}) + mapper.readValue(entry.getPayload().asString(), new TypeReference>() {}) ); Assert.assertEquals(auditInfo, entry.getAuditInfo()); Assert.assertEquals(DATASOURCE, entry.getKey()); @@ -217,12 +217,12 @@ public void testFetchAuditEntriesForAllDataSources() throws Exception Assert.assertEquals(rules, ruleManager.getRules("test_dataSource2")); // test fetch audit entries - List auditEntries = auditManager.fetchAuditHistory("rules", null); + List auditEntries = auditManager.fetchAuditHistory("rules", null); Assert.assertEquals(2, auditEntries.size()); - for (AuditEvent entry : auditEntries) { + for (AuditEntry entry : auditEntries) { Assert.assertEquals( rules, - mapper.readValue(entry.getPayloadAsString(), new TypeReference>() {}) + mapper.readValue(entry.getPayload().asString(), new TypeReference>() {}) ); Assert.assertEquals(auditInfo, entry.getAuditInfo()); } diff --git a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java index e596f2428f32..6a0d1fbc58f6 100644 --- a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; @@ -94,8 +94,8 @@ public boolean isIncludePayloadAsDimensionInMetric() } ); - final AuditEvent auditEvent = createAuditEvent("testKey", "testType", DateTimes.nowUtc()); - auditManager.doAudit(auditEvent); + final AuditEntry entry = createAuditEntry("testKey", "testType", DateTimes.nowUtc()); + auditManager.doAudit(entry); Map> metricEvents = serviceEmitter.getMetricEvents(); Assert.assertEquals(1, metricEvents.size()); @@ -106,24 +106,24 @@ public boolean isIncludePayloadAsDimensionInMetric() ServiceMetricEvent metric = auditMetricEvents.get(0); - final AuditEvent entry = lookupAuditEntryForKey("testKey"); - Assert.assertNotNull(entry); - Assert.assertEquals(entry.getKey(), metric.getUserDims().get("key")); - Assert.assertEquals(entry.getType(), metric.getUserDims().get("type")); - Assert.assertEquals(entry.getPayloadAsString(), metric.getUserDims().get("payload")); - Assert.assertEquals(entry.getAuditInfo().getAuthor(), metric.getUserDims().get("author")); - Assert.assertEquals(entry.getAuditInfo().getComment(), metric.getUserDims().get("comment")); - Assert.assertEquals(entry.getAuditInfo().getIp(), metric.getUserDims().get("remote_address")); + final AuditEntry dbEntry = lookupAuditEntryForKey("testKey"); + Assert.assertNotNull(dbEntry); + Assert.assertEquals(dbEntry.getKey(), metric.getUserDims().get("key")); + Assert.assertEquals(dbEntry.getType(), metric.getUserDims().get("type")); + Assert.assertEquals(dbEntry.getPayload().asString(), metric.getUserDims().get("payload")); + Assert.assertEquals(dbEntry.getAuditInfo().getAuthor(), metric.getUserDims().get("author")); + Assert.assertEquals(dbEntry.getAuditInfo().getComment(), metric.getUserDims().get("comment")); + Assert.assertEquals(dbEntry.getAuditInfo().getIp(), metric.getUserDims().get("remote_address")); } @Test(timeout = 60_000L) public void testCreateAuditEntry() throws IOException { - final AuditEvent auditEvent = createAuditEvent("key1", "type1", DateTimes.nowUtc()); - auditManager.doAudit(auditEvent); + final AuditEntry entry = createAuditEntry("key1", "type1", DateTimes.nowUtc()); + auditManager.doAudit(entry); - AuditEvent dbEvent = lookupAuditEntryForKey(auditEvent.getKey()); - Assert.assertEquals(auditEvent, dbEvent); + AuditEntry dbEntry = lookupAuditEntryForKey(entry.getKey()); + Assert.assertEquals(entry, dbEntry); // Verify emitted metrics Map> metricEvents = serviceEmitter.getMetricEvents(); @@ -134,22 +134,22 @@ public void testCreateAuditEntry() throws IOException Assert.assertEquals(1, auditMetricEvents.size()); ServiceMetricEvent metric = auditMetricEvents.get(0); - Assert.assertEquals(dbEvent.getKey(), metric.getUserDims().get("key")); - Assert.assertEquals(dbEvent.getType(), metric.getUserDims().get("type")); + Assert.assertEquals(dbEntry.getKey(), metric.getUserDims().get("key")); + Assert.assertEquals(dbEntry.getType(), metric.getUserDims().get("type")); Assert.assertNull(metric.getUserDims().get("payload")); - Assert.assertEquals(dbEvent.getAuditInfo().getAuthor(), metric.getUserDims().get("author")); - Assert.assertEquals(dbEvent.getAuditInfo().getComment(), metric.getUserDims().get("comment")); - Assert.assertEquals(dbEvent.getAuditInfo().getIp(), metric.getUserDims().get("remote_address")); + Assert.assertEquals(dbEntry.getAuditInfo().getAuthor(), metric.getUserDims().get("author")); + Assert.assertEquals(dbEntry.getAuditInfo().getComment(), metric.getUserDims().get("comment")); + Assert.assertEquals(dbEntry.getAuditInfo().getIp(), metric.getUserDims().get("remote_address")); } @Test(timeout = 60_000L) public void testFetchAuditHistory() { - final AuditEvent event = createAuditEvent("testKey", "testType", DateTimes.nowUtc()); + final AuditEntry event = createAuditEntry("testKey", "testType", DateTimes.nowUtc()); auditManager.doAudit(event); auditManager.doAudit(event); - List auditEntries = auditManager.fetchAuditHistory( + List auditEntries = auditManager.fetchAuditHistory( "testKey", "testType", Intervals.of("2000-01-01T00:00:00Z/2100-01-03T00:00:00Z") @@ -163,13 +163,13 @@ public void testFetchAuditHistory() @Test(timeout = 60_000L) public void testFetchAuditHistoryByKeyAndTypeWithLimit() { - final AuditEvent entry1 = createAuditEvent("key1", "type1", DateTimes.nowUtc()); - final AuditEvent entry2 = createAuditEvent("key2", "type2", DateTimes.nowUtc()); + final AuditEntry entry1 = createAuditEntry("key1", "type1", DateTimes.nowUtc()); + final AuditEntry entry2 = createAuditEntry("key2", "type2", DateTimes.nowUtc()); auditManager.doAudit(entry1); auditManager.doAudit(entry2); - List auditEntries = auditManager.fetchAuditHistory(entry1.getKey(), entry1.getType(), 1); + List auditEntries = auditManager.fetchAuditHistory(entry1.getKey(), entry1.getType(), 1); Assert.assertEquals(1, auditEntries.size()); Assert.assertEquals(entry1, auditEntries.get(0)); } @@ -177,10 +177,10 @@ public void testFetchAuditHistoryByKeyAndTypeWithLimit() @Test(timeout = 60_000L) public void testRemoveAuditLogsOlderThanWithEntryOlderThanTime() throws IOException { - final AuditEvent entry = createAuditEvent("key1", "type1", DateTimes.nowUtc()); + final AuditEntry entry = createAuditEntry("key1", "type1", DateTimes.nowUtc()); auditManager.doAudit(entry); - AuditEvent dbEntry = lookupAuditEntryForKey(entry.getKey()); + AuditEntry dbEntry = lookupAuditEntryForKey(entry.getKey()); Assert.assertEquals(entry, dbEntry); // Verify that the audit entry gets deleted @@ -191,10 +191,10 @@ public void testRemoveAuditLogsOlderThanWithEntryOlderThanTime() throws IOExcept @Test(timeout = 60_000L) public void testRemoveAuditLogsOlderThanWithEntryNotOlderThanTime() throws IOException { - AuditEvent entry = createAuditEvent("key", "type", DateTimes.nowUtc()); + AuditEntry entry = createAuditEntry("key", "type", DateTimes.nowUtc()); auditManager.doAudit(entry); - AuditEvent dbEntry = lookupAuditEntryForKey(entry.getKey()); + AuditEntry dbEntry = lookupAuditEntryForKey(entry.getKey()); Assert.assertEquals(entry, dbEntry); // Delete old audit logs @@ -207,15 +207,15 @@ public void testRemoveAuditLogsOlderThanWithEntryNotOlderThanTime() throws IOExc @Test(timeout = 60_000L) public void testFetchAuditHistoryByTypeWithLimit() { - final AuditEvent entry1 = createAuditEvent("testKey", "testType", DateTimes.of("2022-01")); - final AuditEvent entry2 = createAuditEvent("testKey", "testType", DateTimes.of("2022-03")); - final AuditEvent entry3 = createAuditEvent("testKey", "testType", DateTimes.of("2022-02")); + final AuditEntry entry1 = createAuditEntry("testKey", "testType", DateTimes.of("2022-01")); + final AuditEntry entry2 = createAuditEntry("testKey", "testType", DateTimes.of("2022-03")); + final AuditEntry entry3 = createAuditEntry("testKey", "testType", DateTimes.of("2022-02")); auditManager.doAudit(entry1); auditManager.doAudit(entry2); auditManager.doAudit(entry3); - List auditEntries = auditManager.fetchAuditHistory("testType", 2); + List auditEntries = auditManager.fetchAuditHistory("testType", 2); Assert.assertEquals(2, auditEntries.size()); Assert.assertEquals(entry2, auditEntries.get(0)); Assert.assertEquals(entry3, auditEntries.get(1)); @@ -248,16 +248,16 @@ public long getMaxPayloadSizeBytes() } ); - final AuditEvent entry = createAuditEvent("key", "type", DateTimes.nowUtc()); + final AuditEntry entry = createAuditEntry("key", "type", DateTimes.nowUtc()); auditManager.doAudit(entry); // Verify that all the fields are the same except for the payload - AuditEvent dbEntry = lookupAuditEntryForKey(entry.getKey()); + AuditEntry dbEntry = lookupAuditEntryForKey(entry.getKey()); Assert.assertEquals(entry.getKey(), dbEntry.getKey()); // Assert.assertNotEquals(entry.getPayload(), dbEntry.getPayload()); Assert.assertEquals( "Payload truncated as it exceeds 'druid.audit.manager.maxPayloadSizeBytes'[10].", - dbEntry.getPayloadAsString() + dbEntry.getPayload().asString() ); Assert.assertEquals(entry.getType(), dbEntry.getType()); Assert.assertEquals(entry.getAuditInfo(), dbEntry.getAuditInfo()); @@ -277,11 +277,11 @@ public long getMaxPayloadSizeBytes() } ); - final AuditEvent entry = createAuditEvent("key", "type", DateTimes.nowUtc()); + final AuditEntry entry = createAuditEntry("key", "type", DateTimes.nowUtc()); auditManager.doAudit(entry); // Verify that the actual payload has been persisted - AuditEvent dbEntry = lookupAuditEntryForKey(entry.getKey()); + AuditEntry dbEntry = lookupAuditEntryForKey(entry.getKey()); Assert.assertEquals(entry, dbEntry); } @@ -306,16 +306,16 @@ public boolean isSkipNullField() payloadMap.put("something", null); auditManager.doAudit( - AuditEvent.builder().key("key1").type("type1").auditInfo(auditInfo).payload(payloadMap).build() + AuditEntry.builder().key("key1").type("type1").auditInfo(auditInfo).payload(payloadMap).build() ); - AuditEvent entryWithNulls = lookupAuditEntryForKey("key1"); - Assert.assertEquals("{\"something\":null,\"version\":\"x\"}", entryWithNulls.getPayloadAsString()); + AuditEntry entryWithNulls = lookupAuditEntryForKey("key1"); + Assert.assertEquals("{\"something\":null,\"version\":\"x\"}", entryWithNulls.getPayload().asString()); auditManagerSkipNull.doAudit( - AuditEvent.builder().key("key2").type("type2").auditInfo(auditInfo).payload(payloadMap).build() + AuditEntry.builder().key("key2").type("type2").auditInfo(auditInfo).payload(payloadMap).build() ); - AuditEvent entryWithoutNulls = lookupAuditEntryForKey("key2"); - Assert.assertEquals("{\"version\":\"x\"}", entryWithoutNulls.getPayloadAsString()); + AuditEntry entryWithoutNulls = lookupAuditEntryForKey("key2"); + Assert.assertEquals("{\"version\":\"x\"}", entryWithoutNulls.getPayload().asString()); } @After @@ -333,7 +333,7 @@ private void dropTable(final String tableName) Assert.assertEquals(0, rowsAffected); } - private AuditEvent lookupAuditEntryForKey(String key) throws IOException + private AuditEntry lookupAuditEntryForKey(String key) throws IOException { byte[] payload = connector.lookup( derbyConnectorRule.metadataTablesConfigSupplier().get().getAuditTable(), @@ -345,8 +345,8 @@ private AuditEvent lookupAuditEntryForKey(String key) throws IOException if (payload == null) { return null; } else { - AuditRecord record = mapper.readValue(payload, AuditRecord.class); - return new AuditEvent( + AuditEntry record = mapper.readValue(payload, AuditEntry.class); + return new AuditEntry( record.getKey(), record.getType(), record.getAuditInfo(), @@ -356,12 +356,12 @@ private AuditEvent lookupAuditEntryForKey(String key) throws IOException } } - private AuditEvent createAuditEvent(String key, String type, DateTime auditTime) + private AuditEntry createAuditEntry(String key, String type, DateTime auditTime) { - return AuditEvent.builder() + return AuditEntry.builder() .key(key) .type(type) - .payloadAsString(StringUtils.format("Test payload for key[%s], type[%s]", key, type)) + .serializedPayload(StringUtils.format("Test payload for key[%s], type[%s]", key, type)) .auditInfo(new AuditInfo("author", "comment", "127.0.0.1")) .auditTime(auditTime) .build(); diff --git a/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java index c2cb5ed2c571..f6ce89649b66 100644 --- a/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java @@ -20,13 +20,14 @@ package org.apache.druid.server.http; import com.google.common.collect.ImmutableList; -import org.apache.druid.audit.AuditEvent; +import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.metadata.MetadataRuleManager; import org.easymock.EasyMock; +import org.joda.time.DateTime; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Before; @@ -51,26 +52,10 @@ public void setUp() @Test public void testGetDatasourceRuleHistoryWithCount() { - AuditEvent entry1 = new AuditEvent( - "testKey", - "testType", - new AuditInfo( - "testAuthor", - "testComment", - "127.0.0.1" - ), - "testPayload", + AuditEntry entry1 = createAuditEntry( DateTimes.of("2013-01-02T00:00:00Z") ); - AuditEvent entry2 = new AuditEvent( - "testKey", - "testType", - new AuditInfo( - "testAuthor", - "testComment", - "127.0.0.1" - ), - "testPayload", + AuditEntry entry2 = createAuditEntry( DateTimes.of("2013-01-01T00:00:00Z") ); EasyMock.expect(auditManager.fetchAuditHistory(EasyMock.eq("datasource1"), EasyMock.eq("rules"), EasyMock.eq(2))) @@ -81,7 +66,7 @@ public void testGetDatasourceRuleHistoryWithCount() RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); Response response = rulesResource.getDatasourceRuleHistory("datasource1", null, 2); - List rulesHistory = (List) response.getEntity(); + List rulesHistory = (List) response.getEntity(); Assert.assertEquals(2, rulesHistory.size()); Assert.assertEquals(entry1, rulesHistory.get(0)); Assert.assertEquals(entry2, rulesHistory.get(1)); @@ -94,26 +79,10 @@ public void testGetDatasourceRuleHistoryWithInterval() { String interval = "P2D/2013-01-02T00:00:00Z"; Interval theInterval = Intervals.of(interval); - AuditEvent entry1 = new AuditEvent( - "testKey", - "testType", - new AuditInfo( - "testAuthor", - "testComment", - "127.0.0.1" - ), - "testPayload", + AuditEntry entry1 = createAuditEntry( DateTimes.of("2013-01-02T00:00:00Z") ); - AuditEvent entry2 = new AuditEvent( - "testKey", - "testType", - new AuditInfo( - "testAuthor", - "testComment", - "127.0.0.1" - ), - "testPayload", + AuditEntry entry2 = createAuditEntry( DateTimes.of("2013-01-01T00:00:00Z") ); EasyMock.expect(auditManager.fetchAuditHistory(EasyMock.eq("datasource1"), EasyMock.eq("rules"), EasyMock.eq(theInterval))) @@ -124,7 +93,7 @@ public void testGetDatasourceRuleHistoryWithInterval() RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); Response response = rulesResource.getDatasourceRuleHistory("datasource1", interval, null); - List rulesHistory = (List) response.getEntity(); + List rulesHistory = (List) response.getEntity(); Assert.assertEquals(2, rulesHistory.size()); Assert.assertEquals(entry1, rulesHistory.get(0)); Assert.assertEquals(entry2, rulesHistory.get(1)); @@ -154,26 +123,10 @@ public void testGetDatasourceRuleHistoryWithWrongCount() @Test public void testGetAllDatasourcesRuleHistoryWithCount() { - AuditEvent entry1 = new AuditEvent( - "testKey", - "testType", - new AuditInfo( - "testAuthor", - "testComment", - "127.0.0.1" - ), - "testPayload", + AuditEntry entry1 = createAuditEntry( DateTimes.of("2013-01-02T00:00:00Z") ); - AuditEvent entry2 = new AuditEvent( - "testKey", - "testType", - new AuditInfo( - "testAuthor", - "testComment", - "127.0.0.1" - ), - "testPayload", + AuditEntry entry2 = createAuditEntry( DateTimes.of("2013-01-01T00:00:00Z") ); EasyMock.expect(auditManager.fetchAuditHistory(EasyMock.eq("rules"), EasyMock.eq(2))) @@ -184,7 +137,7 @@ public void testGetAllDatasourcesRuleHistoryWithCount() RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); Response response = rulesResource.getDatasourceRuleHistory(null, 2); - List rulesHistory = (List) response.getEntity(); + List rulesHistory = (List) response.getEntity(); Assert.assertEquals(2, rulesHistory.size()); Assert.assertEquals(entry1, rulesHistory.get(0)); Assert.assertEquals(entry2, rulesHistory.get(1)); @@ -197,26 +150,10 @@ public void testGetAllDatasourcesRuleHistoryWithInterval() { String interval = "P2D/2013-01-02T00:00:00Z"; Interval theInterval = Intervals.of(interval); - AuditEvent entry1 = new AuditEvent( - "testKey", - "testType", - new AuditInfo( - "testAuthor", - "testComment", - "127.0.0.1" - ), - "testPayload", + AuditEntry entry1 = createAuditEntry( DateTimes.of("2013-01-02T00:00:00Z") ); - AuditEvent entry2 = new AuditEvent( - "testKey", - "testType", - new AuditInfo( - "testAuthor", - "testComment", - "127.0.0.1" - ), - "testPayload", + AuditEntry entry2 = createAuditEntry( DateTimes.of("2013-01-01T00:00:00Z") ); EasyMock.expect(auditManager.fetchAuditHistory(EasyMock.eq("rules"), EasyMock.eq(theInterval))) @@ -227,7 +164,7 @@ public void testGetAllDatasourcesRuleHistoryWithInterval() RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); Response response = rulesResource.getDatasourceRuleHistory(interval, null); - List rulesHistory = (List) response.getEntity(); + List rulesHistory = (List) response.getEntity(); Assert.assertEquals(2, rulesHistory.size()); Assert.assertEquals(entry1, rulesHistory.get(0)); Assert.assertEquals(entry2, rulesHistory.get(1)); @@ -254,4 +191,14 @@ public void testGetAllDatasourcesRuleHistoryWithWrongCount() EasyMock.verify(auditManager); } + private AuditInfo createAuditInfo() + { + return new AuditInfo("testAuthor", "testComment", "127.0.0.1"); + } + + private AuditEntry createAuditEntry(DateTime auditTime) + { + return new AuditEntry("testKey", "testType", createAuditInfo(), AuditEntry.Payload.fromString("testPayload"), auditTime); + } + } From 6c438d509a8a99a14b68065f6502b44e080d8b85 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Thu, 7 Dec 2023 19:19:28 +0530 Subject: [PATCH 12/29] Add identity field in AuditInfo --- .../endpoint/BasicAuthenticatorResource.java | 7 ++-- .../overlord/http/OverlordResource.java | 4 +-- .../org/apache/druid/audit/AuditEntry.java | 23 ++++++------ .../org/apache/druid/audit/AuditInfo.java | 26 +++++++++++--- .../org/apache/druid/audit/AuditInfoTest.java | 36 +++++++++++-------- .../druid/server/audit/AuditSerdeHelper.java | 17 +++++++-- .../druid/server/audit/SQLAuditManager.java | 2 +- .../CoordinatorCompactionConfigsResource.java | 9 ++--- .../server/security/AuthorizationUtils.java | 28 +++++++++++++++ .../metadata/SQLMetadataRuleManagerTest.java | 4 +-- .../server/audit/SQLAuditManagerTest.java | 10 +++--- .../druid/server/http/RulesResourceTest.java | 2 +- 12 files changed, 117 insertions(+), 51 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java index 1bd90e976d56..ca07c58d3be9 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java @@ -29,6 +29,7 @@ import org.apache.druid.security.basic.BasicSecurityResourceFilter; import org.apache.druid.security.basic.authentication.entity.BasicAuthenticatorCredentialUpdate; import org.apache.druid.server.security.AuthValidator; +import org.apache.druid.server.security.AuthorizationUtils; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; @@ -166,7 +167,7 @@ public Response createUser( final Response response = handler.createUser(authenticatorName, userName); if (isSuccess(response)) { - final AuditInfo auditInfo = new AuditInfo(author, comment, req.getRemoteAddr()); + final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(author, comment, req); performAudit(authenticatorName, "users.create", Collections.singletonMap("username", userName), auditInfo); } @@ -198,7 +199,7 @@ public Response deleteUser( final Response response = handler.deleteUser(authenticatorName, userName); if (isSuccess(response)) { - final AuditInfo auditInfo = new AuditInfo(author, comment, req.getRemoteAddr()); + final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(author, comment, req); performAudit(authenticatorName, "users.delete", Collections.singletonMap("username", userName), auditInfo); } @@ -231,7 +232,7 @@ public Response updateUserCredentials( final Response response = handler.updateUserCredentials(authenticatorName, userName, update); if (isSuccess(response)) { - final AuditInfo auditInfo = new AuditInfo(author, comment, req.getRemoteAddr()); + final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(author, comment, req); performAudit(authenticatorName, "users.update", Collections.singletonMap("username", userName), auditInfo); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index d514dc5621bb..215586f86baf 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -232,9 +232,9 @@ public Response taskPost( auditManager.doAudit( AuditEntry.builder() .key(task.getDataSource()) - .type("ingestion.batch") + .type("overlord" + req.getServletPath()) .payload(new TaskIdentifier(task.getId(), task.getGroupId(), task.getType())) - .auditInfo(new AuditInfo(author, comment, req.getRemoteAddr())) + .auditInfo(AuthorizationUtils.buildAuditInfo(author, comment, req)) .build() ); } diff --git a/processing/src/main/java/org/apache/druid/audit/AuditEntry.java b/processing/src/main/java/org/apache/druid/audit/AuditEntry.java index 470905b5a014..a8a420915654 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditEntry.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditEntry.java @@ -95,6 +95,11 @@ public DateTime getAuditTime() return auditTime; } + public static Builder builder() + { + return new Builder(); + } + @Override public boolean equals(Object o) { @@ -119,27 +124,18 @@ public int hashCode() return Objects.hash(key, type, auditInfo, payload, auditTime); } - public static Builder builder() - { - return new Builder(); - } - public static class Builder { private String key; private String type; private AuditInfo auditInfo; - - private String serializedPayload; private Object payload; + private String serializedPayload; private DateTime auditTime; private Builder() { - this.key = null; - this.auditInfo = null; - this.serializedPayload = null; this.auditTime = DateTimes.nowUtc(); } @@ -185,6 +181,9 @@ public AuditEntry build() } } + /** + * Payload of an {@link AuditEntry} that may be specified {@link #raw()} or {@link #serialized()}. + */ public static class Payload { private final String serialized; @@ -200,7 +199,7 @@ public static Payload fromString(String serialized) @Override public String toString() { - return serialized; + return serialized == null ? "" : serialized; } private Payload(String serialized, Object raw) @@ -209,7 +208,7 @@ private Payload(String serialized, Object raw) this.raw = raw; } - public String asString() + public String serialized() { return serialized; } diff --git a/processing/src/main/java/org/apache/druid/audit/AuditInfo.java b/processing/src/main/java/org/apache/druid/audit/AuditInfo.java index cab62c9a0289..206acbd1353d 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditInfo.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditInfo.java @@ -27,27 +27,41 @@ public class AuditInfo { private final String author; + private final String identity; private final String comment; private final String ip; @JsonCreator public AuditInfo( @JsonProperty("author") String author, + @JsonProperty("identity") String identity, @JsonProperty("comment") String comment, @JsonProperty("ip") String ip ) { this.author = author; + this.identity = identity; this.comment = comment; this.ip = ip; } + public AuditInfo(String author, String comment, String ip) + { + this(author, null, comment, ip); + } + @JsonProperty public String getAuthor() { return author; } + @JsonProperty + public String getIdentity() + { + return identity; + } + @JsonProperty public String getComment() { @@ -69,16 +83,17 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - AuditInfo auditInfo = (AuditInfo) o; - return Objects.equals(author, auditInfo.author) - && Objects.equals(comment, auditInfo.comment) - && Objects.equals(ip, auditInfo.ip); + AuditInfo that = (AuditInfo) o; + return Objects.equals(this.author, that.author) + && Objects.equals(this.identity, that.identity) + && Objects.equals(this.comment, that.comment) + && Objects.equals(this.ip, that.ip); } @Override public int hashCode() { - return Objects.hash(author, comment, ip); + return Objects.hash(author, identity, comment, ip); } @Override @@ -86,6 +101,7 @@ public String toString() { return "AuditInfo{" + "author='" + author + '\'' + + ", identity='" + identity + '\'' + ", comment='" + comment + '\'' + ", ip='" + ip + '\'' + '}'; diff --git a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java index 888a4d11f074..c0f3dc69118d 100644 --- a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java +++ b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java @@ -19,12 +19,18 @@ package org.apache.druid.audit; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; + public class AuditInfoTest { + private final ObjectMapper mapper = new DefaultObjectMapper(); + @Test public void testAuditInfoEquality() { @@ -34,21 +40,22 @@ public void testAuditInfoEquality() Assert.assertEquals(auditInfo1.hashCode(), auditInfo2.hashCode()); } + @Test + public void testAuditInfoSerde() throws IOException + { + final AuditInfo auditInfo = new AuditInfo("author", "comment", "ip"); + AuditInfo deserialized = mapper.readValue(mapper.writeValueAsString(auditInfo), AuditInfo.class); + Assert.assertEquals(auditInfo, deserialized); + + final AuditInfo auditInfoWithIdentity = new AuditInfo("author", "identity", "comment", "ip"); + deserialized = mapper.readValue(mapper.writeValueAsString(auditInfoWithIdentity), AuditInfo.class); + Assert.assertEquals(auditInfoWithIdentity, deserialized); + } + @Test(timeout = 60_000L) - public void testAuditEntryEquality() + public void testAuditEntrySerde() throws IOException { - final AuditEntry event1 = new AuditEntry( - "testKey", - "testType", - new AuditInfo( - "testAuthor", - "testComment", - "127.0.0.1" - ), - AuditEntry.Payload.fromString("testPayload"), - DateTimes.of("2013-01-01T00:00:00Z") - ); - final AuditEntry event2 = new AuditEntry( + AuditEntry entry = new AuditEntry( "testKey", "testType", new AuditInfo( @@ -59,7 +66,8 @@ public void testAuditEntryEquality() AuditEntry.Payload.fromString("testPayload"), DateTimes.of("2013-01-01T00:00:00Z") ); - Assert.assertEquals(event1, event2); + AuditEntry serde = mapper.readValue(mapper.writeValueAsString(entry), AuditEntry.class); + Assert.assertEquals(entry, serde); } } diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java index 85b6f527e0b0..29befd71e35c 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java @@ -31,6 +31,11 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +/** + * Audit utility class that can be used by different implementations of + * {@link org.apache.druid.audit.AuditManager} to serialize/deserialize audit + * payloads based on the values configured in {@link AuditManagerConfig}. + */ public class AuditSerdeHelper { /** @@ -60,12 +65,20 @@ public AuditSerdeHelper( this.jsonMapperSkipNulls = jsonMapperSkipNulls; } + /** + * Processes the given AuditEntry for further use such as logging or persistence. + * This involves serializing and truncating the payload based on the values + * configured in {@link AuditManagerConfig}. + * + * @return A new AuditEntry with a serialized payload that can be used for + * logging or persistence. + */ public AuditEntry processAuditEntry(AuditEntry entry) { final AuditEntry.Payload payload = entry.getPayload(); - final String serialized = payload.asString() == null + final String serialized = payload.serialized() == null ? serializePayloadToString(payload.raw()) - : payload.asString(); + : payload.serialized(); final AuditEntry.Payload processedPayload = AuditEntry.Payload.fromString( truncateSerializedAuditPayload(serialized) diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java index d5b7b6de87dc..019f0156f42f 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java @@ -122,7 +122,7 @@ private ServiceMetricEvent.Builder createMetricEventBuilder(AuditEntry entry) .setDimension("created_date", entry.getAuditTime().toString()); if (config.isIncludePayloadAsDimensionInMetric()) { - builder.setDimension("payload", entry.getPayload().asString()); + builder.setDimension("payload", entry.getPayload().serialized()); } return builder; diff --git a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java index e76a7401e2e3..e0e2faed69e2 100644 --- a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java +++ b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java @@ -35,6 +35,7 @@ import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.server.coordinator.DataSourceCompactionConfigHistory; import org.apache.druid.server.http.security.ConfigResourceFilter; +import org.apache.druid.server.security.AuthorizationUtils; import org.joda.time.Interval; import javax.servlet.http.HttpServletRequest; @@ -107,7 +108,7 @@ public Response setCompactionTaskLimit( maxCompactionTaskSlots, useAutoScaleSlots ); - return updateConfigHelper(operator, new AuditInfo(author, comment, req.getRemoteAddr())); + return updateConfigHelper(operator, AuthorizationUtils.buildAuditInfo(author, comment, req)); } @POST @@ -132,7 +133,7 @@ public Response addOrUpdateCompactionConfig( }; return updateConfigHelper( callable, - new AuditInfo(author, comment, req.getRemoteAddr()) + AuthorizationUtils.buildAuditInfo(author, comment, req) ); } @@ -183,7 +184,7 @@ public Response getCompactionConfigHistory( DataSourceCompactionConfigHistory history = new DataSourceCompactionConfigHistory(dataSource); for (AuditEntry audit : auditEntries) { CoordinatorCompactionConfig coordinatorCompactionConfig = configManager.convertBytesToCompactionConfig( - audit.getPayload().asString().getBytes(StandardCharsets.UTF_8) + audit.getPayload().serialized().getBytes(StandardCharsets.UTF_8) ); history.add(coordinatorCompactionConfig, audit.getAuditInfo(), audit.getAuditTime()); } @@ -219,7 +220,7 @@ public Response deleteCompactionConfig( return CoordinatorCompactionConfig.from(current, ImmutableList.copyOf(configs.values())); }; - return updateConfigHelper(callable, new AuditInfo(author, comment, req.getRemoteAddr())); + return updateConfigHelper(callable, AuthorizationUtils.buildAuditInfo(author, comment, req)); } private Response updateConfigHelper( diff --git a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java index 2b2afa9a9cdd..bf54afc8afda 100644 --- a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java +++ b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import org.apache.druid.audit.AuditInfo; import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.ISE; @@ -89,6 +90,33 @@ public static AuthenticationResult authenticationResultFromRequest(final HttpSer return authenticationResult; } + /** + * Extracts the identity from the authentication result if set as an atrribute + * of this request. + */ + public static String getAuthenticatedIdentity(HttpServletRequest request) + { + final AuthenticationResult authenticationResult = (AuthenticationResult) request.getAttribute( + AuthConfig.DRUID_AUTHENTICATION_RESULT + ); + + if (authenticationResult == null) { + return null; + } else { + return authenticationResult.getIdentity(); + } + } + + public static AuditInfo buildAuditInfo(String author, String comment, HttpServletRequest request) + { + return new AuditInfo( + author, + getAuthenticatedIdentity(request), + comment, + request.getRemoteAddr() + ); + } + /** * Check a list of resource-actions to be performed by the identity represented by authenticationResult. * diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java index 6368ab08fb2c..940b770d0868 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java @@ -189,7 +189,7 @@ public void testAuditEntryCreated() throws Exception Assert.assertEquals( rules, - mapper.readValue(entry.getPayload().asString(), new TypeReference>() {}) + mapper.readValue(entry.getPayload().serialized(), new TypeReference>() {}) ); Assert.assertEquals(auditInfo, entry.getAuditInfo()); Assert.assertEquals(DATASOURCE, entry.getKey()); @@ -222,7 +222,7 @@ public void testFetchAuditEntriesForAllDataSources() throws Exception for (AuditEntry entry : auditEntries) { Assert.assertEquals( rules, - mapper.readValue(entry.getPayload().asString(), new TypeReference>() {}) + mapper.readValue(entry.getPayload().serialized(), new TypeReference>() {}) ); Assert.assertEquals(auditInfo, entry.getAuditInfo()); } diff --git a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java index 6a0d1fbc58f6..409b7196752b 100644 --- a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java @@ -110,7 +110,7 @@ public boolean isIncludePayloadAsDimensionInMetric() Assert.assertNotNull(dbEntry); Assert.assertEquals(dbEntry.getKey(), metric.getUserDims().get("key")); Assert.assertEquals(dbEntry.getType(), metric.getUserDims().get("type")); - Assert.assertEquals(dbEntry.getPayload().asString(), metric.getUserDims().get("payload")); + Assert.assertEquals(dbEntry.getPayload().serialized(), metric.getUserDims().get("payload")); Assert.assertEquals(dbEntry.getAuditInfo().getAuthor(), metric.getUserDims().get("author")); Assert.assertEquals(dbEntry.getAuditInfo().getComment(), metric.getUserDims().get("comment")); Assert.assertEquals(dbEntry.getAuditInfo().getIp(), metric.getUserDims().get("remote_address")); @@ -257,7 +257,7 @@ public long getMaxPayloadSizeBytes() // Assert.assertNotEquals(entry.getPayload(), dbEntry.getPayload()); Assert.assertEquals( "Payload truncated as it exceeds 'druid.audit.manager.maxPayloadSizeBytes'[10].", - dbEntry.getPayload().asString() + dbEntry.getPayload().serialized() ); Assert.assertEquals(entry.getType(), dbEntry.getType()); Assert.assertEquals(entry.getAuditInfo(), dbEntry.getAuditInfo()); @@ -309,13 +309,13 @@ public boolean isSkipNullField() AuditEntry.builder().key("key1").type("type1").auditInfo(auditInfo).payload(payloadMap).build() ); AuditEntry entryWithNulls = lookupAuditEntryForKey("key1"); - Assert.assertEquals("{\"something\":null,\"version\":\"x\"}", entryWithNulls.getPayload().asString()); + Assert.assertEquals("{\"something\":null,\"version\":\"x\"}", entryWithNulls.getPayload().serialized()); auditManagerSkipNull.doAudit( AuditEntry.builder().key("key2").type("type2").auditInfo(auditInfo).payload(payloadMap).build() ); AuditEntry entryWithoutNulls = lookupAuditEntryForKey("key2"); - Assert.assertEquals("{\"version\":\"x\"}", entryWithoutNulls.getPayload().asString()); + Assert.assertEquals("{\"version\":\"x\"}", entryWithoutNulls.getPayload().serialized()); } @After @@ -362,7 +362,7 @@ private AuditEntry createAuditEntry(String key, String type, DateTime auditTime) .key(key) .type(type) .serializedPayload(StringUtils.format("Test payload for key[%s], type[%s]", key, type)) - .auditInfo(new AuditInfo("author", "comment", "127.0.0.1")) + .auditInfo(new AuditInfo("author", "identity", "comment", "127.0.0.1")) .auditTime(auditTime) .build(); } diff --git a/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java index f6ce89649b66..eceae4a08787 100644 --- a/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java @@ -193,7 +193,7 @@ public void testGetAllDatasourcesRuleHistoryWithWrongCount() private AuditInfo createAuditInfo() { - return new AuditInfo("testAuthor", "testComment", "127.0.0.1"); + return new AuditInfo("testAuthor", "testIdentity", "testComment", "127.0.0.1"); } private AuditEntry createAuditEntry(DateTime auditTime) From 2f2eebc3a4bdd27bccaa53b0c28a0dfc0b5d39b4 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Thu, 7 Dec 2023 21:28:27 +0530 Subject: [PATCH 13/29] Simplify usage of author and comment headers --- .../endpoint/BasicAuthenticatorResource.java | 20 +- ...dinatorBasicAuthenticatorResourceTest.java | 34 ++-- .../BasicAuthenticatorResourceTest.java | 12 +- .../overlord/http/OverlordResource.java | 13 +- .../overlord/http/OverlordResourceTest.java | 4 +- .../indexing/overlord/http/OverlordTest.java | 12 +- .../guice/SQLMetadataStorageDruidModule.java | 4 +- .../druid/server/audit/AuditLogger.java | 3 +- .../CoordinatorCompactionConfigsResource.java | 14 +- .../CoordinatorDynamicConfigsResource.java | 8 +- .../server/http/DataSourcesResource.java | 18 +- .../http/LookupCoordinatorResource.java | 20 +- .../druid/server/http/RulesResource.java | 7 +- .../server/security/AuthorizationUtils.java | 13 ++ ...rdinatorCompactionConfigsResourceTest.java | 16 -- .../server/http/DataSourcesResourceTest.java | 20 +- .../http/LookupCoordinatorResourceTest.java | 187 +++++------------- 17 files changed, 135 insertions(+), 270 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java index ca07c58d3be9..c9a36c1ca240 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java @@ -34,9 +34,7 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; -import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; -import javax.ws.rs.HeaderParam; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -157,9 +155,7 @@ public Response getUser( public Response createUser( @Context HttpServletRequest req, @PathParam("authenticatorName") final String authenticatorName, - @PathParam("userName") String userName, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment + @PathParam("userName") String userName ) { authValidator.validateAuthenticatorName(authenticatorName); @@ -167,7 +163,7 @@ public Response createUser( final Response response = handler.createUser(authenticatorName, userName); if (isSuccess(response)) { - final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(author, comment, req); + final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(req); performAudit(authenticatorName, "users.create", Collections.singletonMap("username", userName), auditInfo); } @@ -190,16 +186,14 @@ public Response createUser( public Response deleteUser( @Context HttpServletRequest req, @PathParam("authenticatorName") final String authenticatorName, - @PathParam("userName") String userName, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment + @PathParam("userName") String userName ) { authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.deleteUser(authenticatorName, userName); if (isSuccess(response)) { - final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(author, comment, req); + final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(req); performAudit(authenticatorName, "users.delete", Collections.singletonMap("username", userName), auditInfo); } @@ -223,16 +217,14 @@ public Response updateUserCredentials( @Context HttpServletRequest req, @PathParam("authenticatorName") final String authenticatorName, @PathParam("userName") String userName, - BasicAuthenticatorCredentialUpdate update, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment + BasicAuthenticatorCredentialUpdate update ) { authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.updateUserCredentials(authenticatorName, userName, update); if (isSuccess(response)) { - final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(author, comment, req); + final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(req); performAudit(authenticatorName, "users.update", Collections.singletonMap("username", userName), auditInfo); } diff --git a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java index defcf488cb61..848b0252afac 100644 --- a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java +++ b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java @@ -182,9 +182,9 @@ public void testGetAllUsers() Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableSet.of(BasicAuthUtils.ADMIN_NAME, BasicAuthUtils.INTERNAL_USER_NAME), response.getEntity()); - resource.createUser(req, AUTHENTICATOR_NAME, "druid", null, null); - resource.createUser(req, AUTHENTICATOR_NAME, "druid2", null, null); - resource.createUser(req, AUTHENTICATOR_NAME, "druid3", null, null); + resource.createUser(req, AUTHENTICATOR_NAME, "druid"); + resource.createUser(req, AUTHENTICATOR_NAME, "druid2"); + resource.createUser(req, AUTHENTICATOR_NAME, "druid3"); Set expectedUsers = ImmutableSet.of( BasicAuthUtils.ADMIN_NAME, @@ -222,13 +222,13 @@ public void testGetAllUsersSeparateDatabaseTables() Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableSet.of(BasicAuthUtils.ADMIN_NAME, BasicAuthUtils.INTERNAL_USER_NAME), response.getEntity()); - resource.createUser(req, AUTHENTICATOR_NAME, "druid", null, null); - resource.createUser(req, AUTHENTICATOR_NAME, "druid2", null, null); - resource.createUser(req, AUTHENTICATOR_NAME, "druid3", null, null); + resource.createUser(req, AUTHENTICATOR_NAME, "druid"); + resource.createUser(req, AUTHENTICATOR_NAME, "druid2"); + resource.createUser(req, AUTHENTICATOR_NAME, "druid3"); - resource.createUser(req, AUTHENTICATOR_NAME2, "druid4", null, null); - resource.createUser(req, AUTHENTICATOR_NAME2, "druid5", null, null); - resource.createUser(req, AUTHENTICATOR_NAME2, "druid6", null, null); + resource.createUser(req, AUTHENTICATOR_NAME2, "druid4"); + resource.createUser(req, AUTHENTICATOR_NAME2, "druid5"); + resource.createUser(req, AUTHENTICATOR_NAME2, "druid6"); Set expectedUsers = ImmutableSet.of( BasicAuthUtils.ADMIN_NAME, @@ -292,7 +292,7 @@ public void testGetAllUsersSeparateDatabaseTables() @Test public void testCreateDeleteUser() { - Response response = resource.createUser(req, AUTHENTICATOR_NAME, "druid", null, null); + Response response = resource.createUser(req, AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(200, response.getStatus()); response = resource.getUser(req, AUTHENTICATOR_NAME, "druid"); @@ -300,7 +300,7 @@ public void testCreateDeleteUser() BasicAuthenticatorUser expectedUser = new BasicAuthenticatorUser("druid", null); Assert.assertEquals(expectedUser, response.getEntity()); - response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid", null, null); + response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(200, response.getStatus()); response = resource.getCachedSerializedUserMap(req, AUTHENTICATOR_NAME); @@ -310,7 +310,7 @@ public void testCreateDeleteUser() Assert.assertNotNull(cachedUserMap); Assert.assertNull(cachedUserMap.get("druid")); - response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid", null, null); + response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(400, response.getStatus()); Assert.assertEquals(errorMapWithMsg("User [druid] does not exist."), response.getEntity()); @@ -322,15 +322,14 @@ public void testCreateDeleteUser() @Test public void testUserCredentials() { - Response response = resource.createUser(req, AUTHENTICATOR_NAME, "druid", null, null); + Response response = resource.createUser(req, AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(200, response.getStatus()); response = resource.updateUserCredentials( req, AUTHENTICATOR_NAME, "druid", - new BasicAuthenticatorCredentialUpdate("helloworld", null), - null, null + new BasicAuthenticatorCredentialUpdate("helloworld", null) ); Assert.assertEquals(200, response.getStatus()); @@ -377,7 +376,7 @@ public void testUserCredentials() ); Assert.assertArrayEquals(recalculatedHash, hash); - response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid", null, null); + response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(200, response.getStatus()); response = resource.getUser(req, AUTHENTICATOR_NAME, "druid"); @@ -388,8 +387,7 @@ public void testUserCredentials() req, AUTHENTICATOR_NAME, "druid", - new BasicAuthenticatorCredentialUpdate("helloworld", null), - null, null + new BasicAuthenticatorCredentialUpdate("helloworld", null) ); Assert.assertEquals(400, response.getStatus()); Assert.assertEquals(errorMapWithMsg("User [druid] does not exist."), response.getEntity()); diff --git a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResourceTest.java b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResourceTest.java index 69ac7ef7ca1f..6545b880a065 100644 --- a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResourceTest.java +++ b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResourceTest.java @@ -88,37 +88,37 @@ public void getCachedSerializedUserMapWithInvalidAuthenticatorNameShouldReturnEx @Test public void updateUserCredentialsShouldReturnExpectedResponse() { - Assert.assertNotNull(target.updateUserCredentials(req, AUTHENTICATOR_NAME, USER_NAME, update, null, null)); + Assert.assertNotNull(target.updateUserCredentials(req, AUTHENTICATOR_NAME, USER_NAME, update)); } @Test(expected = IllegalArgumentException.class) public void updateUserCredentialsWithInvalidAuthenticatorNameShouldReturnExpectedResponse() { - target.updateUserCredentials(req, INVALID_AUTHENTICATOR_NAME, USER_NAME, update, null, null); + target.updateUserCredentials(req, INVALID_AUTHENTICATOR_NAME, USER_NAME, update); } @Test public void deleteUserShouldReturnExpectedResponse() { - Assert.assertNotNull(target.deleteUser(req, AUTHENTICATOR_NAME, USER_NAME, null, null)); + Assert.assertNotNull(target.deleteUser(req, AUTHENTICATOR_NAME, USER_NAME)); } @Test(expected = IllegalArgumentException.class) public void deleteUserWithInvalidAuthenticatorNameShouldReturnExpectedResponse() { - target.deleteUser(req, INVALID_AUTHENTICATOR_NAME, USER_NAME, null, null); + target.deleteUser(req, INVALID_AUTHENTICATOR_NAME, USER_NAME); } @Test public void createUserShouldReturnExpectedResponse() { - Assert.assertNotNull(target.createUser(req, AUTHENTICATOR_NAME, USER_NAME, null, null)); + Assert.assertNotNull(target.createUser(req, AUTHENTICATOR_NAME, USER_NAME)); } @Test(expected = IllegalArgumentException.class) public void createUserWithInvalidAuthenticatorNameShouldReturnExpectedResponse() { - target.createUser(req, INVALID_AUTHENTICATOR_NAME, USER_NAME, null, null); + target.createUser(req, INVALID_AUTHENTICATOR_NAME, USER_NAME); } @Test diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index 215586f86baf..dcbf1e0ef0ca 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -94,7 +94,6 @@ import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; -import javax.ws.rs.HeaderParam; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -196,8 +195,6 @@ public OverlordResource( @Produces(MediaType.APPLICATION_JSON) public Response taskPost( final Task task, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context final HttpServletRequest req ) { @@ -228,13 +225,15 @@ public Response taskPost( taskQueue.add(task); // Do an audit only if this API was called by a user and not by internal services + final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(req); + final String author = auditInfo.getAuthor(); if (author != null && !author.isEmpty()) { auditManager.doAudit( AuditEntry.builder() .key(task.getDataSource()) - .type("overlord" + req.getServletPath()) + .type("ingest.batch") .payload(new TaskIdentifier(task.getId(), task.getGroupId(), task.getType())) - .auditInfo(AuthorizationUtils.buildAuditInfo(author, comment, req)) + .auditInfo(auditInfo) .build() ); } @@ -555,15 +554,13 @@ public Response getTotalWorkerCapacity() @ResourceFilters(ConfigResourceFilter.class) public Response setWorkerConfig( final WorkerBehaviorConfig workerBehaviorConfig, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context final HttpServletRequest req ) { final SetResult setResult = configManager.set( WorkerBehaviorConfig.CONFIG_KEY, workerBehaviorConfig, - new AuditInfo(author, comment, req.getRemoteAddr()) + AuthorizationUtils.buildAuditInfo(req) ); if (setResult.isOk()) { log.info("Updating Worker configs: %s", workerBehaviorConfig); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java index ae537b85102c..6eed9e32df38 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java @@ -877,7 +877,7 @@ public void testSecuredTaskPost() authConfig ); Task task = NoopTask.create(); - overlordResource.taskPost(task, "", "", req); + overlordResource.taskPost(task, req); } @Test @@ -900,7 +900,7 @@ public void testTaskPostDeniesDatasourceReadUser() Task task = NoopTask.forDatasource(Datasources.WIKIPEDIA); expectedException.expect(ForbiddenException.class); expectedException.expect(ForbiddenException.class); - overlordResource.taskPost(task, "", "", req); + overlordResource.taskPost(task, req); } @Test diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java index 1224a80965e6..04228ab68c2b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java @@ -31,6 +31,7 @@ import org.apache.curator.retry.RetryOneTime; import org.apache.curator.test.TestingServer; import org.apache.curator.test.Timing; +import org.apache.druid.audit.AuditManager; import org.apache.druid.curator.PotentiallyGzippedCompressionProvider; import org.apache.druid.curator.discovery.LatchableServiceAnnouncer; import org.apache.druid.discovery.DruidLeaderSelector; @@ -148,11 +149,14 @@ private void tearDownServerAndCurator() public void setUp() throws Exception { req = EasyMock.createMock(HttpServletRequest.class); - EasyMock.expect(req.getAttribute(AuthConfig.DRUID_ALLOW_UNSECURED_PATH)).andReturn(null).anyTimes(); - EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes(); + EasyMock.expect(req.getHeader(AuditManager.X_DRUID_AUTHOR)).andReturn("author").once(); + EasyMock.expect(req.getHeader(AuditManager.X_DRUID_COMMENT)).andReturn("comment").once(); + EasyMock.expect(req.getRemoteAddr()).andReturn("127.0.0.1").once(); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( new AuthenticationResult("druid", "druid", null, null) ).anyTimes(); + EasyMock.expect(req.getAttribute(AuthConfig.DRUID_ALLOW_UNSECURED_PATH)).andReturn(null).anyTimes(); + EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes(); req.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().anyTimes(); supervisorManager = EasyMock.createMock(SupervisorManager.class); @@ -282,12 +286,12 @@ public void testOverlordRun() throws Exception taskCompletionCountDownLatches.get(goodTaskId).countDown(); waitForTaskStatus(goodTaskId, TaskState.SUCCESS); - response = overlordResource.taskPost(task0, "", "", req); + response = overlordResource.taskPost(task0, req); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("task", taskId0), response.getEntity()); // Duplicate task - should fail - response = overlordResource.taskPost(task0, "", "", req); + response = overlordResource.taskPost(task0, req); Assert.assertEquals(400, response.getStatus()); // Task payload for task_0 should be present in taskStorage diff --git a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java index 1e396013f461..1e8044dcb53a 100644 --- a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java +++ b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java @@ -143,7 +143,7 @@ private void configureAuditDestination(Binder binder) { PolyBind.createChoice( binder, - "druid.audit.destination", + "druid.audit.manager.type", Key.get(AuditManager.class), Key.get(SQLAuditManager.class) ); @@ -160,7 +160,7 @@ private void configureAuditDestination(Binder binder) PolyBind.createChoice( binder, - "druid.audit.destination", + "druid.audit.manager.type", Key.get(AuditManagerConfig.class), Key.get(SQLAuditManagerConfig.class) ); diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java index ca162baf232a..d677ca685db6 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java @@ -30,7 +30,7 @@ public enum Level } private static final String MSG_FORMAT - = "[%s] User[%s], ip[%s] performed action[%s] on key[%s] with comment[%s]. Payload[%s]."; + = "[%s] User[%s], identity[%s], ip[%s] performed action[%s] on key[%s] with comment[%s]. Payload[%s]."; private final Level level; private final Logger logger = new Logger(AuditLogger.class); @@ -45,6 +45,7 @@ public void log(AuditEntry entry) Object[] args = { entry.getAuditTime(), entry.getAuditInfo().getAuthor(), + entry.getAuditInfo().getIdentity(), entry.getAuditInfo().getIp(), entry.getType(), entry.getKey(), diff --git a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java index e0e2faed69e2..3939b8669771 100644 --- a/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java +++ b/server/src/main/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResource.java @@ -41,9 +41,7 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; -import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; -import javax.ws.rs.HeaderParam; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -96,8 +94,6 @@ public Response setCompactionTaskLimit( @QueryParam("ratio") Double compactionTaskSlotRatio, @QueryParam("max") Integer maxCompactionTaskSlots, @QueryParam("useAutoScaleSlots") Boolean useAutoScaleSlots, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context HttpServletRequest req ) { @@ -108,15 +104,13 @@ public Response setCompactionTaskLimit( maxCompactionTaskSlots, useAutoScaleSlots ); - return updateConfigHelper(operator, AuthorizationUtils.buildAuditInfo(author, comment, req)); + return updateConfigHelper(operator, AuthorizationUtils.buildAuditInfo(req)); } @POST @Consumes(MediaType.APPLICATION_JSON) public Response addOrUpdateCompactionConfig( final DataSourceCompactionConfig newConfig, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context HttpServletRequest req ) { @@ -133,7 +127,7 @@ public Response addOrUpdateCompactionConfig( }; return updateConfigHelper( callable, - AuthorizationUtils.buildAuditInfo(author, comment, req) + AuthorizationUtils.buildAuditInfo(req) ); } @@ -202,8 +196,6 @@ public Response getCompactionConfigHistory( @Produces(MediaType.APPLICATION_JSON) public Response deleteCompactionConfig( @PathParam("dataSource") String dataSource, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context HttpServletRequest req ) { @@ -220,7 +212,7 @@ public Response deleteCompactionConfig( return CoordinatorCompactionConfig.from(current, ImmutableList.copyOf(configs.values())); }; - return updateConfigHelper(callable, AuthorizationUtils.buildAuditInfo(author, comment, req)); + return updateConfigHelper(callable, AuthorizationUtils.buildAuditInfo(req)); } private Response updateConfigHelper( diff --git a/server/src/main/java/org/apache/druid/server/http/CoordinatorDynamicConfigsResource.java b/server/src/main/java/org/apache/druid/server/http/CoordinatorDynamicConfigsResource.java index aab0a69632cb..a1ec2ea30e1b 100644 --- a/server/src/main/java/org/apache/druid/server/http/CoordinatorDynamicConfigsResource.java +++ b/server/src/main/java/org/apache/druid/server/http/CoordinatorDynamicConfigsResource.java @@ -20,7 +20,6 @@ package org.apache.druid.server.http; import com.sun.jersey.spi.container.ResourceFilters; -import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.common.config.ConfigManager.SetResult; import org.apache.druid.common.utils.ServletResourceUtils; @@ -28,14 +27,13 @@ import org.apache.druid.server.coordinator.CoordinatorConfigManager; import org.apache.druid.server.coordinator.CoordinatorDynamicConfig; import org.apache.druid.server.http.security.ConfigResourceFilter; +import org.apache.druid.server.security.AuthorizationUtils; import org.joda.time.Interval; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; -import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; -import javax.ws.rs.HeaderParam; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -75,8 +73,6 @@ public Response getDynamicConfigs() @Consumes(MediaType.APPLICATION_JSON) public Response setDynamicConfigs( final CoordinatorDynamicConfig.Builder dynamicConfigBuilder, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context HttpServletRequest req ) { @@ -85,7 +81,7 @@ public Response setDynamicConfigs( final SetResult setResult = manager.setDynamicConfig( dynamicConfigBuilder.build(current), - new AuditInfo(author, comment, req.getRemoteAddr()) + AuthorizationUtils.buildAuditInfo(req) ); if (setResult.isOk()) { diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 89de20a9f5cb..015de569e4ba 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -31,7 +31,6 @@ import it.unimi.dsi.fastutil.objects.Object2LongMap; import org.apache.commons.lang.StringUtils; import org.apache.druid.audit.AuditEntry; -import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DruidDataSource; @@ -59,6 +58,7 @@ import org.apache.druid.server.coordinator.rules.LoadRule; import org.apache.druid.server.coordinator.rules.Rule; import org.apache.druid.server.http.security.DatasourceResourceFilter; +import org.apache.druid.server.security.AuthorizationUtils; import org.apache.druid.server.security.AuthorizerMapper; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; @@ -73,9 +73,7 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; -import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; -import javax.ws.rs.HeaderParam; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -229,8 +227,6 @@ public Response markAsUsedNonOvershadowedSegments( public Response markSegmentsAsUnused( @PathParam("dataSourceName") final String dataSourceName, final MarkDataSourceSegmentsPayload payload, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context final HttpServletRequest req ) { @@ -257,13 +253,13 @@ public Response markSegmentsAsUnused( ); auditPayload = Collections.singletonMap("segmentIds", segmentIds); } - if (auditManager != null && author != null && !author.isEmpty()) { + if (auditManager != null) { auditManager.doAudit( AuditEntry.builder() .key(dataSourceName) .type("segments.markUnused") .payload(auditPayload) - .auditInfo(new AuditInfo(author, comment, req.getRemoteAddr())) + .auditInfo(AuthorizationUtils.buildAuditInfo(req)) .build() ); } @@ -348,7 +344,7 @@ public Response markAsUnusedAllSegmentsOrKillUnusedSegmentsInInterval( boolean killSegments = kill != null && Boolean.valueOf(kill); if (killSegments) { - return killUnusedSegmentsInInterval(dataSourceName, interval, null, null, req); + return killUnusedSegmentsInInterval(dataSourceName, interval, req); } else { MarkSegments markSegments = () -> segmentsMetadataManager.markAsUnusedAllSegmentsInDataSource(dataSourceName); return doMarkSegments("markAsUnusedAllSegments", dataSourceName, markSegments); @@ -362,8 +358,6 @@ public Response markAsUnusedAllSegmentsOrKillUnusedSegmentsInInterval( public Response killUnusedSegmentsInInterval( @PathParam("dataSourceName") final String dataSourceName, @PathParam("interval") final String interval, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context final HttpServletRequest req ) { @@ -379,13 +373,13 @@ public Response killUnusedSegmentsInInterval( overlordClient.runKillTask("api-issued", dataSourceName, theInterval, null), true ); - if (auditManager != null && author != null && !author.isEmpty()) { + if (auditManager != null) { auditManager.doAudit( AuditEntry.builder() .key(dataSourceName) .type("segments.killTask") .payload(ImmutableMap.of("killTaskId", killTaskId, "interval", theInterval)) - .auditInfo(new AuditInfo(author, comment, req.getRemoteAddr())) + .auditInfo(AuthorizationUtils.buildAuditInfo(req)) .build() ); } diff --git a/server/src/main/java/org/apache/druid/server/http/LookupCoordinatorResource.java b/server/src/main/java/org/apache/druid/server/http/LookupCoordinatorResource.java index 8bc21caed54a..9e435e72f952 100644 --- a/server/src/main/java/org/apache/druid/server/http/LookupCoordinatorResource.java +++ b/server/src/main/java/org/apache/druid/server/http/LookupCoordinatorResource.java @@ -29,8 +29,6 @@ import com.google.common.net.HostAndPort; import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; -import org.apache.druid.audit.AuditInfo; -import org.apache.druid.audit.AuditManager; import org.apache.druid.common.utils.ServletResourceUtils; import org.apache.druid.guice.annotations.Json; import org.apache.druid.guice.annotations.Smile; @@ -41,6 +39,7 @@ import org.apache.druid.server.http.security.ConfigResourceFilter; import org.apache.druid.server.lookup.cache.LookupCoordinatorManager; import org.apache.druid.server.lookup.cache.LookupExtractorFactoryMapContainer; +import org.apache.druid.server.security.AuthorizationUtils; import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; @@ -48,7 +47,6 @@ import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; -import javax.ws.rs.HeaderParam; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -149,8 +147,6 @@ public Response getAllLookupSpecs() @Consumes({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE}) public Response updateAllLookups( InputStream in, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context HttpServletRequest req ) { @@ -166,7 +162,7 @@ public Response updateAllLookups( catch (IOException e) { return Response.status(Response.Status.BAD_REQUEST).entity(ServletResourceUtils.sanitizeException(e)).build(); } - if (lookupCoordinatorManager.updateLookups(map, new AuditInfo(author, comment, req.getRemoteAddr()))) { + if (lookupCoordinatorManager.updateLookups(map, AuthorizationUtils.buildAuditInfo(req))) { return Response.status(Response.Status.ACCEPTED).entity(map).build(); } else { throw new RuntimeException("Unknown error updating configuration"); @@ -183,8 +179,6 @@ public Response updateAllLookups( @Path("/config/{tier}") public Response deleteTier( @PathParam("tier") String tier, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context HttpServletRequest req ) { @@ -195,7 +189,7 @@ public Response deleteTier( .build(); } - if (lookupCoordinatorManager.deleteTier(tier, new AuditInfo(author, comment, req.getRemoteAddr()))) { + if (lookupCoordinatorManager.deleteTier(tier, AuthorizationUtils.buildAuditInfo(req))) { return Response.status(Response.Status.ACCEPTED).build(); } else { return Response.status(Response.Status.NOT_FOUND).build(); @@ -213,8 +207,6 @@ public Response deleteTier( public Response deleteLookup( @PathParam("tier") String tier, @PathParam("lookup") String lookup, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context HttpServletRequest req ) { @@ -231,7 +223,7 @@ public Response deleteLookup( .build(); } - if (lookupCoordinatorManager.deleteLookup(tier, lookup, new AuditInfo(author, comment, req.getRemoteAddr()))) { + if (lookupCoordinatorManager.deleteLookup(tier, lookup, AuthorizationUtils.buildAuditInfo(req))) { return Response.status(Response.Status.ACCEPTED).build(); } else { return Response.status(Response.Status.NOT_FOUND).build(); @@ -249,8 +241,6 @@ public Response deleteLookup( public Response createOrUpdateLookup( @PathParam("tier") String tier, @PathParam("lookup") String lookup, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, InputStream in, @Context HttpServletRequest req ) @@ -280,7 +270,7 @@ public Response createOrUpdateLookup( tier, lookup, lookupSpec, - new AuditInfo(author, comment, req.getRemoteAddr()) + AuthorizationUtils.buildAuditInfo(req) )) { return Response.status(Response.Status.ACCEPTED).build(); } else { diff --git a/server/src/main/java/org/apache/druid/server/http/RulesResource.java b/server/src/main/java/org/apache/druid/server/http/RulesResource.java index beb223a6cc65..63769a3e76c3 100644 --- a/server/src/main/java/org/apache/druid/server/http/RulesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/RulesResource.java @@ -30,13 +30,12 @@ import org.apache.druid.server.coordinator.rules.Rule; import org.apache.druid.server.http.security.RulesResourceFilter; import org.apache.druid.server.http.security.StateResourceFilter; +import org.apache.druid.server.security.AuthorizationUtils; import org.joda.time.Interval; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; -import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; -import javax.ws.rs.HeaderParam; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -102,13 +101,11 @@ public Response getDatasourceRules( public Response setDatasourceRules( @PathParam("dataSourceName") final String dataSourceName, final List rules, - @HeaderParam(AuditManager.X_DRUID_AUTHOR) @DefaultValue("") final String author, - @HeaderParam(AuditManager.X_DRUID_COMMENT) @DefaultValue("") final String comment, @Context HttpServletRequest req ) { try { - final AuditInfo auditInfo = new AuditInfo(author, comment, req.getRemoteAddr()); + final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(req); if (databaseRuleManager.overrideRule(dataSourceName, rules, auditInfo)) { return Response.ok().build(); } else { diff --git a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java index bf54afc8afda..f913443514d2 100644 --- a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java +++ b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java @@ -23,6 +23,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import org.apache.druid.audit.AuditInfo; +import org.apache.druid.audit.AuditManager; import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.ISE; @@ -117,6 +118,18 @@ public static AuditInfo buildAuditInfo(String author, String comment, HttpServle ); } + public static AuditInfo buildAuditInfo(HttpServletRequest request) + { + final String author = request.getHeader(AuditManager.X_DRUID_AUTHOR); + final String comment = request.getHeader(AuditManager.X_DRUID_COMMENT); + return new AuditInfo( + author == null ? "" : author, + getAuthenticatedIdentity(request), + comment == null ? "" : comment, + request.getRemoteAddr() + ); + } + /** * Check a list of resource-actions to be performed by the identity represented by authenticationResult. * diff --git a/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java b/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java index 8d9b02401905..6d65e4acdbcc 100644 --- a/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java @@ -154,8 +154,6 @@ public void testSetCompactionTaskLimitWithExistingConfig() compactionTaskSlotRatio, maxCompactionTaskSlots, true, - author, - comment, mockHttpServletRequest ); Assert.assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); @@ -199,8 +197,6 @@ public void testAddOrUpdateCompactionConfigWithExistingConfig() String comment = "hello"; Response result = coordinatorCompactionConfigsResource.addOrUpdateCompactionConfig( newConfig, - author, - comment, mockHttpServletRequest ); Assert.assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); @@ -252,8 +248,6 @@ public void testDeleteCompactionConfigWithExistingConfig() String comment = "hello"; Response result = coordinatorCompactionConfigsResource.deleteCompactionConfig( datasourceName, - author, - comment, mockHttpServletRequest ); Assert.assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); @@ -277,8 +271,6 @@ public void testUpdateShouldRetryIfRetryableException() coordinatorCompactionConfigsResource.addOrUpdateCompactionConfig( NEW_CONFIG, - "author", - "test", mockHttpServletRequest ); @@ -308,8 +300,6 @@ public void testUpdateShouldNotRetryIfNotRetryableException() coordinatorCompactionConfigsResource.addOrUpdateCompactionConfig( NEW_CONFIG, - "author", - "test", mockHttpServletRequest ); @@ -357,8 +347,6 @@ public void testSetCompactionTaskLimitWithoutExistingConfig() compactionTaskSlotRatio, maxCompactionTaskSlots, true, - author, - comment, mockHttpServletRequest ); Assert.assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); @@ -414,8 +402,6 @@ public void testAddOrUpdateCompactionConfigWithoutExistingConfig() String comment = "hello"; Response result = coordinatorCompactionConfigsResource.addOrUpdateCompactionConfig( newConfig, - author, - comment, mockHttpServletRequest ); Assert.assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); @@ -445,8 +431,6 @@ public void testDeleteCompactionConfigWithoutExistingConfigShouldFailAsDatasourc String comment = "hello"; Response result = coordinatorCompactionConfigsResource.deleteCompactionConfig( DATASOURCE_NOT_EXISTS, - author, - comment, mockHttpServletRequest ); Assert.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), result.getStatus()); diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 2413f96577ad..6b71a5a033ba 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -601,7 +601,7 @@ public void testKillSegmentsInIntervalInDataSource() DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, overlordClient, null, null, null); - Response response = dataSourcesResource.killUnusedSegmentsInInterval("datasource1", interval, null, null, request); + Response response = dataSourcesResource.killUnusedSegmentsInInterval("datasource1", interval, request); Assert.assertEquals(200, response.getStatus()); Assert.assertNull(response.getEntity()); @@ -1046,7 +1046,7 @@ public void testMarkSegmentsAsUnused() DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 1), response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1077,7 +1077,7 @@ public void testMarkSegmentsAsUnusedNoChanges() DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 0), response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1110,7 +1110,7 @@ public void testMarkSegmentsAsUnusedException() DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(500, response.getStatus()); Assert.assertNotNull(response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1132,7 +1132,7 @@ public void testMarkAsUnusedSegmentsInInterval() DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 1), response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1155,7 +1155,7 @@ public void testMarkAsUnusedSegmentsInIntervalNoChanges() DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 0), response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1179,7 +1179,7 @@ public void testMarkAsUnusedSegmentsInIntervalException() DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(500, response.getStatus()); Assert.assertNotNull(response.getEntity()); EasyMock.verify(segmentsMetadataManager, inventoryView, server); @@ -1191,7 +1191,7 @@ public void testMarkSegmentsAsUnusedNullPayload() DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", null, "", "", request); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", null, request); Assert.assertEquals(400, response.getStatus()); Assert.assertNotNull(response.getEntity()); Assert.assertEquals( @@ -1209,7 +1209,7 @@ public void testMarkSegmentsAsUnusedInvalidPayload() final DataSourcesResource.MarkDataSourceSegmentsPayload payload = new DataSourcesResource.MarkDataSourceSegmentsPayload(null, null); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(400, response.getStatus()); Assert.assertNotNull(response.getEntity()); } @@ -1223,7 +1223,7 @@ public void testMarkSegmentsAsUnusedInvalidPayloadBothArguments() final DataSourcesResource.MarkDataSourceSegmentsPayload payload = new DataSourcesResource.MarkDataSourceSegmentsPayload(Intervals.of("2010-01-01/P1D"), ImmutableSet.of()); - Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, "", "", request); + Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(400, response.getStatus()); Assert.assertNotNull(response.getEntity()); } diff --git a/server/src/test/java/org/apache/druid/server/http/LookupCoordinatorResourceTest.java b/server/src/test/java/org/apache/druid/server/http/LookupCoordinatorResourceTest.java index 365eac6888ed..a56e135a9980 100644 --- a/server/src/test/java/org/apache/druid/server/http/LookupCoordinatorResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/LookupCoordinatorResourceTest.java @@ -26,11 +26,14 @@ import com.google.common.io.ByteSource; import com.google.common.net.HostAndPort; import org.apache.druid.audit.AuditInfo; +import org.apache.druid.audit.AuditManager; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.lookup.LookupsState; import org.apache.druid.server.lookup.cache.LookupCoordinatorManager; import org.apache.druid.server.lookup.cache.LookupExtractorFactoryMapContainer; +import org.apache.druid.server.security.AuthConfig; +import org.apache.druid.server.security.AuthenticationResult; import org.easymock.Capture; import org.easymock.EasyMock; import org.junit.Assert; @@ -43,6 +46,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -89,6 +93,10 @@ public InputStream openStream() throws IOException private static final Map> NODES_LOOKUP_STATE = ImmutableMap.of(LOOKUP_NODE, LOOKUP_STATE); + private static final AuthenticationResult AUTH_RESULT + = new AuthenticationResult("id", "authorizer", "authBy", Collections.emptyMap()); + private final AuditInfo auditInfo + = new AuditInfo("some author", "id", "some comment", "127.0.0.1"); @Test public void testSimpleGet() @@ -289,12 +297,7 @@ public void testExceptionalGetLookup() @Test public void testSimpleDeleteTier() { - final String author = "some author"; - final String comment = "some comment"; - final String ip = "127.0.0.1"; - - final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); - EasyMock.expect(request.getRemoteAddr()).andReturn(ip).once(); + final HttpServletRequest request = createMockRequest(false); final Capture auditInfoCapture = Capture.newInstance(); final LookupCoordinatorManager lookupCoordinatorManager = EasyMock.createStrictMock( @@ -313,17 +316,12 @@ public void testSimpleDeleteTier() ); final Response response = lookupCoordinatorResource.deleteTier( LOOKUP_TIER, - author, - comment, request ); Assert.assertEquals(202, response.getStatus()); Assert.assertTrue(auditInfoCapture.hasCaptured()); - final AuditInfo auditInfo = auditInfoCapture.getValue(); - Assert.assertEquals(author, auditInfo.getAuthor()); - Assert.assertEquals(comment, auditInfo.getComment()); - Assert.assertEquals(ip, auditInfo.getIp()); + Assert.assertEquals(auditInfo, auditInfoCapture.getValue()); EasyMock.verify(lookupCoordinatorManager, request); } @@ -331,12 +329,7 @@ public void testSimpleDeleteTier() @Test public void testSimpleDelete() { - final String author = "some author"; - final String comment = "some comment"; - final String ip = "127.0.0.1"; - - final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); - EasyMock.expect(request.getRemoteAddr()).andReturn(ip).once(); + final HttpServletRequest request = createMockRequest(false); final Capture auditInfoCapture = Capture.newInstance(); final LookupCoordinatorManager lookupCoordinatorManager = EasyMock.createStrictMock( @@ -357,17 +350,12 @@ public void testSimpleDelete() final Response response = lookupCoordinatorResource.deleteLookup( LOOKUP_TIER, LOOKUP_NAME, - author, - comment, request ); Assert.assertEquals(202, response.getStatus()); Assert.assertTrue(auditInfoCapture.hasCaptured()); - final AuditInfo auditInfo = auditInfoCapture.getValue(); - Assert.assertEquals(author, auditInfo.getAuthor()); - Assert.assertEquals(comment, auditInfo.getComment()); - Assert.assertEquals(ip, auditInfo.getIp()); + Assert.assertEquals(auditInfo, auditInfoCapture.getValue()); EasyMock.verify(lookupCoordinatorManager, request); } @@ -376,12 +364,7 @@ public void testSimpleDelete() @Test public void testMissingDelete() { - final String author = "some author"; - final String comment = "some comment"; - final String ip = "127.0.0.1"; - - final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); - EasyMock.expect(request.getRemoteAddr()).andReturn(ip).once(); + final HttpServletRequest request = createMockRequest(false); final Capture auditInfoCapture = Capture.newInstance(); final LookupCoordinatorManager lookupCoordinatorManager = EasyMock.createStrictMock( @@ -402,17 +385,12 @@ public void testMissingDelete() final Response response = lookupCoordinatorResource.deleteLookup( LOOKUP_TIER, LOOKUP_NAME, - author, - comment, request ); Assert.assertEquals(404, response.getStatus()); Assert.assertTrue(auditInfoCapture.hasCaptured()); - final AuditInfo auditInfo = auditInfoCapture.getValue(); - Assert.assertEquals(author, auditInfo.getAuthor()); - Assert.assertEquals(comment, auditInfo.getComment()); - Assert.assertEquals(ip, auditInfo.getIp()); + Assert.assertEquals(auditInfo, auditInfoCapture.getValue()); EasyMock.verify(lookupCoordinatorManager, request); } @@ -421,13 +399,9 @@ public void testMissingDelete() @Test public void testExceptionalDelete() { - final String author = "some author"; - final String comment = "some comment"; - final String ip = "127.0.0.1"; final String errMsg = "some error"; - final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); - EasyMock.expect(request.getRemoteAddr()).andReturn(ip).once(); + final HttpServletRequest request = createMockRequest(false); final Capture auditInfoCapture = Capture.newInstance(); final LookupCoordinatorManager lookupCoordinatorManager = EasyMock.createStrictMock( @@ -448,18 +422,13 @@ public void testExceptionalDelete() final Response response = lookupCoordinatorResource.deleteLookup( LOOKUP_TIER, LOOKUP_NAME, - author, - comment, request ); Assert.assertEquals(500, response.getStatus()); Assert.assertEquals(ImmutableMap.of("error", errMsg), response.getEntity()); Assert.assertTrue(auditInfoCapture.hasCaptured()); - final AuditInfo auditInfo = auditInfoCapture.getValue(); - Assert.assertEquals(author, auditInfo.getAuthor()); - Assert.assertEquals(comment, auditInfo.getComment()); - Assert.assertEquals(ip, auditInfo.getIp()); + Assert.assertEquals(auditInfo, auditInfoCapture.getValue()); EasyMock.verify(lookupCoordinatorManager, request); } @@ -475,24 +444,18 @@ public void testInvalidDelete() MAPPER, MAPPER ); - Assert.assertEquals(400, lookupCoordinatorResource.deleteLookup("foo", null, null, null, null).getStatus()); - Assert.assertEquals(400, lookupCoordinatorResource.deleteLookup(null, null, null, null, null).getStatus()); - Assert.assertEquals(400, lookupCoordinatorResource.deleteLookup(null, "foo", null, null, null).getStatus()); - Assert.assertEquals(400, lookupCoordinatorResource.deleteLookup("foo", "", null, null, null).getStatus()); - Assert.assertEquals(400, lookupCoordinatorResource.deleteLookup("", "foo", null, null, null).getStatus()); + Assert.assertEquals(400, lookupCoordinatorResource.deleteLookup("foo", null, null).getStatus()); + Assert.assertEquals(400, lookupCoordinatorResource.deleteLookup(null, null, null).getStatus()); + Assert.assertEquals(400, lookupCoordinatorResource.deleteLookup(null, "foo", null).getStatus()); + Assert.assertEquals(400, lookupCoordinatorResource.deleteLookup("foo", "", null).getStatus()); + Assert.assertEquals(400, lookupCoordinatorResource.deleteLookup("", "foo", null).getStatus()); EasyMock.verify(lookupCoordinatorManager); } @Test public void testSimpleNew() throws Exception { - final String author = "some author"; - final String comment = "some comment"; - final String ip = "127.0.0.1"; - - final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); - EasyMock.expect(request.getContentType()).andReturn(MediaType.APPLICATION_JSON).once(); - EasyMock.expect(request.getRemoteAddr()).andReturn(ip).once(); + final HttpServletRequest request = createMockRequest(true); final Capture auditInfoCapture = Capture.newInstance(); final LookupCoordinatorManager lookupCoordinatorManager = EasyMock.createStrictMock( LookupCoordinatorManager.class); @@ -509,17 +472,12 @@ public void testSimpleNew() throws Exception ); final Response response = lookupCoordinatorResource.updateAllLookups( SINGLE_TIER_MAP_SOURCE.openStream(), - author, - comment, request ); Assert.assertEquals(202, response.getStatus()); Assert.assertTrue(auditInfoCapture.hasCaptured()); - final AuditInfo auditInfo = auditInfoCapture.getValue(); - Assert.assertEquals(author, auditInfo.getAuthor()); - Assert.assertEquals(comment, auditInfo.getComment()); - Assert.assertEquals(ip, auditInfo.getIp()); + Assert.assertEquals(auditInfo, auditInfoCapture.getValue()); EasyMock.verify(lookupCoordinatorManager, request); } @@ -527,14 +485,9 @@ public void testSimpleNew() throws Exception @Test public void testExceptionalNew() throws Exception { - final String author = "some author"; - final String comment = "some comment"; - final String ip = "127.0.0.1"; final String errMsg = "some error"; - final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); - EasyMock.expect(request.getContentType()).andReturn(MediaType.APPLICATION_JSON).once(); - EasyMock.expect(request.getRemoteAddr()).andReturn(ip).once(); + final HttpServletRequest request = createMockRequest(true); final Capture auditInfoCapture = Capture.newInstance(); final LookupCoordinatorManager lookupCoordinatorManager = EasyMock.createStrictMock( @@ -553,18 +506,13 @@ public void testExceptionalNew() throws Exception ); final Response response = lookupCoordinatorResource.updateAllLookups( SINGLE_TIER_MAP_SOURCE.openStream(), - author, - comment, request ); Assert.assertEquals(500, response.getStatus()); Assert.assertEquals(ImmutableMap.of("error", errMsg), response.getEntity()); Assert.assertTrue(auditInfoCapture.hasCaptured()); - final AuditInfo auditInfo = auditInfoCapture.getValue(); - Assert.assertEquals(author, auditInfo.getAuthor()); - Assert.assertEquals(comment, auditInfo.getComment()); - Assert.assertEquals(ip, auditInfo.getIp()); + Assert.assertEquals(auditInfo, auditInfoCapture.getValue()); EasyMock.verify(lookupCoordinatorManager, request); } @@ -572,13 +520,7 @@ public void testExceptionalNew() throws Exception @Test public void testFailedNew() throws Exception { - final String author = "some author"; - final String comment = "some comment"; - final String ip = "127.0.0.1"; - - final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); - EasyMock.expect(request.getContentType()).andReturn(MediaType.APPLICATION_JSON).once(); - EasyMock.expect(request.getRemoteAddr()).andReturn(ip).once(); + final HttpServletRequest request = createMockRequest(true); final Capture auditInfoCapture = Capture.newInstance(); final LookupCoordinatorManager lookupCoordinatorManager = EasyMock.createStrictMock( @@ -597,18 +539,13 @@ public void testFailedNew() throws Exception ); final Response response = lookupCoordinatorResource.updateAllLookups( SINGLE_TIER_MAP_SOURCE.openStream(), - author, - comment, request ); Assert.assertEquals(500, response.getStatus()); Assert.assertEquals(ImmutableMap.of("error", "Unknown error updating configuration"), response.getEntity()); Assert.assertTrue(auditInfoCapture.hasCaptured()); - final AuditInfo auditInfo = auditInfoCapture.getValue(); - Assert.assertEquals(author, auditInfo.getAuthor()); - Assert.assertEquals(comment, auditInfo.getComment()); - Assert.assertEquals(ip, auditInfo.getIp()); + Assert.assertEquals(auditInfo, auditInfoCapture.getValue()); EasyMock.verify(lookupCoordinatorManager, request); } @@ -616,13 +553,7 @@ public void testFailedNew() throws Exception @Test public void testSimpleNewLookup() throws Exception { - final String author = "some author"; - final String comment = "some comment"; - final String ip = "127.0.0.1"; - - final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); - EasyMock.expect(request.getContentType()).andReturn(MediaType.APPLICATION_JSON).once(); - EasyMock.expect(request.getRemoteAddr()).andReturn(ip).once(); + final HttpServletRequest request = createMockRequest(true); final Capture auditInfoCapture = Capture.newInstance(); final LookupCoordinatorManager lookupCoordinatorManager = EasyMock.createStrictMock( @@ -644,18 +575,13 @@ public void testSimpleNewLookup() throws Exception final Response response = lookupCoordinatorResource.createOrUpdateLookup( LOOKUP_TIER, LOOKUP_NAME, - author, - comment, EMPTY_MAP_SOURCE.openStream(), request ); Assert.assertEquals(202, response.getStatus()); Assert.assertTrue(auditInfoCapture.hasCaptured()); - final AuditInfo auditInfo = auditInfoCapture.getValue(); - Assert.assertEquals(author, auditInfo.getAuthor()); - Assert.assertEquals(comment, auditInfo.getComment()); - Assert.assertEquals(ip, auditInfo.getIp()); + Assert.assertEquals(auditInfo, auditInfoCapture.getValue()); EasyMock.verify(lookupCoordinatorManager, request); } @@ -664,13 +590,7 @@ public void testSimpleNewLookup() throws Exception @Test public void testDBErrNewLookup() throws Exception { - final String author = "some author"; - final String comment = "some comment"; - final String ip = "127.0.0.1"; - - final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); - EasyMock.expect(request.getContentType()).andReturn(MediaType.APPLICATION_JSON).once(); - EasyMock.expect(request.getRemoteAddr()).andReturn(ip).once(); + final HttpServletRequest request = createMockRequest(true); final Capture auditInfoCapture = Capture.newInstance(); final LookupCoordinatorManager lookupCoordinatorManager = EasyMock.createStrictMock( @@ -692,8 +612,6 @@ public void testDBErrNewLookup() throws Exception final Response response = lookupCoordinatorResource.createOrUpdateLookup( LOOKUP_TIER, LOOKUP_NAME, - author, - comment, EMPTY_MAP_SOURCE.openStream(), request ); @@ -701,10 +619,7 @@ public void testDBErrNewLookup() throws Exception Assert.assertEquals(500, response.getStatus()); Assert.assertEquals(ImmutableMap.of("error", "Unknown error updating configuration"), response.getEntity()); Assert.assertTrue(auditInfoCapture.hasCaptured()); - final AuditInfo auditInfo = auditInfoCapture.getValue(); - Assert.assertEquals(author, auditInfo.getAuthor()); - Assert.assertEquals(comment, auditInfo.getComment()); - Assert.assertEquals(ip, auditInfo.getIp()); + Assert.assertEquals(auditInfo, auditInfoCapture.getValue()); EasyMock.verify(lookupCoordinatorManager, request); } @@ -713,13 +628,8 @@ public void testDBErrNewLookup() throws Exception public void testExceptionalNewLookup() throws Exception { final String errMsg = "error message"; - final String author = "some author"; - final String comment = "some comment"; - final String ip = "127.0.0.1"; - final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); - EasyMock.expect(request.getContentType()).andReturn(MediaType.APPLICATION_JSON).once(); - EasyMock.expect(request.getRemoteAddr()).andReturn(ip).once(); + final HttpServletRequest request = createMockRequest(true); final Capture auditInfoCapture = Capture.newInstance(); final LookupCoordinatorManager lookupCoordinatorManager = EasyMock.createStrictMock( @@ -741,8 +651,6 @@ public void testExceptionalNewLookup() throws Exception final Response response = lookupCoordinatorResource.createOrUpdateLookup( LOOKUP_TIER, LOOKUP_NAME, - author, - comment, EMPTY_MAP_SOURCE.openStream(), request ); @@ -750,10 +658,7 @@ public void testExceptionalNewLookup() throws Exception Assert.assertEquals(500, response.getStatus()); Assert.assertEquals(ImmutableMap.of("error", errMsg), response.getEntity()); Assert.assertTrue(auditInfoCapture.hasCaptured()); - final AuditInfo auditInfo = auditInfoCapture.getValue(); - Assert.assertEquals(author, auditInfo.getAuthor()); - Assert.assertEquals(comment, auditInfo.getComment()); - Assert.assertEquals(ip, auditInfo.getIp()); + Assert.assertEquals(auditInfo, auditInfoCapture.getValue()); EasyMock.verify(lookupCoordinatorManager, request); } @@ -762,10 +667,6 @@ public void testExceptionalNewLookup() throws Exception @Test public void testNullValsNewLookup() throws Exception { - final String author = "some author"; - final String comment = "some comment"; - final String ip = "127.0.0.1"; - final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); final LookupCoordinatorManager lookupCoordinatorManager = EasyMock.createStrictMock( @@ -781,8 +682,6 @@ public void testNullValsNewLookup() throws Exception Assert.assertEquals(400, lookupCoordinatorResource.createOrUpdateLookup( null, LOOKUP_NAME, - author, - comment, EMPTY_MAP_SOURCE.openStream(), request ).getStatus()); @@ -790,8 +689,6 @@ public void testNullValsNewLookup() throws Exception Assert.assertEquals(400, lookupCoordinatorResource.createOrUpdateLookup( LOOKUP_TIER, null, - author, - comment, EMPTY_MAP_SOURCE.openStream(), request ).getStatus()); @@ -799,8 +696,6 @@ public void testNullValsNewLookup() throws Exception Assert.assertEquals(400, lookupCoordinatorResource.createOrUpdateLookup( LOOKUP_TIER, "", - author, - comment, EMPTY_MAP_SOURCE.openStream(), request ).getStatus()); @@ -808,8 +703,6 @@ public void testNullValsNewLookup() throws Exception Assert.assertEquals(400, lookupCoordinatorResource.createOrUpdateLookup( "", LOOKUP_NAME, - author, - comment, EMPTY_MAP_SOURCE.openStream(), request ).getStatus()); @@ -1225,4 +1118,18 @@ public void testGetEmptyAllLookupSpecs() Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus()); EasyMock.verify(lookupCoordinatorManager); } + + private HttpServletRequest createMockRequest(boolean hasJsonPayload) + { + final HttpServletRequest request = EasyMock.createStrictMock(HttpServletRequest.class); + if (hasJsonPayload) { + EasyMock.expect(request.getContentType()).andReturn(MediaType.APPLICATION_JSON).once(); + } + EasyMock.expect(request.getHeader(AuditManager.X_DRUID_AUTHOR)).andReturn("some author").once(); + EasyMock.expect(request.getHeader(AuditManager.X_DRUID_COMMENT)).andReturn("some comment").once(); + EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(AUTH_RESULT).once(); + EasyMock.expect(request.getRemoteAddr()).andReturn("127.0.0.1").once(); + + return request; + } } From 956e6a5590969ac8556b3e37f97a2a59a38e2c75 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Fri, 8 Dec 2023 09:20:38 +0530 Subject: [PATCH 14/29] Fix tests --- ...dinatorBasicAuthenticatorResourceTest.java | 103 ++++++++++++------ .../server/security/AuthorizationUtils.java | 20 ++-- 2 files changed, 81 insertions(+), 42 deletions(-) diff --git a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java index 848b0252afac..9dccab0f2c91 100644 --- a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java +++ b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java @@ -36,7 +36,9 @@ import org.apache.druid.security.basic.authentication.entity.BasicAuthenticatorCredentialUpdate; import org.apache.druid.security.basic.authentication.entity.BasicAuthenticatorCredentials; import org.apache.druid.security.basic.authentication.entity.BasicAuthenticatorUser; +import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthValidator; +import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.server.security.AuthenticatorMapper; import org.easymock.EasyMock; import org.junit.After; @@ -51,6 +53,7 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.Response; +import java.util.Collections; import java.util.Map; import java.util.Set; @@ -81,6 +84,11 @@ public class CoordinatorBasicAuthenticatorResourceTest public void setUp() { req = EasyMock.createStrictMock(HttpServletRequest.class); + EasyMock.expect(req.getHeader(AuditManager.X_DRUID_AUTHOR)).andReturn("author").anyTimes(); + EasyMock.expect(req.getHeader(AuditManager.X_DRUID_COMMENT)).andReturn("comment").anyTimes(); + EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( + new AuthenticationResult("id", "authorizer", "authBy", Collections.emptyMap()) + ).anyTimes(); EasyMock.expect(req.getRemoteAddr()).andReturn("127.0.0.1").anyTimes(); EasyMock.replay(req); @@ -162,12 +170,15 @@ public void setUp() public void tearDown() { storageUpdater.stop(); + if (req != null) { + EasyMock.verify(req); + } } @Test public void testInvalidAuthenticator() { - Response response = resource.getAllUsers(req, "invalidName"); + Response response = resource.getAllUsers(mockHttpRequestNoAudit(), "invalidName"); Assert.assertEquals(400, response.getStatus()); Assert.assertEquals( errorMapWithMsg("Basic authenticator with name [invalidName] does not exist."), @@ -178,13 +189,13 @@ public void testInvalidAuthenticator() @Test public void testGetAllUsers() { - Response response = resource.getAllUsers(req, AUTHENTICATOR_NAME); + Response response = resource.getAllUsers(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableSet.of(BasicAuthUtils.ADMIN_NAME, BasicAuthUtils.INTERNAL_USER_NAME), response.getEntity()); - resource.createUser(req, AUTHENTICATOR_NAME, "druid"); - resource.createUser(req, AUTHENTICATOR_NAME, "druid2"); - resource.createUser(req, AUTHENTICATOR_NAME, "druid3"); + resource.createUser(mockHttpRequest(), AUTHENTICATOR_NAME, "druid"); + resource.createUser(mockHttpRequest(), AUTHENTICATOR_NAME, "druid2"); + resource.createUser(mockHttpRequest(), AUTHENTICATOR_NAME, "druid3"); Set expectedUsers = ImmutableSet.of( BasicAuthUtils.ADMIN_NAME, @@ -194,12 +205,12 @@ public void testGetAllUsers() "druid3" ); - response = resource.getAllUsers(req, AUTHENTICATOR_NAME); + response = resource.getAllUsers(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(expectedUsers, response.getEntity()); // Verify cached user map is also getting updated - response = resource.getCachedSerializedUserMap(req, AUTHENTICATOR_NAME); + response = resource.getCachedSerializedUserMap(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME); Assert.assertEquals(200, response.getStatus()); Assert.assertTrue(response.getEntity() instanceof byte[]); Map cachedUserMap = BasicAuthUtils.deserializeAuthenticatorUserMap(objectMapper, (byte[]) response.getEntity()); @@ -218,17 +229,17 @@ public void testGetAllUsers() @Test public void testGetAllUsersSeparateDatabaseTables() { - Response response = resource.getAllUsers(req, AUTHENTICATOR_NAME); + Response response = resource.getAllUsers(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableSet.of(BasicAuthUtils.ADMIN_NAME, BasicAuthUtils.INTERNAL_USER_NAME), response.getEntity()); - resource.createUser(req, AUTHENTICATOR_NAME, "druid"); - resource.createUser(req, AUTHENTICATOR_NAME, "druid2"); - resource.createUser(req, AUTHENTICATOR_NAME, "druid3"); + resource.createUser(mockHttpRequest(), AUTHENTICATOR_NAME, "druid"); + resource.createUser(mockHttpRequest(), AUTHENTICATOR_NAME, "druid2"); + resource.createUser(mockHttpRequest(), AUTHENTICATOR_NAME, "druid3"); - resource.createUser(req, AUTHENTICATOR_NAME2, "druid4"); - resource.createUser(req, AUTHENTICATOR_NAME2, "druid5"); - resource.createUser(req, AUTHENTICATOR_NAME2, "druid6"); + resource.createUser(mockHttpRequest(), AUTHENTICATOR_NAME2, "druid4"); + resource.createUser(mockHttpRequest(), AUTHENTICATOR_NAME2, "druid5"); + resource.createUser(mockHttpRequest(), AUTHENTICATOR_NAME2, "druid6"); Set expectedUsers = ImmutableSet.of( BasicAuthUtils.ADMIN_NAME, @@ -246,12 +257,12 @@ public void testGetAllUsersSeparateDatabaseTables() "druid6" ); - response = resource.getAllUsers(req, AUTHENTICATOR_NAME); + response = resource.getAllUsers(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(expectedUsers, response.getEntity()); // Verify cached user map for AUTHENTICATOR_NAME authenticator is also getting updated - response = resource.getCachedSerializedUserMap(req, AUTHENTICATOR_NAME); + response = resource.getCachedSerializedUserMap(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME); Assert.assertEquals(200, response.getStatus()); Assert.assertTrue(response.getEntity() instanceof byte[]); @@ -267,12 +278,12 @@ public void testGetAllUsersSeparateDatabaseTables() Assert.assertNotNull(cachedUserMap.get("druid3")); Assert.assertEquals(cachedUserMap.get("druid3").getName(), "druid3"); - response = resource.getAllUsers(req, AUTHENTICATOR_NAME2); + response = resource.getAllUsers(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME2); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(expectedUsers2, response.getEntity()); // Verify cached user map for each AUTHENTICATOR_NAME2 is also getting updated - response = resource.getCachedSerializedUserMap(req, AUTHENTICATOR_NAME2); + response = resource.getCachedSerializedUserMap(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME2); Assert.assertEquals(200, response.getStatus()); Assert.assertTrue(response.getEntity() instanceof byte[]); @@ -292,29 +303,29 @@ public void testGetAllUsersSeparateDatabaseTables() @Test public void testCreateDeleteUser() { - Response response = resource.createUser(req, AUTHENTICATOR_NAME, "druid"); + Response response = resource.createUser(mockHttpRequest(), AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(200, response.getStatus()); - response = resource.getUser(req, AUTHENTICATOR_NAME, "druid"); + response = resource.getUser(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(200, response.getStatus()); BasicAuthenticatorUser expectedUser = new BasicAuthenticatorUser("druid", null); Assert.assertEquals(expectedUser, response.getEntity()); - response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid"); + response = resource.deleteUser(mockHttpRequest(), AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(200, response.getStatus()); - response = resource.getCachedSerializedUserMap(req, AUTHENTICATOR_NAME); + response = resource.getCachedSerializedUserMap(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME); Assert.assertEquals(200, response.getStatus()); Assert.assertTrue(response.getEntity() instanceof byte[]); Map cachedUserMap = BasicAuthUtils.deserializeAuthenticatorUserMap(objectMapper, (byte[]) response.getEntity()); Assert.assertNotNull(cachedUserMap); Assert.assertNull(cachedUserMap.get("druid")); - response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid"); + response = resource.deleteUser(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(400, response.getStatus()); Assert.assertEquals(errorMapWithMsg("User [druid] does not exist."), response.getEntity()); - response = resource.getUser(req, AUTHENTICATOR_NAME, "druid"); + response = resource.getUser(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(400, response.getStatus()); Assert.assertEquals(errorMapWithMsg("User [druid] does not exist."), response.getEntity()); } @@ -322,18 +333,18 @@ public void testCreateDeleteUser() @Test public void testUserCredentials() { - Response response = resource.createUser(req, AUTHENTICATOR_NAME, "druid"); + Response response = resource.createUser(mockHttpRequest(), AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(200, response.getStatus()); response = resource.updateUserCredentials( - req, + mockHttpRequest(), AUTHENTICATOR_NAME, "druid", new BasicAuthenticatorCredentialUpdate("helloworld", null) ); Assert.assertEquals(200, response.getStatus()); - response = resource.getUser(req, AUTHENTICATOR_NAME, "druid"); + response = resource.getUser(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(200, response.getStatus()); BasicAuthenticatorUser actualUser = (BasicAuthenticatorUser) response.getEntity(); Assert.assertEquals("druid", actualUser.getName()); @@ -353,7 +364,7 @@ public void testUserCredentials() ); Assert.assertArrayEquals(recalculatedHash, hash); - response = resource.getCachedSerializedUserMap(req, AUTHENTICATOR_NAME); + response = resource.getCachedSerializedUserMap(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME); Assert.assertEquals(200, response.getStatus()); Assert.assertTrue(response.getEntity() instanceof byte[]); Map cachedUserMap = BasicAuthUtils.deserializeAuthenticatorUserMap(objectMapper, (byte[]) response.getEntity()); @@ -376,21 +387,49 @@ public void testUserCredentials() ); Assert.assertArrayEquals(recalculatedHash, hash); - response = resource.deleteUser(req, AUTHENTICATOR_NAME, "druid"); + response = resource.deleteUser(mockHttpRequest(), AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(200, response.getStatus()); - - response = resource.getUser(req, AUTHENTICATOR_NAME, "druid"); +/* + response = resource.getUser(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(400, response.getStatus()); Assert.assertEquals(errorMapWithMsg("User [druid] does not exist."), response.getEntity()); response = resource.updateUserCredentials( - req, + mockHttpRequest(), AUTHENTICATOR_NAME, "druid", new BasicAuthenticatorCredentialUpdate("helloworld", null) ); Assert.assertEquals(400, response.getStatus()); Assert.assertEquals(errorMapWithMsg("User [druid] does not exist."), response.getEntity()); + */ + } + + private HttpServletRequest mockHttpRequestNoAudit() + { + if (req != null) { + EasyMock.verify(req); + } + req = EasyMock.createStrictMock(HttpServletRequest.class); + EasyMock.replay(req); + return req; + } + + private HttpServletRequest mockHttpRequest() + { + if (req != null) { + EasyMock.verify(req); + } + req = EasyMock.createStrictMock(HttpServletRequest.class); + EasyMock.expect(req.getHeader(AuditManager.X_DRUID_AUTHOR)).andReturn("author").once(); + EasyMock.expect(req.getHeader(AuditManager.X_DRUID_COMMENT)).andReturn("comment").once(); + EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( + new AuthenticationResult("id", "authorizer", "authBy", Collections.emptyMap()) + ).once(); + EasyMock.expect(req.getRemoteAddr()).andReturn("127.0.0.1").once(); + EasyMock.replay(req); + + return req; } private static Map errorMapWithMsg(String errorMsg) diff --git a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java index f913443514d2..03251fac01c4 100644 --- a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java +++ b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java @@ -108,16 +108,16 @@ public static String getAuthenticatedIdentity(HttpServletRequest request) } } - public static AuditInfo buildAuditInfo(String author, String comment, HttpServletRequest request) - { - return new AuditInfo( - author, - getAuthenticatedIdentity(request), - comment, - request.getRemoteAddr() - ); - } - + /** + * Builds an AuditInfo for the given request by extracting the following from + * it: + *
    + *
  • Header {@link AuditManager#X_DRUID_AUTHOR}
    • + *
  • Header {@link AuditManager#X_DRUID_COMMENT}
    • + *
  • Attribute {@link AuthConfig#DRUID_AUTHENTICATION_RESULT}
    • + *
  • IP address using {@link HttpServletRequest#getRemoteAddr()}
    • + *
+ */ public static AuditInfo buildAuditInfo(HttpServletRequest request) { final String author = request.getHeader(AuditManager.X_DRUID_AUTHOR); From 614409163847c5081a235fdf34159d9f8da9a7d7 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Fri, 8 Dec 2023 17:38:41 +0530 Subject: [PATCH 15/29] Add RequestInfo to AuditEntry --- .../org/apache/druid/audit/AuditEntry.java | 22 +++- .../org/apache/druid/audit/AuditInfo.java | 3 + .../org/apache/druid/audit/RequestInfo.java | 107 +++++++++++++++++ .../org/apache/druid/audit/AuditInfoTest.java | 1 + .../guice/SQLMetadataStorageDruidModule.java | 29 +---- .../druid/server/audit/AuditLogger.java | 3 +- .../server}/audit/AuditManagerConfig.java | 11 +- .../druid/server/audit/AuditSerdeHelper.java | 2 +- .../server/audit/LoggingAuditManager.java | 10 +- .../audit/LoggingAuditManagerConfig.java | 1 - .../druid/server/audit/SQLAuditManager.java | 9 +- .../server/audit/SQLAuditManagerConfig.java | 1 - .../server/audit/AuditManagerConfigTest.java | 110 ++++++++++++++++++ .../server/audit/SQLAuditManagerTest.java | 9 +- .../druid/server/http/RulesResourceTest.java | 19 ++- 15 files changed, 289 insertions(+), 48 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/audit/RequestInfo.java rename {processing/src/main/java/org/apache/druid => server/src/main/java/org/apache/druid/server}/audit/AuditManagerConfig.java (66%) create mode 100644 server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java diff --git a/processing/src/main/java/org/apache/druid/audit/AuditEntry.java b/processing/src/main/java/org/apache/druid/audit/AuditEntry.java index a8a420915654..696143b07b16 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditEntry.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditEntry.java @@ -26,6 +26,7 @@ import org.apache.druid.java.util.common.DateTimes; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.util.Objects; /** @@ -37,6 +38,7 @@ public class AuditEntry private final String key; private final String type; private final AuditInfo auditInfo; + private final RequestInfo request; private final Payload payload; private final DateTime auditTime; @@ -45,6 +47,7 @@ public AuditEntry( @JsonProperty("key") String key, @JsonProperty("type") String type, @JsonProperty("auditInfo") AuditInfo authorInfo, + @JsonProperty("request") @Nullable RequestInfo requestInfo, @JsonProperty("payload") Payload payload, @JsonProperty("auditTime") DateTime auditTime ) @@ -55,6 +58,7 @@ public AuditEntry( this.key = key; this.type = type; this.auditInfo = authorInfo; + this.request = requestInfo; this.auditTime = auditTime == null ? DateTimes.nowUtc() : auditTime; this.payload = payload == null ? Payload.fromString("") : payload; } @@ -77,6 +81,15 @@ public AuditInfo getAuditInfo() return auditInfo; } + /** + * Details of the REST API request associated with this audit, if any. + */ + @JsonProperty + public RequestInfo getRequest() + { + return request; + } + /** * Non-null payload of the audit event. */ @@ -129,6 +142,7 @@ public static class Builder private String key; private String type; private AuditInfo auditInfo; + private RequestInfo requestInfo; private Object payload; private String serializedPayload; @@ -157,6 +171,12 @@ public Builder auditInfo(AuditInfo auditInfo) return this; } + public Builder request(RequestInfo requestInfo) + { + this.requestInfo = requestInfo; + return this; + } + public Builder serializedPayload(String serializedPayload) { this.serializedPayload = serializedPayload; @@ -177,7 +197,7 @@ public Builder auditTime(DateTime auditTime) public AuditEntry build() { - return new AuditEntry(key, type, auditInfo, new Payload(serializedPayload, payload), auditTime); + return new AuditEntry(key, type, auditInfo, requestInfo, new Payload(serializedPayload, payload), auditTime); } } diff --git a/processing/src/main/java/org/apache/druid/audit/AuditInfo.java b/processing/src/main/java/org/apache/druid/audit/AuditInfo.java index 206acbd1353d..8487219511bd 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditInfo.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditInfo.java @@ -24,6 +24,9 @@ import java.util.Objects; +/** + * Contains information about the author who performed an audited operation. + */ public class AuditInfo { private final String author; diff --git a/processing/src/main/java/org/apache/druid/audit/RequestInfo.java b/processing/src/main/java/org/apache/druid/audit/RequestInfo.java new file mode 100644 index 000000000000..e9a97e6154c6 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/audit/RequestInfo.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.audit; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +/** + * Contains information about a REST API request that was audited. + */ +public class RequestInfo +{ + private final String service; + private final String method; + private final String path; + private final String queryParams; + + @JsonCreator + public RequestInfo( + @JsonProperty("service") String service, + @JsonProperty("method") String method, + @JsonProperty("path") String path, + @JsonProperty("queryParams") String queryParams + ) + { + this.service = service; + this.method = method; + this.path = path; + this.queryParams = queryParams; + } + + @JsonProperty + public String getService() + { + return service; + } + + @JsonProperty + public String getMethod() + { + return method; + } + + @JsonProperty + public String getPath() + { + return path; + } + + @JsonProperty + public String getQueryParams() + { + return queryParams; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RequestInfo that = (RequestInfo) o; + return Objects.equals(this.service, that.service) + && Objects.equals(this.method, that.method) + && Objects.equals(this.path, that.path) + && Objects.equals(this.queryParams, that.queryParams); + } + + @Override + public int hashCode() + { + return Objects.hash(service, method, path, queryParams); + } + + @Override + public String toString() + { + return "RequestInfo{" + + "service='" + service + '\'' + + ", method='" + method + '\'' + + ", path='" + path + '\'' + + ", queryParams='" + queryParams + '\'' + + '}'; + } +} diff --git a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java index c0f3dc69118d..390ef302274b 100644 --- a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java +++ b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java @@ -63,6 +63,7 @@ public void testAuditEntrySerde() throws IOException "testComment", "127.0.0.1" ), + new RequestInfo("overlord", "GET", "/segments", "?abc=1"), AuditEntry.Payload.fromString("testPayload"), DateTimes.of("2013-01-01T00:00:00Z") ); diff --git a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java index 1e8044dcb53a..836115d47a17 100644 --- a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java +++ b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java @@ -24,7 +24,6 @@ import com.google.inject.Module; import com.google.inject.multibindings.MapBinder; import org.apache.druid.audit.AuditManager; -import org.apache.druid.audit.AuditManagerConfig; import org.apache.druid.indexer.MetadataStorageUpdaterJobHandler; import org.apache.druid.indexer.SQLMetadataStorageUpdaterJobHandler; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; @@ -47,11 +46,10 @@ import org.apache.druid.metadata.SegmentsMetadataManagerProvider; import org.apache.druid.metadata.SqlSegmentsMetadataManager; import org.apache.druid.metadata.SqlSegmentsMetadataManagerProvider; +import org.apache.druid.server.audit.AuditManagerConfig; import org.apache.druid.server.audit.AuditSerdeHelper; import org.apache.druid.server.audit.LoggingAuditManager; -import org.apache.druid.server.audit.LoggingAuditManagerConfig; import org.apache.druid.server.audit.SQLAuditManager; -import org.apache.druid.server.audit.SQLAuditManagerConfig; public class SQLMetadataStorageDruidModule implements Module { @@ -87,7 +85,7 @@ public void createBindingChoices(Binder binder, String defaultValue) PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageUpdaterJobHandler.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataSupervisorManager.class), defaultValue); - configureAuditDestination(binder); + configureAuditManager(binder); } @Override @@ -139,8 +137,10 @@ public void configure(Binder binder) .in(LazySingleton.class); } - private void configureAuditDestination(Binder binder) + private void configureAuditManager(Binder binder) { + JsonConfigProvider.bind(binder, "druid.audit.manager", AuditManagerConfig.class); + PolyBind.createChoice( binder, "druid.audit.manager.type", @@ -158,23 +158,6 @@ private void configureAuditDestination(Binder binder) .to(SQLAuditManager.class) .in(LazySingleton.class); - PolyBind.createChoice( - binder, - "druid.audit.manager.type", - Key.get(AuditManagerConfig.class), - Key.get(SQLAuditManagerConfig.class) - ); - final MapBinder auditManagerConfigBinder - = PolyBind.optionBinder(binder, Key.get(AuditManagerConfig.class)); - auditManagerConfigBinder - .addBinding("log") - .to(LoggingAuditManagerConfig.class) - .in(LazySingleton.class); - auditManagerConfigBinder - .addBinding("sql") - .to(SQLAuditManagerConfig.class) - .in(LazySingleton.class); - - binder.bind(AuditSerdeHelper.class); + binder.bind(AuditSerdeHelper.class).in(LazySingleton.class); } } diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java index d677ca685db6..a8c2692c0b67 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java @@ -30,7 +30,7 @@ public enum Level } private static final String MSG_FORMAT - = "[%s] User[%s], identity[%s], ip[%s] performed action[%s] on key[%s] with comment[%s]. Payload[%s]."; + = "User[%s], identity[%s], IP[%s] performed action[%s] on key[%s] with comment[%s]. Payload[%s]."; private final Level level; private final Logger logger = new Logger(AuditLogger.class); @@ -43,7 +43,6 @@ public AuditLogger(Level level) public void log(AuditEntry entry) { Object[] args = { - entry.getAuditTime(), entry.getAuditInfo().getAuthor(), entry.getAuditInfo().getIdentity(), entry.getAuditInfo().getIp(), diff --git a/processing/src/main/java/org/apache/druid/audit/AuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/AuditManagerConfig.java similarity index 66% rename from processing/src/main/java/org/apache/druid/audit/AuditManagerConfig.java rename to server/src/main/java/org/apache/druid/server/audit/AuditManagerConfig.java index 0858a466e058..2cb5acd3c82f 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditManagerConfig.java @@ -17,8 +17,17 @@ * under the License. */ -package org.apache.druid.audit; +package org.apache.druid.server.audit; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = SQLAuditManagerConfig.class) +@JsonSubTypes(value = { + @Type(name = "log", value = LoggingAuditManagerConfig.class), + @Type(name = "sql", value = SQLAuditManagerConfig.class) +}) public interface AuditManagerConfig { boolean isSkipNullField(); diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java index 29befd71e35c..76a02d05ea2a 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import org.apache.druid.audit.AuditEntry; -import org.apache.druid.audit.AuditManagerConfig; import org.apache.druid.guice.annotations.Json; import org.apache.druid.guice.annotations.JsonNonNull; import org.apache.druid.java.util.common.StringUtils; @@ -87,6 +86,7 @@ public AuditEntry processAuditEntry(AuditEntry entry) entry.getKey(), entry.getType(), entry.getAuditInfo(), + entry.getRequest(), processedPayload, entry.getAuditTime() ); diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java index c83155207bb0..22ad94f80a7c 100644 --- a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java @@ -22,6 +22,7 @@ import com.google.inject.Inject; import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditManager; +import org.apache.druid.error.DruidException; import org.joda.time.Interval; import java.util.Collections; @@ -38,12 +39,17 @@ public class LoggingAuditManager implements AuditManager @Inject public LoggingAuditManager( - LoggingAuditManagerConfig config, + AuditManagerConfig config, AuditSerdeHelper serdeHelper ) { this.serdeHelper = serdeHelper; - this.auditLogger = new AuditLogger(config.getLogLevel()); + if (!(config instanceof LoggingAuditManagerConfig)) { + throw DruidException.defensive("Config[%s] is not an instance of LoggingAuditManagerConfig", config); + } + + LoggingAuditManagerConfig logAuditConfig = (LoggingAuditManagerConfig) config; + this.auditLogger = new AuditLogger(logAuditConfig.getLogLevel()); } @Override diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java index eba8d412c507..6d6ab6e18d6e 100644 --- a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java @@ -20,7 +20,6 @@ package org.apache.druid.server.audit; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.audit.AuditManagerConfig; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.HumanReadableBytesRange; diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java index 019f0156f42f..0ab855a0e2d3 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java @@ -24,6 +24,7 @@ import com.google.inject.Inject; import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditManager; +import org.apache.druid.error.DruidException; import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.guice.annotations.Json; import org.apache.druid.java.util.common.DateTimes; @@ -65,7 +66,7 @@ public class SQLAuditManager implements AuditManager @Inject public SQLAuditManager( - SQLAuditManagerConfig config, + AuditManagerConfig config, AuditSerdeHelper serdeHelper, SQLMetadataConnector connector, Supplier dbTables, @@ -79,8 +80,12 @@ public SQLAuditManager( this.emitter = emitter; this.jsonMapper = jsonMapper; this.serdeHelper = serdeHelper; - this.config = config; this.resultMapper = new AuditEntryMapper(); + + if (!(config instanceof SQLAuditManagerConfig)) { + throw DruidException.defensive("Config[%s] is not an instance of SQLAuditManagerConfig", config); + } + this.config = (SQLAuditManagerConfig) config; } @LifecycleStart diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java index 2392d5f66e9a..aa012bb29599 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java @@ -20,7 +20,6 @@ package org.apache.druid.server.audit; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.audit.AuditManagerConfig; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.HumanReadableBytesRange; diff --git a/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java b/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java new file mode 100644 index 000000000000..e599532b4fcd --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.audit; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Injector; +import org.apache.druid.guice.GuiceInjectors; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.guice.JsonConfigurator; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Properties; + +public class AuditManagerConfigTest +{ + private static final String CONFIG_BASE = "druid.audit.manager"; + + @Test + public void testDefaultAuditConfig() + { + final Injector injector = createInjector(); + final JsonConfigProvider provider = JsonConfigProvider.of( + CONFIG_BASE, + AuditManagerConfig.class + ); + + provider.inject(new Properties(), injector.getInstance(JsonConfigurator.class)); + final AuditManagerConfig config = provider.get(); + Assert.assertTrue(config instanceof SQLAuditManagerConfig); + + final SQLAuditManagerConfig sqlAuditConfig = (SQLAuditManagerConfig) config; + Assert.assertFalse(sqlAuditConfig.isSkipNullField()); + Assert.assertFalse(sqlAuditConfig.isIncludePayloadAsDimensionInMetric()); + Assert.assertEquals(-1, sqlAuditConfig.getMaxPayloadSizeBytes()); + Assert.assertEquals(7 * 86400 * 1000, sqlAuditConfig.getAuditHistoryMillis()); + } + + @Test + public void testDefaultLogAuditConfig() + { + final Injector injector = createInjector(); + final JsonConfigProvider provider = JsonConfigProvider.of( + CONFIG_BASE, + AuditManagerConfig.class + ); + + final Properties props = new Properties(); + props.setProperty("druid.audit.manager.type", "log"); + + provider.inject(props, injector.getInstance(JsonConfigurator.class)); + final AuditManagerConfig config = provider.get(); + Assert.assertTrue(config instanceof LoggingAuditManagerConfig); + + final LoggingAuditManagerConfig logAuditConfig = (LoggingAuditManagerConfig) config; + Assert.assertFalse(logAuditConfig.isSkipNullField()); + Assert.assertEquals(-1, logAuditConfig.getMaxPayloadSizeBytes()); + Assert.assertEquals(AuditLogger.Level.INFO, logAuditConfig.getLogLevel()); + } + + @Test + public void testLogAuditConfig() + { + final Injector injector = createInjector(); + final JsonConfigProvider provider = JsonConfigProvider.of( + CONFIG_BASE, + AuditManagerConfig.class + ); + + final Properties props = new Properties(); + props.setProperty("druid.audit.manager.type", "log"); + props.setProperty("druid.audit.manager.logLevel", "WARN"); + + provider.inject(props, injector.getInstance(JsonConfigurator.class)); + + final AuditManagerConfig config = provider.get(); + Assert.assertTrue(config instanceof LoggingAuditManagerConfig); + + final LoggingAuditManagerConfig logAuditConfig = (LoggingAuditManagerConfig) config; + Assert.assertFalse(logAuditConfig.isSkipNullField()); + Assert.assertEquals(-1, logAuditConfig.getMaxPayloadSizeBytes()); + Assert.assertEquals(AuditLogger.Level.WARN, logAuditConfig.getLogLevel()); + } + + private Injector createInjector() + { + return GuiceInjectors.makeStartupInjectorWithModules( + ImmutableList.of( + binder -> JsonConfigProvider.bind(binder, CONFIG_BASE, AuditManagerConfig.class) + ) + ); + } +} diff --git a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java index 409b7196752b..31e9d85ee493 100644 --- a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java @@ -345,14 +345,7 @@ private AuditEntry lookupAuditEntryForKey(String key) throws IOException if (payload == null) { return null; } else { - AuditEntry record = mapper.readValue(payload, AuditEntry.class); - return new AuditEntry( - record.getKey(), - record.getType(), - record.getAuditInfo(), - record.getPayload(), - record.getAuditTime() - ); + return mapper.readValue(payload, AuditEntry.class); } } diff --git a/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java index eceae4a08787..ab4bff0f58c2 100644 --- a/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java @@ -85,7 +85,11 @@ public void testGetDatasourceRuleHistoryWithInterval() AuditEntry entry2 = createAuditEntry( DateTimes.of("2013-01-01T00:00:00Z") ); - EasyMock.expect(auditManager.fetchAuditHistory(EasyMock.eq("datasource1"), EasyMock.eq("rules"), EasyMock.eq(theInterval))) + EasyMock.expect(auditManager.fetchAuditHistory( + EasyMock.eq("datasource1"), + EasyMock.eq("rules"), + EasyMock.eq(theInterval) + )) .andReturn(ImmutableList.of(entry1, entry2)) .once(); EasyMock.replay(auditManager); @@ -105,8 +109,8 @@ public void testGetDatasourceRuleHistoryWithInterval() public void testGetDatasourceRuleHistoryWithWrongCount() { EasyMock.expect(auditManager.fetchAuditHistory(EasyMock.eq("datasource1"), EasyMock.eq("rules"), EasyMock.eq(-1))) - .andThrow(new IllegalArgumentException("Limit must be greater than zero!")) - .once(); + .andThrow(new IllegalArgumentException("Limit must be greater than zero!")) + .once(); EasyMock.replay(auditManager); RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); @@ -176,8 +180,8 @@ public void testGetAllDatasourcesRuleHistoryWithInterval() public void testGetAllDatasourcesRuleHistoryWithWrongCount() { EasyMock.expect(auditManager.fetchAuditHistory(EasyMock.eq("rules"), EasyMock.eq(-1))) - .andThrow(new IllegalArgumentException("Limit must be greater than zero!")) - .once(); + .andThrow(new IllegalArgumentException("Limit must be greater than zero!")) + .once(); EasyMock.replay(auditManager); RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); @@ -198,7 +202,10 @@ private AuditInfo createAuditInfo() private AuditEntry createAuditEntry(DateTime auditTime) { - return new AuditEntry("testKey", "testType", createAuditInfo(), AuditEntry.Payload.fromString("testPayload"), auditTime); + return new AuditEntry( + "testKey", "testType", createAuditInfo(), null, + AuditEntry.Payload.fromString("testPayload"), auditTime + ); } } From 5e5a2fc4f07d37475eb9b6fcbab3d3e98a364f56 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Fri, 8 Dec 2023 18:53:38 +0530 Subject: [PATCH 16/29] Fix tests --- .../endpoint/BasicAuthenticatorResource.java | 46 +++-- .../overlord/http/OverlordResource.java | 1 + .../indexing/overlord/http/OverlordTest.java | 7 +- .../org/apache/druid/audit/RequestInfo.java | 16 +- .../druid/server/audit/AuditLogger.java | 3 +- .../server/http/DataSourcesResource.java | 38 ++-- .../server/security/AuthorizationUtils.java | 11 ++ .../server/http/DataSourcesResourceTest.java | 163 +++++++++--------- 8 files changed, 149 insertions(+), 136 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java index c9a36c1ca240..60e84faaf289 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java @@ -23,7 +23,6 @@ import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; import org.apache.druid.audit.AuditEntry; -import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.guice.LazySingleton; import org.apache.druid.security.basic.BasicSecurityResourceFilter; @@ -161,11 +160,7 @@ public Response createUser( authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.createUser(authenticatorName, userName); - - if (isSuccess(response)) { - final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(req); - performAudit(authenticatorName, "users.create", Collections.singletonMap("username", userName), auditInfo); - } + performAudit(authenticatorName, "basicAuth.createUser", userName, req, response); return response; } @@ -191,11 +186,7 @@ public Response deleteUser( { authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.deleteUser(authenticatorName, userName); - - if (isSuccess(response)) { - final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(req); - performAudit(authenticatorName, "users.delete", Collections.singletonMap("username", userName), auditInfo); - } + performAudit(authenticatorName, "basicAuth.deleteUser", userName, req, response); return response; } @@ -222,11 +213,7 @@ public Response updateUserCredentials( { authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.updateUserCredentials(authenticatorName, userName, update); - - if (isSuccess(response)) { - final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(req); - performAudit(authenticatorName, "users.update", Collections.singletonMap("username", userName), auditInfo); - } + performAudit(authenticatorName, "basicAuth.updateUserCreds", userName, req, response); return response; } @@ -278,15 +265,24 @@ private boolean isSuccess(Response response) return responseCode >= 200 && responseCode < 300; } - private void performAudit(String authenticatorName, String action, Object payload, AuditInfo auditInfo) + private void performAudit( + String authenticatorName, + String action, + String updatedUser, + HttpServletRequest request, + Response response + ) { - auditManager.doAudit( - AuditEntry.builder() - .key(authenticatorName) - .type(action) - .auditInfo(auditInfo) - .payload(payload) - .build() - ); + if (isSuccess(response)) { + auditManager.doAudit( + AuditEntry.builder() + .key(authenticatorName) + .type(action) + .auditInfo(AuthorizationUtils.buildAuditInfo(request)) + .request(AuthorizationUtils.buildRequestInfo("coordinator", request)) + .payload(Collections.singletonMap("username", updatedUser)) + .build() + ); + } } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index dcbf1e0ef0ca..9129aba001df 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -232,6 +232,7 @@ public Response taskPost( AuditEntry.builder() .key(task.getDataSource()) .type("ingest.batch") + .request(AuthorizationUtils.buildRequestInfo("overlord", req)) .payload(new TaskIdentifier(task.getId(), task.getGroupId(), task.getType())) .auditInfo(auditInfo) .build() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java index 04228ab68c2b..b0df15ce4ef5 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java @@ -155,6 +155,10 @@ public void setUp() throws Exception EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( new AuthenticationResult("druid", "druid", null, null) ).anyTimes(); + EasyMock.expect(req.getMethod()).andReturn("GET").anyTimes(); + EasyMock.expect(req.getRequestURI()).andReturn("/request/uri").anyTimes(); + EasyMock.expect(req.getQueryString()).andReturn("query=string").anyTimes(); + EasyMock.expect(req.getAttribute(AuthConfig.DRUID_ALLOW_UNSECURED_PATH)).andReturn(null).anyTimes(); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes(); req.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); @@ -264,13 +268,14 @@ public void testOverlordRun() throws Exception final TaskStorageQueryAdapter taskStorageQueryAdapter = new TaskStorageQueryAdapter(taskStorage, taskLockbox, taskMaster); final WorkerTaskRunnerQueryAdapter workerTaskRunnerQueryAdapter = new WorkerTaskRunnerQueryAdapter(taskMaster, null); // Test Overlord resource stuff + AuditManager auditManager = EasyMock.createNiceMock(AuditManager.class); overlordResource = new OverlordResource( taskMaster, taskStorageQueryAdapter, new IndexerMetadataStorageAdapter(taskStorageQueryAdapter, null), null, null, - null, + auditManager, AuthTestUtils.TEST_AUTHORIZER_MAPPER, workerTaskRunnerQueryAdapter, null, diff --git a/processing/src/main/java/org/apache/druid/audit/RequestInfo.java b/processing/src/main/java/org/apache/druid/audit/RequestInfo.java index e9a97e6154c6..706fdca2f01a 100644 --- a/processing/src/main/java/org/apache/druid/audit/RequestInfo.java +++ b/processing/src/main/java/org/apache/druid/audit/RequestInfo.java @@ -31,20 +31,20 @@ public class RequestInfo { private final String service; private final String method; - private final String path; + private final String uri; private final String queryParams; @JsonCreator public RequestInfo( @JsonProperty("service") String service, @JsonProperty("method") String method, - @JsonProperty("path") String path, + @JsonProperty("uri") String uri, @JsonProperty("queryParams") String queryParams ) { this.service = service; this.method = method; - this.path = path; + this.uri = uri; this.queryParams = queryParams; } @@ -61,9 +61,9 @@ public String getMethod() } @JsonProperty - public String getPath() + public String getUri() { - return path; + return uri; } @JsonProperty @@ -84,14 +84,14 @@ public boolean equals(Object o) RequestInfo that = (RequestInfo) o; return Objects.equals(this.service, that.service) && Objects.equals(this.method, that.method) - && Objects.equals(this.path, that.path) + && Objects.equals(this.uri, that.uri) && Objects.equals(this.queryParams, that.queryParams); } @Override public int hashCode() { - return Objects.hash(service, method, path, queryParams); + return Objects.hash(service, method, uri, queryParams); } @Override @@ -100,7 +100,7 @@ public String toString() return "RequestInfo{" + "service='" + service + '\'' + ", method='" + method + '\'' + - ", path='" + path + '\'' + + ", path='" + uri + '\'' + ", queryParams='" + queryParams + '\'' + '}'; } diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java index a8c2692c0b67..8aa2213180fe 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java @@ -30,7 +30,7 @@ public enum Level } private static final String MSG_FORMAT - = "User[%s], identity[%s], IP[%s] performed action[%s] on key[%s] with comment[%s]. Payload[%s]."; + = "User[%s], identity[%s], IP[%s] performed action[%s] on key[%s] with comment[%s]. Request[%s], payload[%s]."; private final Level level; private final Logger logger = new Logger(AuditLogger.class); @@ -49,6 +49,7 @@ public void log(AuditEntry entry) entry.getType(), entry.getKey(), entry.getAuditInfo().getComment(), + entry.getRequest(), entry.getPayload() }; switch (level) { diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 015de569e4ba..af98c886b972 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -253,16 +253,15 @@ public Response markSegmentsAsUnused( ); auditPayload = Collections.singletonMap("segmentIds", segmentIds); } - if (auditManager != null) { - auditManager.doAudit( - AuditEntry.builder() - .key(dataSourceName) - .type("segments.markUnused") - .payload(auditPayload) - .auditInfo(AuthorizationUtils.buildAuditInfo(req)) - .build() - ); - } + auditManager.doAudit( + AuditEntry.builder() + .key(dataSourceName) + .type("markSegmentsAsUnused") + .payload(auditPayload) + .auditInfo(AuthorizationUtils.buildAuditInfo(req)) + .request(AuthorizationUtils.buildRequestInfo("coordinator", req)) + .build() + ); return numUpdatedSegments; }; return doMarkSegmentsWithPayload("markSegmentsAsUnused", dataSourceName, payload, markSegments); @@ -373,16 +372,15 @@ public Response killUnusedSegmentsInInterval( overlordClient.runKillTask("api-issued", dataSourceName, theInterval, null), true ); - if (auditManager != null) { - auditManager.doAudit( - AuditEntry.builder() - .key(dataSourceName) - .type("segments.killTask") - .payload(ImmutableMap.of("killTaskId", killTaskId, "interval", theInterval)) - .auditInfo(AuthorizationUtils.buildAuditInfo(req)) - .build() - ); - } + auditManager.doAudit( + AuditEntry.builder() + .key(dataSourceName) + .type("killUnusedSegmentsInInterval") + .payload(ImmutableMap.of("killTaskId", killTaskId, "interval", theInterval)) + .auditInfo(AuthorizationUtils.buildAuditInfo(req)) + .request(AuthorizationUtils.buildRequestInfo("coordinator", req)) + .build() + ); return Response.ok().build(); } catch (Exception e) { diff --git a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java index 03251fac01c4..d05e7cb49040 100644 --- a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java +++ b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java @@ -24,6 +24,7 @@ import com.google.common.collect.Lists; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; +import org.apache.druid.audit.RequestInfo; import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.ISE; @@ -130,6 +131,16 @@ public static AuditInfo buildAuditInfo(HttpServletRequest request) ); } + public static RequestInfo buildRequestInfo(String service, HttpServletRequest request) + { + return new RequestInfo( + service, + request.getMethod(), + request.getRequestURI(), + request.getQueryString() + ); + } + /** * Check a list of resource-actions to be performed by the identity represented by authenticationResult. * diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 6b71a5a033ba..386a486c1638 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -28,6 +28,7 @@ import com.google.common.util.concurrent.Futures; import it.unimi.dsi.fastutil.objects.Object2LongMap; import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap; +import org.apache.druid.audit.AuditManager; import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; @@ -89,6 +90,7 @@ public class DataSourcesResourceTest private List dataSegmentList; private HttpServletRequest request; private SegmentsMetadataManager segmentsMetadataManager; + private AuditManager auditManager; @Before public void setUp() @@ -96,6 +98,7 @@ public void setUp() request = EasyMock.createStrictMock(HttpServletRequest.class); inventoryView = EasyMock.createStrictMock(CoordinatorServerView.class); server = EasyMock.niceMock(DruidServer.class); + auditManager = EasyMock.niceMock(AuditManager.class); dataSegmentList = new ArrayList<>(); dataSegmentList.add( new DataSegment( @@ -180,8 +183,7 @@ public void testGetFullQueryableDataSources() EasyMock.expectLastCall().times(1); EasyMock.replay(inventoryView, server, request); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.getQueryableDataSources("full", null, request); Set result = (Set) response.getEntity(); Assert.assertEquals(200, response.getStatus()); @@ -255,9 +257,8 @@ public Access authorize(AuthenticationResult authenticationResult1, Resource res } }; - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, null, authMapper, null, - null - ); + DataSourcesResource dataSourcesResource = + new DataSourcesResource(inventoryView, null, null, null, authMapper, null, auditManager); Response response = dataSourcesResource.getQueryableDataSources("full", null, request); Set result = (Set) response.getEntity(); @@ -295,8 +296,7 @@ public void testGetSimpleQueryableDataSources() EasyMock.expectLastCall().times(1); EasyMock.replay(inventoryView, server, request); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.getQueryableDataSources(null, "simple", request); Assert.assertEquals(200, response.getStatus()); List> results = (List>) response.getEntity(); @@ -319,8 +319,7 @@ public void testFullGetTheDataSource() EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server)).atLeastOnce(); EasyMock.replay(inventoryView, server); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.getDataSource("datasource1", "full"); ImmutableDruidDataSource result = (ImmutableDruidDataSource) response.getEntity(); Assert.assertEquals(200, response.getStatus()); @@ -335,8 +334,7 @@ public void testNullGetTheDataSource() EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server)).atLeastOnce(); EasyMock.replay(inventoryView, server); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Assert.assertEquals(204, dataSourcesResource.getDataSource("none", null).getStatus()); EasyMock.verify(inventoryView, server); } @@ -353,8 +351,7 @@ public void testSimpleGetTheDataSource() EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server)).atLeastOnce(); EasyMock.replay(inventoryView, server); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.getDataSource("datasource1", null); Assert.assertEquals(200, response.getStatus()); Map> result = (Map>) response.getEntity(); @@ -387,7 +384,7 @@ public void testSimpleGetTheDataSourceManyTiers() EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server, server2, server3)).atLeastOnce(); EasyMock.replay(inventoryView, server, server2, server3); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.getDataSource("datasource1", null); Assert.assertEquals(200, response.getStatus()); Map> result = (Map>) response.getEntity(); @@ -425,7 +422,7 @@ public void testSimpleGetTheDataSourceWithReplicatedSegments() EasyMock.replay(inventoryView); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.getDataSource("datasource1", null); Assert.assertEquals(200, response.getStatus()); Map> result1 = (Map>) response.getEntity(); @@ -469,15 +466,14 @@ public void testGetSegmentDataSourceIntervals() List expectedIntervals = new ArrayList<>(); expectedIntervals.add(Intervals.of("2010-01-22T00:00:00.000Z/2010-01-23T00:00:00.000Z")); expectedIntervals.add(Intervals.of("2010-01-01T00:00:00.000Z/2010-01-02T00:00:00.000Z")); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.getIntervalsWithServedSegmentsOrAllServedSegmentsPerIntervals( "invalidDataSource", null, null ); - Assert.assertEquals(response.getEntity(), null); + Assert.assertNull(response.getEntity()); response = dataSourcesResource.getIntervalsWithServedSegmentsOrAllServedSegmentsPerIntervals( "datasource1", @@ -529,15 +525,14 @@ public void testGetServedSegmentsInIntervalInDataSource() EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server)).atLeastOnce(); EasyMock.replay(inventoryView); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.getServedSegmentsInInterval( "invalidDataSource", "2010-01-01/P1D", null, null ); - Assert.assertEquals(null, response.getEntity()); + Assert.assertNull(response.getEntity()); response = dataSourcesResource.getServedSegmentsInInterval( "datasource1", @@ -591,16 +586,17 @@ public void testGetServedSegmentsInIntervalInDataSource() @Test public void testKillSegmentsInIntervalInDataSource() { - String interval = "2010-01-01_P1D"; + String interval = "2010-01-01/P1D"; Interval theInterval = Intervals.of(interval.replace('_', '/')); OverlordClient overlordClient = EasyMock.createStrictMock(OverlordClient.class); EasyMock.expect(overlordClient.runKillTask("api-issued", "datasource1", theInterval, null)) - .andReturn(Futures.immediateFuture(null)); + .andReturn(Futures.immediateFuture("kill_task_1")); EasyMock.replay(overlordClient, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, overlordClient, null, null, null); + new DataSourcesResource(inventoryView, null, null, overlordClient, null, null, auditManager); + prepareRequestForAudit(); Response response = dataSourcesResource.killUnusedSegmentsInInterval("datasource1", interval, request); Assert.assertEquals(200, response.getStatus()); @@ -614,7 +610,7 @@ public void testMarkAsUnusedAllSegmentsInDataSource() OverlordClient overlordClient = EasyMock.createStrictMock(OverlordClient.class); EasyMock.replay(overlordClient, server); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, null, overlordClient, null, null, null); + new DataSourcesResource(inventoryView, null, null, overlordClient, null, null, auditManager); try { Response response = dataSourcesResource.markAsUnusedAllSegmentsOrKillUnusedSegmentsInInterval("datasource", "true", "???", request); @@ -637,7 +633,7 @@ public void testIsHandOffComplete() Rule loadRule = new IntervalLoadRule(Intervals.of("2013-01-02T00:00:00Z/2013-01-03T00:00:00Z"), null, null); Rule dropRule = new IntervalDropRule(Intervals.of("2013-01-01T00:00:00Z/2013-01-02T00:00:00Z")); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, databaseRuleManager, null, null, null, null); + new DataSourcesResource(inventoryView, null, databaseRuleManager, null, null, null, auditManager); // test dropped EasyMock.expect(databaseRuleManager.getRulesWithDefault("dataSource1")) @@ -706,9 +702,7 @@ public void testMarkSegmentAsUsed() EasyMock.expect(segmentsMetadataManager.markSegmentAsUsed(segment.getId().toString())).andReturn(true).once(); EasyMock.replay(segmentsMetadataManager); - DataSourcesResource dataSourcesResource = new DataSourcesResource(null, segmentsMetadataManager, null, null, null, null, - null - ); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.markSegmentAsUsed(segment.getDataSource(), segment.getId().toString()); Assert.assertEquals(200, response.getStatus()); @@ -722,9 +716,7 @@ public void testMarkSegmentAsUsedNoChange() EasyMock.expect(segmentsMetadataManager.markSegmentAsUsed(segment.getId().toString())).andReturn(false).once(); EasyMock.replay(segmentsMetadataManager); - DataSourcesResource dataSourcesResource = new DataSourcesResource(null, segmentsMetadataManager, null, null, null, null, - null - ); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.markSegmentAsUsed(segment.getDataSource(), segment.getId().toString()); Assert.assertEquals(200, response.getStatus()); @@ -744,9 +736,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsInterval() EasyMock.expect(server.getDataSource("datasource1")).andReturn(dataSource).once(); EasyMock.replay(segmentsMetadataManager, inventoryView, server); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); - + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", new DataSourcesResource.MarkDataSourceSegmentsPayload(interval, null) @@ -767,8 +757,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsIntervalNoneUpdated() EasyMock.expect(server.getDataSource("datasource1")).andReturn(dataSource).once(); EasyMock.replay(segmentsMetadataManager, inventoryView, server); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -790,8 +779,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsSet() throws UnknownSegmentIdsE EasyMock.expect(server.getDataSource("datasource1")).andReturn(dataSource).once(); EasyMock.replay(segmentsMetadataManager, inventoryView, server); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -813,8 +801,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsIntervalException() EasyMock.expect(server.getDataSource("datasource1")).andReturn(dataSource).once(); EasyMock.replay(segmentsMetadataManager, inventoryView, server); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -831,8 +818,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsNoDataSource() EasyMock.expect(server.getDataSource("datasource1")).andReturn(null).once(); EasyMock.replay(segmentsMetadataManager, inventoryView, server); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -845,8 +831,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsNoDataSource() @Test public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadNoArguments() { - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -858,8 +843,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadNoArguments() @Test public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadBothArguments() { - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -871,8 +855,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadBothArguments() @Test public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadEmptyArray() { - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments( "datasource1", @@ -884,8 +867,7 @@ public void testMarkAsUsedNonOvershadowedSegmentsInvalidPayloadEmptyArray() @Test public void testMarkAsUsedNonOvershadowedSegmentsNoPayload() { - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.markAsUsedNonOvershadowedSegments("datasource1", null); Assert.assertEquals(400, response.getStatus()); @@ -1044,8 +1026,8 @@ public void testMarkSegmentsAsUnused() .collect(Collectors.toSet()) ); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); + prepareRequestForAudit(); Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 1), response.getEntity()); @@ -1075,8 +1057,8 @@ public void testMarkSegmentsAsUnusedNoChanges() .collect(Collectors.toSet()) ); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); + prepareRequestForAudit(); Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 0), response.getEntity()); @@ -1109,7 +1091,7 @@ public void testMarkSegmentsAsUnusedException() ); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + createResource(); Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(500, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1130,8 +1112,8 @@ public void testMarkAsUnusedSegmentsInInterval() final DataSourcesResource.MarkDataSourceSegmentsPayload payload = new DataSourcesResource.MarkDataSourceSegmentsPayload(theInterval, null); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); + prepareRequestForAudit(); Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 1), response.getEntity()); @@ -1153,8 +1135,8 @@ public void testMarkAsUnusedSegmentsInIntervalNoChanges() final DataSourcesResource.MarkDataSourceSegmentsPayload payload = new DataSourcesResource.MarkDataSourceSegmentsPayload(theInterval, null); - DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + DataSourcesResource dataSourcesResource = createResource(); + prepareRequestForAudit(); Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(ImmutableMap.of("numChangedSegments", 0), response.getEntity()); @@ -1178,7 +1160,7 @@ public void testMarkAsUnusedSegmentsInIntervalException() new DataSourcesResource.MarkDataSourceSegmentsPayload(theInterval, null); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + createResource(); Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload, request); Assert.assertEquals(500, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1189,7 +1171,7 @@ public void testMarkAsUnusedSegmentsInIntervalException() public void testMarkSegmentsAsUnusedNullPayload() { DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + createResource(); Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", null, request); Assert.assertEquals(400, response.getStatus()); @@ -1204,7 +1186,7 @@ public void testMarkSegmentsAsUnusedNullPayload() public void testMarkSegmentsAsUnusedInvalidPayload() { DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + createResource(); final DataSourcesResource.MarkDataSourceSegmentsPayload payload = new DataSourcesResource.MarkDataSourceSegmentsPayload(null, null); @@ -1218,7 +1200,7 @@ public void testMarkSegmentsAsUnusedInvalidPayload() public void testMarkSegmentsAsUnusedInvalidPayloadBothArguments() { DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + createResource(); final DataSourcesResource.MarkDataSourceSegmentsPayload payload = new DataSourcesResource.MarkDataSourceSegmentsPayload(Intervals.of("2010-01-01/P1D"), ImmutableSet.of()); @@ -1231,9 +1213,7 @@ public void testMarkSegmentsAsUnusedInvalidPayloadBothArguments() @Test public void testGetDatasourceLoadstatusForceMetadataRefreshNull() { - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, - null - ); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", null, null, null, null, null); Assert.assertEquals(400, response.getStatus()); } @@ -1314,9 +1294,7 @@ public void testGetDatasourceLoadstatusDefault() EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, - null - ); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, null, null, null); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1332,7 +1310,7 @@ public void testGetDatasourceLoadstatusDefault() EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); - dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + dataSourcesResource = createResource(); response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, null, null, null); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1395,9 +1373,7 @@ public void testGetDatasourceLoadstatusSimple() EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, - null - ); + DataSourcesResource dataSourcesResource = createResource(); Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, "simple", null, null); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1413,7 +1389,7 @@ public void testGetDatasourceLoadstatusSimple() EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); - dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null, null); + dataSourcesResource = createResource(); response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, "simple", null, null); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1468,9 +1444,8 @@ public void testGetDatasourceLoadstatusFull() EasyMock.replay(segmentsMetadataManager, druidCoordinator); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, druidCoordinator, - null - ); + DataSourcesResource dataSourcesResource = + new DataSourcesResource(null, segmentsMetadataManager, null, null, null, druidCoordinator, auditManager); Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, null, "full", null); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1527,9 +1502,8 @@ public void testGetDatasourceLoadstatusFullAndComputeUsingClusterView() EasyMock.replay(segmentsMetadataManager, druidCoordinator); - DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, druidCoordinator, - null - ); + DataSourcesResource dataSourcesResource = + new DataSourcesResource(null, segmentsMetadataManager, null, null, null, druidCoordinator, auditManager); Response response = dataSourcesResource.getDatasourceLoadstatus("datasource1", true, null, null, "full", "computeUsingClusterView"); Assert.assertEquals(200, response.getStatus()); Assert.assertNotNull(response.getEntity()); @@ -1569,4 +1543,31 @@ private DataSegment createSegment(Interval interval, String version, int partiti 0, 0 ); } + + private void prepareRequestForAudit() + { + EasyMock.expect(request.getHeader(AuditManager.X_DRUID_AUTHOR)).andReturn("author").anyTimes(); + EasyMock.expect(request.getHeader(AuditManager.X_DRUID_COMMENT)).andReturn("comment").anyTimes(); + EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(null).anyTimes(); + EasyMock.expect(request.getRemoteAddr()).andReturn("127.0.0.1").anyTimes(); + + EasyMock.expect(request.getMethod()).andReturn("POST").anyTimes(); + EasyMock.expect(request.getRequestURI()).andReturn("/request/uri").anyTimes(); + EasyMock.expect(request.getQueryString()).andReturn("query=string").anyTimes(); + + EasyMock.replay(request); + } + + private DataSourcesResource createResource() + { + return new DataSourcesResource( + inventoryView, + segmentsMetadataManager, + null, + null, + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + null, + auditManager + ); + } } From 71ae17515ba09d218f30177d0614b22e297e839f Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Fri, 8 Dec 2023 19:19:09 +0530 Subject: [PATCH 17/29] Minor cleanup --- .../endpoint/BasicAuthenticatorResource.java | 8 ++++---- .../druid/server/security/AuthorizationUtils.java | 3 +++ .../druid/metadata/SQLMetadataRuleManagerTest.java | 4 +++- .../http/CoordinatorCompactionConfigsResourceTest.java | 10 ---------- .../src/dialogs/history-dialog/history-dialog.tsx | 2 +- 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java index 60e84faaf289..7881fcd5343a 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java @@ -160,7 +160,7 @@ public Response createUser( authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.createUser(authenticatorName, userName); - performAudit(authenticatorName, "basicAuth.createUser", userName, req, response); + performAuditIfSuccess(authenticatorName, "basicAuth.createUser", userName, req, response); return response; } @@ -186,7 +186,7 @@ public Response deleteUser( { authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.deleteUser(authenticatorName, userName); - performAudit(authenticatorName, "basicAuth.deleteUser", userName, req, response); + performAuditIfSuccess(authenticatorName, "basicAuth.deleteUser", userName, req, response); return response; } @@ -213,7 +213,7 @@ public Response updateUserCredentials( { authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.updateUserCredentials(authenticatorName, userName, update); - performAudit(authenticatorName, "basicAuth.updateUserCreds", userName, req, response); + performAuditIfSuccess(authenticatorName, "basicAuth.updateUserCreds", userName, req, response); return response; } @@ -265,7 +265,7 @@ private boolean isSuccess(Response response) return responseCode >= 200 && responseCode < 300; } - private void performAudit( + private void performAuditIfSuccess( String authenticatorName, String action, String updatedUser, diff --git a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java index d05e7cb49040..431819da8a42 100644 --- a/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java +++ b/server/src/main/java/org/apache/druid/server/security/AuthorizationUtils.java @@ -131,6 +131,9 @@ public static AuditInfo buildAuditInfo(HttpServletRequest request) ); } + /** + * Builds a RequestInfo object that can be used for auditing purposes. + */ public static RequestInfo buildRequestInfo(String service, HttpServletRequest request) { return new RequestInfo( diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java index 940b770d0868..5a9fc0f13b01 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java @@ -74,7 +74,9 @@ public void setUp() tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); connector.createAuditTable(); auditManager = new SQLAuditManager( - new SQLAuditManagerConfig(), null, connector, + new SQLAuditManagerConfig(), + null, + connector, Suppliers.ofInstance(tablesConfig), new NoopServiceEmitter(), mapper diff --git a/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java b/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java index 6d65e4acdbcc..c31364ac9fd9 100644 --- a/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/CoordinatorCompactionConfigsResourceTest.java @@ -148,8 +148,6 @@ public void testSetCompactionTaskLimitWithExistingConfig() double compactionTaskSlotRatio = 0.5; int maxCompactionTaskSlots = 9; - String author = "maytas"; - String comment = "hello"; Response result = coordinatorCompactionConfigsResource.setCompactionTaskLimit( compactionTaskSlotRatio, maxCompactionTaskSlots, @@ -193,8 +191,6 @@ public void testAddOrUpdateCompactionConfigWithExistingConfig() null, ImmutableMap.of("key", "val") ); - String author = "maytas"; - String comment = "hello"; Response result = coordinatorCompactionConfigsResource.addOrUpdateCompactionConfig( newConfig, mockHttpServletRequest @@ -244,8 +240,6 @@ public void testDeleteCompactionConfigWithExistingConfig() ) ).thenReturn(originalConfig); - String author = "maytas"; - String comment = "hello"; Response result = coordinatorCompactionConfigsResource.deleteCompactionConfig( datasourceName, mockHttpServletRequest @@ -341,8 +335,6 @@ public void testSetCompactionTaskLimitWithoutExistingConfig() double compactionTaskSlotRatio = 0.5; int maxCompactionTaskSlots = 9; - String author = "maytas"; - String comment = "hello"; Response result = coordinatorCompactionConfigsResource.setCompactionTaskLimit( compactionTaskSlotRatio, maxCompactionTaskSlots, @@ -427,8 +419,6 @@ public void testDeleteCompactionConfigWithoutExistingConfigShouldFailAsDatasourc ArgumentMatchers.eq(CoordinatorCompactionConfig.empty()) ) ).thenReturn(CoordinatorCompactionConfig.empty()); - String author = "maytas"; - String comment = "hello"; Response result = coordinatorCompactionConfigsResource.deleteCompactionConfig( DATASOURCE_NOT_EXISTS, mockHttpServletRequest diff --git a/web-console/src/dialogs/history-dialog/history-dialog.tsx b/web-console/src/dialogs/history-dialog/history-dialog.tsx index 6b7f219e6471..d204f316d3ed 100644 --- a/web-console/src/dialogs/history-dialog/history-dialog.tsx +++ b/web-console/src/dialogs/history-dialog/history-dialog.tsx @@ -60,7 +60,7 @@ export const HistoryDialog = React.memo(function HistoryDialog(props: HistoryDia let content: JSX.Element; if (historyRecords.length === 0) { - content =
No history records available. Try setting 'druid.audit.destiation=sql' to record config changes in metadata store.
; + content =
No history records available
; } else { content = ( From 737b7966eabf910482149543914d09d936e84141 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Fri, 8 Dec 2023 20:54:09 +0530 Subject: [PATCH 18/29] Fix tests and coverage --- .../org/apache/druid/audit/AuditInfoTest.java | 8 +++++ .../audit/LoggingAuditManagerConfig.java | 3 +- .../server/audit/SQLAuditManagerConfig.java | 6 ++-- .../metadata/SQLMetadataRuleManagerTest.java | 7 ++-- .../server/audit/AuditManagerConfigTest.java | 34 +++++++++++++++++-- 5 files changed, 46 insertions(+), 12 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java index 390ef302274b..38860efa9f0d 100644 --- a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java +++ b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java @@ -71,4 +71,12 @@ public void testAuditEntrySerde() throws IOException Assert.assertEquals(entry, serde); } + @Test + public void testRequestInfoEquality() throws IOException + { + RequestInfo requestInfo = new RequestInfo("overlord", "GET", "/uri", "a=b"); + RequestInfo deserialized = mapper.readValue(mapper.writeValueAsString(requestInfo), RequestInfo.class); + Assert.assertEquals(requestInfo, deserialized); + } + } diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java index 6d6ab6e18d6e..6140936675c0 100644 --- a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java @@ -25,7 +25,6 @@ public class LoggingAuditManagerConfig implements AuditManagerConfig { - @JsonProperty private final AuditLogger.Level logLevel = AuditLogger.Level.INFO; @@ -37,7 +36,7 @@ public class LoggingAuditManagerConfig implements AuditManagerConfig private final HumanReadableBytes maxPayloadSizeBytes = HumanReadableBytes.valueOf(-1); @JsonProperty - private final boolean skipNullField = false; + private final Boolean skipNullField = false; @Override public boolean isSkipNullField() diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java index aa012bb29599..290072965732 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java @@ -28,10 +28,10 @@ public class SQLAuditManagerConfig implements AuditManagerConfig { @JsonProperty - private final long auditHistoryMillis = 7 * 24 * 60 * 60 * 1000L; // 1 WEEK + private final Long auditHistoryMillis = 7 * 24 * 60 * 60 * 1000L; // 1 WEEK @JsonProperty - private final boolean includePayloadAsDimensionInMetric = false; + private final Boolean includePayloadAsDimensionInMetric = false; @JsonProperty @HumanReadableBytesRange( @@ -41,7 +41,7 @@ public class SQLAuditManagerConfig implements AuditManagerConfig private final HumanReadableBytes maxPayloadSizeBytes = HumanReadableBytes.valueOf(-1); @JsonProperty - private final boolean skipNullField = false; + private final Boolean skipNullField = false; public long getAuditHistoryMillis() { diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java index 5a9fc0f13b01..a08ed0e86d56 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java @@ -34,7 +34,7 @@ import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.segment.TestHelper; +import org.apache.druid.server.audit.AuditSerdeHelper; import org.apache.druid.server.audit.SQLAuditManager; import org.apache.druid.server.audit.SQLAuditManagerConfig; import org.apache.druid.server.coordinator.rules.IntervalLoadRule; @@ -65,7 +65,6 @@ public class SQLMetadataRuleManagerTest private AuditManager auditManager; private SQLMetadataSegmentPublisher publisher; private final ObjectMapper mapper = new DefaultObjectMapper(); - private final ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); @Before public void setUp() @@ -75,7 +74,7 @@ public void setUp() connector.createAuditTable(); auditManager = new SQLAuditManager( new SQLAuditManagerConfig(), - null, + new AuditSerdeHelper(new SQLAuditManagerConfig(), mapper, mapper), connector, Suppliers.ofInstance(tablesConfig), new NoopServiceEmitter(), @@ -87,7 +86,7 @@ public void setUp() ruleManager = new SQLMetadataRuleManager(mapper, managerConfig, tablesConfig, connector, auditManager); connector.createSegmentTable(); publisher = new SQLMetadataSegmentPublisher( - jsonMapper, + mapper, derbyConnectorRule.metadataTablesConfigSupplier().get(), connector ); diff --git a/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java b/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java index e599532b4fcd..e9f714273684 100644 --- a/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java @@ -54,7 +54,7 @@ public void testDefaultAuditConfig() } @Test - public void testDefaultLogAuditConfig() + public void testLogAuditConfigWithDefaults() { final Injector injector = createInjector(); final JsonConfigProvider provider = JsonConfigProvider.of( @@ -76,7 +76,7 @@ public void testDefaultLogAuditConfig() } @Test - public void testLogAuditConfig() + public void testLogAuditConfigWithOverrides() { final Injector injector = createInjector(); final JsonConfigProvider provider = JsonConfigProvider.of( @@ -89,7 +89,7 @@ public void testLogAuditConfig() props.setProperty("druid.audit.manager.logLevel", "WARN"); provider.inject(props, injector.getInstance(JsonConfigurator.class)); - + final AuditManagerConfig config = provider.get(); Assert.assertTrue(config instanceof LoggingAuditManagerConfig); @@ -99,6 +99,34 @@ public void testLogAuditConfig() Assert.assertEquals(AuditLogger.Level.WARN, logAuditConfig.getLogLevel()); } + @Test + public void testSqlAuditConfigWithOverrides() + { + final Injector injector = createInjector(); + final JsonConfigProvider provider = JsonConfigProvider.of( + CONFIG_BASE, + AuditManagerConfig.class + ); + + final Properties props = new Properties(); + props.setProperty("druid.audit.manager.type", "sql"); + props.setProperty("druid.audit.manager.skipNullField", "true"); + props.setProperty("druid.audit.manager.maxPayloadSizeBytes", "100"); + props.setProperty("druid.audit.manager.auditHistoryMillis", "1000"); + props.setProperty("druid.audit.manager.includePayloadAsDimensionInMetric", "true"); + + provider.inject(props, injector.getInstance(JsonConfigurator.class)); + + final AuditManagerConfig config = provider.get(); + Assert.assertTrue(config instanceof SQLAuditManagerConfig); + + final SQLAuditManagerConfig sqlAuditConfig = (SQLAuditManagerConfig) config; + Assert.assertTrue(sqlAuditConfig.isSkipNullField()); + Assert.assertTrue(sqlAuditConfig.isIncludePayloadAsDimensionInMetric()); + Assert.assertEquals(100, sqlAuditConfig.getMaxPayloadSizeBytes()); + Assert.assertEquals(1000L, sqlAuditConfig.getAuditHistoryMillis()); + } + private Injector createInjector() { return GuiceInjectors.makeStartupInjectorWithModules( From aae788cdc5d0f53bd54c84c93e63e28534e5e084 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Fri, 8 Dec 2023 23:25:44 +0530 Subject: [PATCH 19/29] Fix checks and tests --- .../CoordinatorBasicAuthenticatorResourceTest.java | 8 +++++--- .../org/apache/druid/java/util/common/logger/Logger.java | 5 ----- .../apache/druid/metadata/SqlSegmentsMetadataQuery.java | 4 ++-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java index 9dccab0f2c91..5dec8646573b 100644 --- a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java +++ b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/CoordinatorBasicAuthenticatorResourceTest.java @@ -389,20 +389,19 @@ public void testUserCredentials() response = resource.deleteUser(mockHttpRequest(), AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(200, response.getStatus()); -/* + response = resource.getUser(mockHttpRequestNoAudit(), AUTHENTICATOR_NAME, "druid"); Assert.assertEquals(400, response.getStatus()); Assert.assertEquals(errorMapWithMsg("User [druid] does not exist."), response.getEntity()); response = resource.updateUserCredentials( - mockHttpRequest(), + mockHttpRequestNoAudit(), AUTHENTICATOR_NAME, "druid", new BasicAuthenticatorCredentialUpdate("helloworld", null) ); Assert.assertEquals(400, response.getStatus()); Assert.assertEquals(errorMapWithMsg("User [druid] does not exist."), response.getEntity()); - */ } private HttpServletRequest mockHttpRequestNoAudit() @@ -427,6 +426,9 @@ private HttpServletRequest mockHttpRequest() new AuthenticationResult("id", "authorizer", "authBy", Collections.emptyMap()) ).once(); EasyMock.expect(req.getRemoteAddr()).andReturn("127.0.0.1").once(); + EasyMock.expect(req.getMethod()).andReturn("GET").once(); + EasyMock.expect(req.getRequestURI()).andReturn("uri").once(); + EasyMock.expect(req.getQueryString()).andReturn("a=b").once(); EasyMock.replay(req); return req; diff --git a/processing/src/main/java/org/apache/druid/java/util/common/logger/Logger.java b/processing/src/main/java/org/apache/druid/java/util/common/logger/Logger.java index 1639bf5378f8..59f2eb839b98 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/logger/Logger.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/logger/Logger.java @@ -220,11 +220,6 @@ public void error(Throwable t, String message, Object... formatArgs) logException(log::error, t, StringUtils.nonStrictFormat(message, formatArgs)); } - public void assertionError(String message, Object... formatArgs) - { - log.error("ASSERTION_ERROR: " + message, formatArgs); - } - public void debugSegments(@Nullable final Collection segments, @Nullable String preamble) { if (log.isDebugEnabled()) { diff --git a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java index 61fc919a8be5..13034f1fc842 100644 --- a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java +++ b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java @@ -451,8 +451,8 @@ private static int computeNumChangedSegments(List segmentIds, int[] segm for (int i = 0; i < segmentChanges.length; i++) { int numUpdatedRows = segmentChanges[i]; if (numUpdatedRows < 0) { - log.assertionError( - "Negative number of rows updated for segment id [%s]: %d", + log.error( + "ASSERTION_ERROR: Negative number of rows updated for segment id [%s]: %d", segmentIds.get(i), numUpdatedRows ); From ce7a17f5e3fdc6ec97895d11207a590e0bd1d566 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 12 Dec 2023 07:35:57 +0530 Subject: [PATCH 20/29] Set author header when using OverlordClient.runTask --- .../endpoint/BasicAuthenticatorResource.java | 2 +- .../overlord/http/OverlordResource.java | 23 ++++++-------- .../org/apache/druid/audit/AuditEntry.java | 16 +++++++--- .../org/apache/druid/audit/AuditInfo.java | 5 ---- .../org/apache/druid/audit/AuditManager.java | 7 ++++- .../org/apache/druid/audit/AuditInfoTest.java | 30 ++++++++++++------- .../config/JacksonConfigManagerTest.java | 1 + .../rpc/indexing/OverlordClientImpl.java | 4 ++- .../druid/server/audit/AuditLogger.java | 2 +- .../druid/server/audit/AuditSerdeHelper.java | 2 +- .../duty/KillCompactionConfig.java | 1 + .../cache/LookupCoordinatorManager.java | 2 +- .../metadata/SQLMetadataRuleManagerTest.java | 2 +- .../server/audit/SQLAuditManagerTest.java | 2 +- .../CoordinatorSimulationBuilder.java | 2 +- .../cache/LookupCoordinatorManagerTest.java | 15 +--------- 16 files changed, 59 insertions(+), 57 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java index 7881fcd5343a..8fb8a873c2c5 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java @@ -273,7 +273,7 @@ private void performAuditIfSuccess( Response response ) { - if (isSuccess(response)) { + if (updatedUser != null && isSuccess(response)) { auditManager.doAudit( AuditEntry.builder() .key(authenticatorName) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index 9129aba001df..125e1bf1295d 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -224,20 +224,15 @@ public Response taskPost( try { taskQueue.add(task); - // Do an audit only if this API was called by a user and not by internal services - final AuditInfo auditInfo = AuthorizationUtils.buildAuditInfo(req); - final String author = auditInfo.getAuthor(); - if (author != null && !author.isEmpty()) { - auditManager.doAudit( - AuditEntry.builder() - .key(task.getDataSource()) - .type("ingest.batch") - .request(AuthorizationUtils.buildRequestInfo("overlord", req)) - .payload(new TaskIdentifier(task.getId(), task.getGroupId(), task.getType())) - .auditInfo(auditInfo) - .build() - ); - } + auditManager.doAudit( + AuditEntry.builder() + .key(task.getDataSource()) + .type("task") + .request(AuthorizationUtils.buildRequestInfo("overlord", req)) + .payload(new TaskIdentifier(task.getId(), task.getGroupId(), task.getType())) + .auditInfo(AuthorizationUtils.buildAuditInfo(req)) + .build() + ); return Response.ok(ImmutableMap.of("task", task.getId())).build(); } diff --git a/processing/src/main/java/org/apache/druid/audit/AuditEntry.java b/processing/src/main/java/org/apache/druid/audit/AuditEntry.java index 696143b07b16..3a1789ae46db 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditEntry.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditEntry.java @@ -23,10 +23,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.base.Preconditions; +import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.DateTimes; import org.joda.time.DateTime; -import javax.annotation.Nullable; import java.util.Objects; /** @@ -47,7 +47,7 @@ public AuditEntry( @JsonProperty("key") String key, @JsonProperty("type") String type, @JsonProperty("auditInfo") AuditInfo authorInfo, - @JsonProperty("request") @Nullable RequestInfo requestInfo, + @JsonProperty("request") RequestInfo request, @JsonProperty("payload") Payload payload, @JsonProperty("auditTime") DateTime auditTime ) @@ -58,7 +58,7 @@ public AuditEntry( this.key = key; this.type = type; this.auditInfo = authorInfo; - this.request = requestInfo; + this.request = request; this.auditTime = auditTime == null ? DateTimes.nowUtc() : auditTime; this.payload = payload == null ? Payload.fromString("") : payload; } @@ -128,13 +128,14 @@ public boolean equals(Object o) && Objects.equals(this.key, that.key) && Objects.equals(this.type, that.type) && Objects.equals(this.auditInfo, that.auditInfo) + && Objects.equals(this.request, that.request) && Objects.equals(this.payload, that.payload); } @Override public int hashCode() { - return Objects.hash(key, type, auditInfo, payload, auditTime); + return Objects.hash(key, type, auditInfo, request, payload, auditTime); } public static class Builder @@ -197,6 +198,13 @@ public Builder auditTime(DateTime auditTime) public AuditEntry build() { + if (payload != null && serializedPayload != null) { + throw DruidException.defensive( + "Either payload[%s] or serializedPayload[%s] must be specified, not both.", + payload, serializedPayload + ); + } + return new AuditEntry(key, type, auditInfo, requestInfo, new Payload(serializedPayload, payload), auditTime); } } diff --git a/processing/src/main/java/org/apache/druid/audit/AuditInfo.java b/processing/src/main/java/org/apache/druid/audit/AuditInfo.java index 8487219511bd..f5e009c53837 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditInfo.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditInfo.java @@ -48,11 +48,6 @@ public AuditInfo( this.ip = ip; } - public AuditInfo(String author, String comment, String ip) - { - this(author, null, comment, ip); - } - @JsonProperty public String getAuthor() { diff --git a/processing/src/main/java/org/apache/druid/audit/AuditManager.java b/processing/src/main/java/org/apache/druid/audit/AuditManager.java index 039fc3d5bc9c..c336b390ff12 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditManager.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditManager.java @@ -30,9 +30,14 @@ public interface AuditManager { String X_DRUID_AUTHOR = "X-Druid-Author"; - String X_DRUID_COMMENT = "X-Druid-Comment"; + /** + * Value of header {@link #X_DRUID_AUTHOR} used by Druid services so that they + * can be distinguished from external requests. + */ + String AUTHOR_DRUID_SYSTEM = "druid_system"; + void doAudit(AuditEntry event); /** diff --git a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java index 38860efa9f0d..e481391c4547 100644 --- a/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java +++ b/processing/src/test/java/org/apache/druid/audit/AuditInfoTest.java @@ -34,8 +34,8 @@ public class AuditInfoTest @Test public void testAuditInfoEquality() { - final AuditInfo auditInfo1 = new AuditInfo("druid", "test equality", "127.0.0.1"); - final AuditInfo auditInfo2 = new AuditInfo("druid", "test equality", "127.0.0.1"); + final AuditInfo auditInfo1 = new AuditInfo("druid", "id", "test equality", "127.0.0.1"); + final AuditInfo auditInfo2 = new AuditInfo("druid", "id", "test equality", "127.0.0.1"); Assert.assertEquals(auditInfo1, auditInfo2); Assert.assertEquals(auditInfo1.hashCode(), auditInfo2.hashCode()); } @@ -43,32 +43,40 @@ public void testAuditInfoEquality() @Test public void testAuditInfoSerde() throws IOException { - final AuditInfo auditInfo = new AuditInfo("author", "comment", "ip"); + final AuditInfo auditInfo = new AuditInfo("author", null, "comment", "ip"); AuditInfo deserialized = mapper.readValue(mapper.writeValueAsString(auditInfo), AuditInfo.class); Assert.assertEquals(auditInfo, deserialized); final AuditInfo auditInfoWithIdentity = new AuditInfo("author", "identity", "comment", "ip"); deserialized = mapper.readValue(mapper.writeValueAsString(auditInfoWithIdentity), AuditInfo.class); Assert.assertEquals(auditInfoWithIdentity, deserialized); + + Assert.assertNotEquals(auditInfo, auditInfoWithIdentity); } @Test(timeout = 60_000L) public void testAuditEntrySerde() throws IOException { - AuditEntry entry = new AuditEntry( + final AuditEntry original = new AuditEntry( "testKey", "testType", - new AuditInfo( - "testAuthor", - "testComment", - "127.0.0.1" - ), + new AuditInfo("testAuthor", "testIdentity", "testComment", "127.0.0.1"), new RequestInfo("overlord", "GET", "/segments", "?abc=1"), AuditEntry.Payload.fromString("testPayload"), DateTimes.of("2013-01-01T00:00:00Z") ); - AuditEntry serde = mapper.readValue(mapper.writeValueAsString(entry), AuditEntry.class); - Assert.assertEquals(entry, serde); + AuditEntry deserialized = mapper.readValue(mapper.writeValueAsString(original), AuditEntry.class); + Assert.assertEquals(original, deserialized); + } + + @Test + public void testAuditEntrySerdeIsBackwardsCompatible() throws IOException + { + final String json = "{\"key\": \"a\", \"type\": \"b\", \"auditInfo\": {}, \"payload\":\"Truncated\"}"; + AuditEntry entry = mapper.readValue(json, AuditEntry.class); + Assert.assertEquals("a", entry.getKey()); + Assert.assertEquals("b", entry.getType()); + Assert.assertEquals(AuditEntry.Payload.fromString("Truncated"), entry.getPayload()); } @Test diff --git a/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java b/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java index 85c548157c26..0d0393459445 100644 --- a/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java +++ b/processing/src/test/java/org/apache/druid/common/config/JacksonConfigManagerTest.java @@ -65,6 +65,7 @@ public void testSet() TestConfig val = new TestConfig("version", "string", 3); AuditInfo auditInfo = new AuditInfo( "testAuthor", + "testIdentity", "testComment", "127.0.0.1" ); diff --git a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java index d7fab4b75fa2..676848f9961e 100644 --- a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java +++ b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import org.apache.druid.audit.AuditManager; import org.apache.druid.client.JsonParserIterator; import org.apache.druid.client.indexing.IndexingTotalWorkerCapacityInfo; import org.apache.druid.client.indexing.IndexingWorkerInfo; @@ -96,7 +97,8 @@ public ListenableFuture runTask(final String taskId, final Object taskObje return FutureUtils.transform( client.asyncRequest( new RequestBuilder(HttpMethod.POST, "/druid/indexer/v1/task") - .jsonContent(jsonMapper, taskObject), + .jsonContent(jsonMapper, taskObject) + .header(AuditManager.X_DRUID_AUTHOR, AuditManager.AUTHOR_DRUID_SYSTEM), new BytesFullResponseHandler() ), holder -> { diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java index 8aa2213180fe..f99636619455 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java @@ -30,7 +30,7 @@ public enum Level } private static final String MSG_FORMAT - = "User[%s], identity[%s], IP[%s] performed action[%s] on key[%s] with comment[%s]. Request[%s], payload[%s]."; + = "User[%s], identity[%s], IP[%s] performed action[%s] on key[%s] with comment[%s], request[%s], payload[%s]."; private final Level level; private final Logger logger = new Logger(AuditLogger.class); diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java index 76a02d05ea2a..1450de66e30f 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java @@ -44,7 +44,7 @@ public class AuditSerdeHelper private static final String PAYLOAD_TRUNCATED_MSG = "Payload truncated as it exceeds 'druid.audit.manager.maxPayloadSizeBytes'"; private static final String SERIALIZE_ERROR_MSG = - "Error serializing payload"; + "Error serializing payload. Check logs for details."; private static final Logger log = new Logger(AuditSerdeHelper.class); private final ObjectMapper jsonMapper; diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/KillCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/KillCompactionConfig.java index e039632d30df..304b54e7cf84 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/KillCompactionConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/KillCompactionConfig.java @@ -133,6 +133,7 @@ private int tryDeleteCompactionConfigs() throws RetryableException return updated; }, new AuditInfo( + "KillCompactionConfig", "KillCompactionConfig", "CoordinatorDuty for automatic deletion of compaction config", "" diff --git a/server/src/main/java/org/apache/druid/server/lookup/cache/LookupCoordinatorManager.java b/server/src/main/java/org/apache/druid/server/lookup/cache/LookupCoordinatorManager.java index 2662de6b981b..fcb1a81fd5fe 100644 --- a/server/src/main/java/org/apache/druid/server/lookup/cache/LookupCoordinatorManager.java +++ b/server/src/main/java/org/apache/druid/server/lookup/cache/LookupCoordinatorManager.java @@ -519,7 +519,7 @@ private void initializeLookupsConfigWatcher() configManager.set( LOOKUP_CONFIG_KEY, converted, - new AuditInfo("autoConversion", "autoConversion", "127.0.0.1") + new AuditInfo("autoConversion", "autoConversion", "autoConversion", "127.0.0.1") ); } } diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java index a08ed0e86d56..059c6e175c57 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java @@ -367,7 +367,7 @@ private void dropTable(final String tableName) private AuditInfo createAuditInfo(String comment) { - return new AuditInfo("test", comment, "127.0.0.1"); + return new AuditInfo("test", "id", comment, "127.0.0.1"); } } diff --git a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java index 31e9d85ee493..fd76a90c5cfb 100644 --- a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java @@ -299,7 +299,7 @@ public boolean isSkipNullField() } ); - AuditInfo auditInfo = new AuditInfo("testAuthor", "testComment", "127.0.0.1"); + AuditInfo auditInfo = new AuditInfo("testAuthor", "testIdentity", "testComment", "127.0.0.1"); final Map payloadMap = new TreeMap<>(); payloadMap.put("version", "x"); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java index b5174c8f02f3..f341ebf5df99 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java @@ -335,7 +335,7 @@ public void setRetentionRules(String datasource, Rule... rules) env.ruleManager.overrideRule( datasource, Arrays.asList(rules), - new AuditInfo("sim", "sim", "localhost") + new AuditInfo("sim", "sim", "sim", "localhost") ); } diff --git a/server/src/test/java/org/apache/druid/server/lookup/cache/LookupCoordinatorManagerTest.java b/server/src/test/java/org/apache/druid/server/lookup/cache/LookupCoordinatorManagerTest.java index 94699ae34868..c94d8e436683 100644 --- a/server/src/test/java/org/apache/druid/server/lookup/cache/LookupCoordinatorManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/lookup/cache/LookupCoordinatorManagerTest.java @@ -104,6 +104,7 @@ public class LookupCoordinatorManagerTest Collections.emptySet() ); + private final AuditInfo auditInfo = new AuditInfo("author", "identify", "comment", "127.0.0.1"); private static StubServiceEmitter SERVICE_EMITTER; @BeforeClass @@ -522,8 +523,6 @@ public Map> getKnownLook } }; manager.start(); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); - Assert.assertThrows(ISE.class, () -> manager.updateLookups(TIERED_LOOKUP_MAP_V0, auditInfo)); } @@ -546,7 +545,6 @@ public Map> getKnownLook }; manager.start(); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); EasyMock.reset(configManager); EasyMock.expect( configManager.set( @@ -578,7 +576,6 @@ public Map> getKnownLook }; manager.start(); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); EasyMock.reset(configManager); EasyMock.expect(configManager.set( EasyMock.eq(LookupCoordinatorManager.LOOKUP_CONFIG_KEY), @@ -598,7 +595,6 @@ public void testUpdateLookupsAddsNewLookup() ImmutableMap.of("prop", "old") ); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); final LookupCoordinatorManager manager = new LookupCoordinatorManager( client, druidNodeDiscoveryProvider, @@ -660,7 +656,6 @@ public void testUpdateLookupsOnlyUpdatesToTier() "v0", ImmutableMap.of("prop", "old") ); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); final LookupCoordinatorManager manager = new LookupCoordinatorManager( client, druidNodeDiscoveryProvider, @@ -731,7 +726,6 @@ public Map> getKnownLook } }; manager.start(); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); EasyMock.reset(configManager); EasyMock.expect(configManager.set( EasyMock.eq(LookupCoordinatorManager.LOOKUP_CONFIG_KEY), @@ -761,7 +755,6 @@ public Map> getKnownLook } }; manager.start(); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); try { manager.updateLookups(TIERED_LOOKUP_MAP_V0, auditInfo); @@ -779,7 +772,6 @@ public void testUpdateLookupsAddsNewTier() ImmutableMap.of("prop", "old") ); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); final LookupCoordinatorManager manager = new LookupCoordinatorManager( client, druidNodeDiscoveryProvider, @@ -853,7 +845,6 @@ public Map> getKnownLook } }; manager.start(); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); EasyMock.reset(configManager); EasyMock.expect( configManager.set( @@ -899,7 +890,6 @@ public Map> getKnownLook } }; manager.start(); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); EasyMock.reset(configManager); EasyMock.expect( configManager.set( @@ -944,7 +934,6 @@ public Map> getKnownLook } }; manager.start(); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); EasyMock.reset(configManager); EasyMock.expect( configManager.set( @@ -985,7 +974,6 @@ public Map> getKnownLook } }; manager.start(); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); Assert.assertFalse(manager.deleteLookup(LOOKUP_TIER, "foo", auditInfo)); } @@ -1007,7 +995,6 @@ public Map> getKnownLook } }; manager.start(); - final AuditInfo auditInfo = new AuditInfo("author", "comment", "localhost"); Assert.assertFalse(manager.deleteLookup(LOOKUP_TIER, "foo", auditInfo)); } From 5853e2a45031a13da4f83b4b742f141a26231af0 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 12 Dec 2023 07:36:31 +0530 Subject: [PATCH 21/29] Use constructors in configs for better null-handling --- .../audit/LoggingAuditManagerConfig.java | 20 ++++++++-- .../server/audit/SQLAuditManagerConfig.java | 24 +++++++++-- .../metadata/SQLMetadataRuleManagerTest.java | 6 ++- .../server/audit/SQLAuditManagerTest.java | 40 +++---------------- 4 files changed, 47 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java index 6140936675c0..0ea37aacca8c 100644 --- a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java @@ -19,24 +19,38 @@ package org.apache.druid.server.audit; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.common.config.Configs; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.HumanReadableBytesRange; public class LoggingAuditManagerConfig implements AuditManagerConfig { @JsonProperty - private final AuditLogger.Level logLevel = AuditLogger.Level.INFO; + private final AuditLogger.Level logLevel; @JsonProperty @HumanReadableBytesRange( min = -1, message = "maxPayloadSizeBytes must either be -1 (for disabling the check) or a non negative number" ) - private final HumanReadableBytes maxPayloadSizeBytes = HumanReadableBytes.valueOf(-1); + private final HumanReadableBytes maxPayloadSizeBytes; @JsonProperty - private final Boolean skipNullField = false; + private final boolean skipNullField; + + @JsonCreator + public LoggingAuditManagerConfig( + @JsonProperty("logLevel") AuditLogger.Level logLevel, + @JsonProperty("maxPayloadSizeBytes") HumanReadableBytes maxPayloadSizeBytes, + @JsonProperty("skipNullField") Boolean skipNullField + ) + { + this.logLevel = Configs.valueOrDefault(logLevel, AuditLogger.Level.INFO); + this.maxPayloadSizeBytes = Configs.valueOrDefault(maxPayloadSizeBytes, HumanReadableBytes.valueOf(-1)); + this.skipNullField = Configs.valueOrDefault(skipNullField, false); + } @Override public boolean isSkipNullField() diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java index 290072965732..898511132ada 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java @@ -19,7 +19,9 @@ package org.apache.druid.server.audit; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.common.config.Configs; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.HumanReadableBytesRange; @@ -28,20 +30,34 @@ public class SQLAuditManagerConfig implements AuditManagerConfig { @JsonProperty - private final Long auditHistoryMillis = 7 * 24 * 60 * 60 * 1000L; // 1 WEEK + private final long auditHistoryMillis; @JsonProperty - private final Boolean includePayloadAsDimensionInMetric = false; + private final boolean includePayloadAsDimensionInMetric; @JsonProperty @HumanReadableBytesRange( min = -1, message = "maxPayloadSizeBytes must either be -1 (for disabling the check) or a non negative number" ) - private final HumanReadableBytes maxPayloadSizeBytes = HumanReadableBytes.valueOf(-1); + private final HumanReadableBytes maxPayloadSizeBytes; @JsonProperty - private final Boolean skipNullField = false; + private final boolean skipNullField; + + @JsonCreator + public SQLAuditManagerConfig( + @JsonProperty("maxPayloadSizeBytes") HumanReadableBytes maxPayloadSizeBytes, + @JsonProperty("skipNullField") Boolean skipNullField, + @JsonProperty("auditHistoryMillis") Long auditHistoryMillis, + @JsonProperty("includePayloadAsDimensionInMetric") Boolean includePayloadAsDimensionInMetric + ) + { + this.maxPayloadSizeBytes = Configs.valueOrDefault(maxPayloadSizeBytes, HumanReadableBytes.valueOf(-1)); + this.skipNullField = Configs.valueOrDefault(skipNullField, false); + this.auditHistoryMillis = Configs.valueOrDefault(auditHistoryMillis, 7 * 24 * 60 * 60 * 1000L); + this.includePayloadAsDimensionInMetric = Configs.valueOrDefault(includePayloadAsDimensionInMetric, false); + } public long getAuditHistoryMillis() { diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java index 059c6e175c57..94c1c15c1541 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java @@ -72,9 +72,11 @@ public void setUp() connector = derbyConnectorRule.getConnector(); tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); connector.createAuditTable(); + + final SQLAuditManagerConfig auditManagerConfig = new SQLAuditManagerConfig(null, null, null, null); auditManager = new SQLAuditManager( - new SQLAuditManagerConfig(), - new AuditSerdeHelper(new SQLAuditManagerConfig(), mapper, mapper), + auditManagerConfig, + new AuditSerdeHelper(auditManagerConfig, mapper, mapper), connector, Suppliers.ofInstance(tablesConfig), new NoopServiceEmitter(), diff --git a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java index fd76a90c5cfb..176996250474 100644 --- a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java @@ -25,6 +25,7 @@ import org.apache.druid.audit.AuditInfo; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; @@ -65,7 +66,7 @@ public void setUp() serviceEmitter = new StubServiceEmitter("audit-test", "localhost"); connector = derbyConnectorRule.getConnector(); connector.createAuditTable(); - auditManager = createAuditManager(new SQLAuditManagerConfig()); + auditManager = createAuditManager(new SQLAuditManagerConfig(null, null, null, null)); } private SQLAuditManager createAuditManager(SQLAuditManagerConfig config) @@ -84,14 +85,7 @@ private SQLAuditManager createAuditManager(SQLAuditManagerConfig config) public void testAuditMetricEventWithPayload() throws IOException { SQLAuditManager auditManager = createAuditManager( - new SQLAuditManagerConfig() - { - @Override - public boolean isIncludePayloadAsDimensionInMetric() - { - return true; - } - } + new SQLAuditManagerConfig(null, null, null, true) ); final AuditEntry entry = createAuditEntry("testKey", "testType", DateTimes.nowUtc()); @@ -236,16 +230,8 @@ public void testFetchAuditHistoryLimitZero() @Test(timeout = 60_000L) public void testCreateAuditEntryWithPayloadOverSkipPayloadLimit() throws IOException { - final int maxPayloadSize = 10; final SQLAuditManager auditManager = createAuditManager( - new SQLAuditManagerConfig() - { - @Override - public long getMaxPayloadSizeBytes() - { - return maxPayloadSize; - } - } + new SQLAuditManagerConfig(HumanReadableBytes.valueOf(10), null, null, null) ); final AuditEntry entry = createAuditEntry("key", "type", DateTimes.nowUtc()); @@ -267,14 +253,7 @@ public long getMaxPayloadSizeBytes() public void testCreateAuditEntryWithPayloadUnderSkipPayloadLimit() throws IOException { SQLAuditManager auditManager = createAuditManager( - new SQLAuditManagerConfig() - { - @Override - public long getMaxPayloadSizeBytes() - { - return 500; - } - } + new SQLAuditManagerConfig(HumanReadableBytes.valueOf(500), null, null, null) ); final AuditEntry entry = createAuditEntry("key", "type", DateTimes.nowUtc()); @@ -289,14 +268,7 @@ public long getMaxPayloadSizeBytes() public void testCreateAuditEntryWithSkipNullsInPayload() throws IOException { final SQLAuditManager auditManagerSkipNull = createAuditManager( - new SQLAuditManagerConfig() - { - @Override - public boolean isSkipNullField() - { - return true; - } - } + new SQLAuditManagerConfig(null, true, null, null) ); AuditInfo auditInfo = new AuditInfo("testAuthor", "testIdentity", "testComment", "127.0.0.1"); From d0efb3b25f652c0887f07bc72ddd56f6720cb43d Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 12 Dec 2023 07:46:41 +0530 Subject: [PATCH 22/29] Add config druid.audit.manager.auditSystemRequests --- .../indexing/overlord/http/OverlordResource.java | 1 - .../apache/druid/server/audit/AuditManagerConfig.java | 2 ++ .../druid/server/audit/LoggingAuditManagerConfig.java | 11 +++++++++++ .../druid/server/audit/SQLAuditManagerConfig.java | 10 ++++++++++ .../druid/metadata/SQLMetadataRuleManagerTest.java | 3 +-- .../druid/server/audit/AuditManagerConfigTest.java | 4 ++++ .../druid/server/audit/SQLAuditManagerTest.java | 10 +++++----- 7 files changed, 33 insertions(+), 8 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index 125e1bf1295d..3e75b50dbbe8 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -29,7 +29,6 @@ import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; import org.apache.druid.audit.AuditEntry; -import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.client.indexing.ClientTaskQuery; import org.apache.druid.common.config.ConfigManager.SetResult; diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/AuditManagerConfig.java index 2cb5acd3c82f..6f174da2db12 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditManagerConfig.java @@ -33,4 +33,6 @@ public interface AuditManagerConfig boolean isSkipNullField(); long getMaxPayloadSizeBytes(); + + boolean isAuditSystemRequests(); } diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java index 0ea37aacca8c..a06c778b412f 100644 --- a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java @@ -40,14 +40,19 @@ public class LoggingAuditManagerConfig implements AuditManagerConfig @JsonProperty private final boolean skipNullField; + @JsonProperty + private final boolean auditSystemRequests; + @JsonCreator public LoggingAuditManagerConfig( @JsonProperty("logLevel") AuditLogger.Level logLevel, + @JsonProperty("auditSystemRequests") Boolean auditSystemRequests, @JsonProperty("maxPayloadSizeBytes") HumanReadableBytes maxPayloadSizeBytes, @JsonProperty("skipNullField") Boolean skipNullField ) { this.logLevel = Configs.valueOrDefault(logLevel, AuditLogger.Level.INFO); + this.auditSystemRequests = Configs.valueOrDefault(auditSystemRequests, false); this.maxPayloadSizeBytes = Configs.valueOrDefault(maxPayloadSizeBytes, HumanReadableBytes.valueOf(-1)); this.skipNullField = Configs.valueOrDefault(skipNullField, false); } @@ -64,6 +69,12 @@ public long getMaxPayloadSizeBytes() return maxPayloadSizeBytes.getBytes(); } + @Override + public boolean isAuditSystemRequests() + { + return auditSystemRequests; + } + public AuditLogger.Level getLogLevel() { return logLevel; diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java index 898511132ada..c9a0bd7a70d1 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java @@ -45,14 +45,19 @@ public class SQLAuditManagerConfig implements AuditManagerConfig @JsonProperty private final boolean skipNullField; + @JsonProperty + private final boolean auditSystemRequests; + @JsonCreator public SQLAuditManagerConfig( + @JsonProperty("auditSystemRequests") Boolean auditSystemRequests, @JsonProperty("maxPayloadSizeBytes") HumanReadableBytes maxPayloadSizeBytes, @JsonProperty("skipNullField") Boolean skipNullField, @JsonProperty("auditHistoryMillis") Long auditHistoryMillis, @JsonProperty("includePayloadAsDimensionInMetric") Boolean includePayloadAsDimensionInMetric ) { + this.auditSystemRequests = Configs.valueOrDefault(auditSystemRequests, false); this.maxPayloadSizeBytes = Configs.valueOrDefault(maxPayloadSizeBytes, HumanReadableBytes.valueOf(-1)); this.skipNullField = Configs.valueOrDefault(skipNullField, false); this.auditHistoryMillis = Configs.valueOrDefault(auditHistoryMillis, 7 * 24 * 60 * 60 * 1000L); @@ -81,4 +86,9 @@ public boolean isSkipNullField() return skipNullField; } + @Override + public boolean isAuditSystemRequests() + { + return auditSystemRequests; + } } diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java index 94c1c15c1541..17442cf2ca04 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java @@ -19,7 +19,6 @@ package org.apache.druid.metadata; - import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Suppliers; @@ -73,7 +72,7 @@ public void setUp() tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); connector.createAuditTable(); - final SQLAuditManagerConfig auditManagerConfig = new SQLAuditManagerConfig(null, null, null, null); + final SQLAuditManagerConfig auditManagerConfig = new SQLAuditManagerConfig(null, null, null, null, null); auditManager = new SQLAuditManager( auditManagerConfig, new AuditSerdeHelper(auditManagerConfig, mapper, mapper), diff --git a/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java b/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java index e9f714273684..1ae61a79f6e0 100644 --- a/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java @@ -47,6 +47,7 @@ public void testDefaultAuditConfig() Assert.assertTrue(config instanceof SQLAuditManagerConfig); final SQLAuditManagerConfig sqlAuditConfig = (SQLAuditManagerConfig) config; + Assert.assertFalse(sqlAuditConfig.isAuditSystemRequests()); Assert.assertFalse(sqlAuditConfig.isSkipNullField()); Assert.assertFalse(sqlAuditConfig.isIncludePayloadAsDimensionInMetric()); Assert.assertEquals(-1, sqlAuditConfig.getMaxPayloadSizeBytes()); @@ -70,6 +71,7 @@ public void testLogAuditConfigWithDefaults() Assert.assertTrue(config instanceof LoggingAuditManagerConfig); final LoggingAuditManagerConfig logAuditConfig = (LoggingAuditManagerConfig) config; + Assert.assertFalse(logAuditConfig.isAuditSystemRequests()); Assert.assertFalse(logAuditConfig.isSkipNullField()); Assert.assertEquals(-1, logAuditConfig.getMaxPayloadSizeBytes()); Assert.assertEquals(AuditLogger.Level.INFO, logAuditConfig.getLogLevel()); @@ -87,6 +89,7 @@ public void testLogAuditConfigWithOverrides() final Properties props = new Properties(); props.setProperty("druid.audit.manager.type", "log"); props.setProperty("druid.audit.manager.logLevel", "WARN"); + props.setProperty("druid.audit.manager.auditSystemRequests", "true"); provider.inject(props, injector.getInstance(JsonConfigurator.class)); @@ -94,6 +97,7 @@ public void testLogAuditConfigWithOverrides() Assert.assertTrue(config instanceof LoggingAuditManagerConfig); final LoggingAuditManagerConfig logAuditConfig = (LoggingAuditManagerConfig) config; + Assert.assertTrue(logAuditConfig.isAuditSystemRequests()); Assert.assertFalse(logAuditConfig.isSkipNullField()); Assert.assertEquals(-1, logAuditConfig.getMaxPayloadSizeBytes()); Assert.assertEquals(AuditLogger.Level.WARN, logAuditConfig.getLogLevel()); diff --git a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java index 176996250474..8e15aa4e2ce0 100644 --- a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java @@ -66,7 +66,7 @@ public void setUp() serviceEmitter = new StubServiceEmitter("audit-test", "localhost"); connector = derbyConnectorRule.getConnector(); connector.createAuditTable(); - auditManager = createAuditManager(new SQLAuditManagerConfig(null, null, null, null)); + auditManager = createAuditManager(new SQLAuditManagerConfig(null, null, null, null, null)); } private SQLAuditManager createAuditManager(SQLAuditManagerConfig config) @@ -85,7 +85,7 @@ private SQLAuditManager createAuditManager(SQLAuditManagerConfig config) public void testAuditMetricEventWithPayload() throws IOException { SQLAuditManager auditManager = createAuditManager( - new SQLAuditManagerConfig(null, null, null, true) + new SQLAuditManagerConfig(null, null, null, null, true) ); final AuditEntry entry = createAuditEntry("testKey", "testType", DateTimes.nowUtc()); @@ -231,7 +231,7 @@ public void testFetchAuditHistoryLimitZero() public void testCreateAuditEntryWithPayloadOverSkipPayloadLimit() throws IOException { final SQLAuditManager auditManager = createAuditManager( - new SQLAuditManagerConfig(HumanReadableBytes.valueOf(10), null, null, null) + new SQLAuditManagerConfig(null, HumanReadableBytes.valueOf(10), null, null, null) ); final AuditEntry entry = createAuditEntry("key", "type", DateTimes.nowUtc()); @@ -253,7 +253,7 @@ public void testCreateAuditEntryWithPayloadOverSkipPayloadLimit() throws IOExcep public void testCreateAuditEntryWithPayloadUnderSkipPayloadLimit() throws IOException { SQLAuditManager auditManager = createAuditManager( - new SQLAuditManagerConfig(HumanReadableBytes.valueOf(500), null, null, null) + new SQLAuditManagerConfig(null, HumanReadableBytes.valueOf(500), null, null, null) ); final AuditEntry entry = createAuditEntry("key", "type", DateTimes.nowUtc()); @@ -268,7 +268,7 @@ public void testCreateAuditEntryWithPayloadUnderSkipPayloadLimit() throws IOExce public void testCreateAuditEntryWithSkipNullsInPayload() throws IOException { final SQLAuditManager auditManagerSkipNull = createAuditManager( - new SQLAuditManagerConfig(null, true, null, null) + new SQLAuditManagerConfig(null, null, true, null, null) ); AuditInfo auditInfo = new AuditInfo("testAuthor", "testIdentity", "testComment", "127.0.0.1"); From ed3eceea59fe9cc094b1942e53a80dcc7484ae59 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 12 Dec 2023 07:54:34 +0530 Subject: [PATCH 23/29] Use config auditSystemRequests --- .../java/org/apache/druid/audit/AuditManager.java | 8 ++++++++ .../druid/server/audit/LoggingAuditManager.java | 11 +++++++---- .../apache/druid/server/audit/SQLAuditManager.java | 4 ++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/audit/AuditManager.java b/processing/src/main/java/org/apache/druid/audit/AuditManager.java index c336b390ff12..95d81dd49a0b 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditManager.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditManager.java @@ -38,6 +38,14 @@ public interface AuditManager */ String AUTHOR_DRUID_SYSTEM = "druid_system"; + /** + * @return true if the audited event was initiated by the Druid system itself. + */ + default boolean isSystemRequest(AuditInfo auditInfo) + { + return AUTHOR_DRUID_SYSTEM.equals(auditInfo.getAuthor()); + } + void doAudit(AuditEntry event); /** diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java index 22ad94f80a7c..68840d7a3ca0 100644 --- a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java @@ -35,6 +35,7 @@ public class LoggingAuditManager implements AuditManager { private final AuditLogger auditLogger; + private final LoggingAuditManagerConfig managerConfig; private final AuditSerdeHelper serdeHelper; @Inject @@ -43,19 +44,21 @@ public LoggingAuditManager( AuditSerdeHelper serdeHelper ) { - this.serdeHelper = serdeHelper; if (!(config instanceof LoggingAuditManagerConfig)) { throw DruidException.defensive("Config[%s] is not an instance of LoggingAuditManagerConfig", config); } - LoggingAuditManagerConfig logAuditConfig = (LoggingAuditManagerConfig) config; - this.auditLogger = new AuditLogger(logAuditConfig.getLogLevel()); + this.managerConfig = (LoggingAuditManagerConfig) config; + this.serdeHelper = serdeHelper; + this.auditLogger = new AuditLogger(managerConfig.getLogLevel()); } @Override public void doAudit(AuditEntry entry) { - auditLogger.log(serdeHelper.processAuditEntry(entry)); + if (managerConfig.isAuditSystemRequests() || !isSystemRequest(entry.getAuditInfo())) { + auditLogger.log(serdeHelper.processAuditEntry(entry)); + } } @Override diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java index 0ab855a0e2d3..cabbfb9251b6 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java @@ -136,6 +136,10 @@ private ServiceMetricEvent.Builder createMetricEventBuilder(AuditEntry entry) @Override public void doAudit(AuditEntry event, Handle handle) throws IOException { + if (isSystemRequest(event.getAuditInfo()) && !config.isAuditSystemRequests()) { + return; + } + emitter.emit(createMetricEventBuilder(event).setMetric("config/audit", 1)); final AuditEntry record = serdeHelper.processAuditEntry(event); From f42d9a69a88be629dd5eeba143d8868251591620 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Thu, 14 Dec 2023 21:31:40 +0530 Subject: [PATCH 24/29] More audits, attempt to debug failing IT --- .../endpoint/BasicAuthenticatorResource.java | 9 +- .../endpoint/BasicAuthorizerResource.java | 115 +++++++++++++++-- ...oordinatorBasicAuthorizerResourceTest.java | 10 +- .../endpoint/BasicAuthorizerResourceTest.java | 2 +- .../ITBasicAuthConfigurationTest.java | 120 +++++++----------- .../druid/server/audit/AuditLogger.java | 3 +- 6 files changed, 162 insertions(+), 97 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java index 8fb8a873c2c5..d77358dd7d70 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java @@ -160,7 +160,7 @@ public Response createUser( authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.createUser(authenticatorName, userName); - performAuditIfSuccess(authenticatorName, "basicAuth.createUser", userName, req, response); + performAuditIfSuccess(authenticatorName, userName, req, response); return response; } @@ -186,7 +186,7 @@ public Response deleteUser( { authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.deleteUser(authenticatorName, userName); - performAuditIfSuccess(authenticatorName, "basicAuth.deleteUser", userName, req, response); + performAuditIfSuccess(authenticatorName, userName, req, response); return response; } @@ -213,7 +213,7 @@ public Response updateUserCredentials( { authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.updateUserCredentials(authenticatorName, userName, update); - performAuditIfSuccess(authenticatorName, "basicAuth.updateUserCreds", userName, req, response); + performAuditIfSuccess(authenticatorName, userName, req, response); return response; } @@ -267,7 +267,6 @@ private boolean isSuccess(Response response) private void performAuditIfSuccess( String authenticatorName, - String action, String updatedUser, HttpServletRequest request, Response response @@ -277,7 +276,7 @@ private void performAuditIfSuccess( auditManager.doAudit( AuditEntry.builder() .key(authenticatorName) - .type(action) + .type("basicAuthentication") .auditInfo(AuthorizationUtils.buildAuditInfo(request)) .request(AuthorizationUtils.buildRequestInfo("coordinator", request)) .payload(Collections.singletonMap("username", updatedUser)) diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResource.java index cb8a9fa2a240..c295228149b9 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResource.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes; import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; +import org.apache.druid.audit.AuditEntry; +import org.apache.druid.audit.AuditManager; import org.apache.druid.guice.LazySingleton; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.security.basic.BasicSecurityResourceFilter; import org.apache.druid.security.basic.authorization.entity.BasicAuthorizerGroupMapping; import org.apache.druid.server.security.AuthValidator; +import org.apache.druid.server.security.AuthorizationUtils; import org.apache.druid.server.security.ResourceAction; import javax.servlet.http.HttpServletRequest; @@ -48,15 +52,18 @@ public class BasicAuthorizerResource { private final BasicAuthorizerResourceHandler resourceHandler; private final AuthValidator authValidator; + private final AuditManager auditManager; @Inject public BasicAuthorizerResource( BasicAuthorizerResourceHandler resourceHandler, - AuthValidator authValidator + AuthValidator authValidator, + AuditManager auditManager ) { this.resourceHandler = resourceHandler; this.authValidator = authValidator; + this.auditManager = auditManager; } /** @@ -198,7 +205,11 @@ public Response createUser( ) { authValidator.validateAuthorizerName(authorizerName); - return resourceHandler.createUser(authorizerName, userName); + + final Response response = resourceHandler.createUser(authorizerName, userName); + performAuditIfSuccess(authorizerName, req, response, "Create user[%s]", userName); + + return response; } /** @@ -221,7 +232,11 @@ public Response deleteUser( ) { authValidator.validateAuthorizerName(authorizerName); - return resourceHandler.deleteUser(authorizerName, userName); + + final Response response = resourceHandler.deleteUser(authorizerName, userName); + performAuditIfSuccess(authorizerName, req, response, "Delete user[%s]", userName); + + return response; } /** @@ -245,10 +260,21 @@ public Response createGroupMapping( ) { authValidator.validateAuthorizerName(authorizerName); - return resourceHandler.createGroupMapping( + final Response response = resourceHandler.createGroupMapping( authorizerName, new BasicAuthorizerGroupMapping(groupMappingName, groupMapping.getGroupPattern(), groupMapping.getRoles()) ); + performAuditIfSuccess( + authorizerName, + req, + response, + "Create groupMapping[%s] with pattern[%s], roles[%s]", + groupMappingName, + groupMapping.getGroupPattern(), + groupMapping.getRoles() + ); + + return response; } /** @@ -271,7 +297,11 @@ public Response deleteGroupMapping( ) { authValidator.validateAuthorizerName(authorizerName); - return resourceHandler.deleteGroupMapping(authorizerName, groupMappingName); + + final Response response = resourceHandler.deleteGroupMapping(authorizerName, groupMappingName); + performAuditIfSuccess(authorizerName, req, response, "Delete groupMapping[%s]", groupMappingName); + + return response; } /** @@ -338,7 +368,11 @@ public Response createRole( ) { authValidator.validateAuthorizerName(authorizerName); - return resourceHandler.createRole(authorizerName, roleName); + + final Response response = resourceHandler.createRole(authorizerName, roleName); + performAuditIfSuccess(authorizerName, req, response, "Create role[%s]", roleName); + + return response; } /** @@ -361,7 +395,11 @@ public Response deleteRole( ) { authValidator.validateAuthorizerName(authorizerName); - return resourceHandler.deleteRole(authorizerName, roleName); + + final Response response = resourceHandler.deleteRole(authorizerName, roleName); + performAuditIfSuccess(authorizerName, req, response, "Delete role[%s]", roleName); + + return response; } /** @@ -386,7 +424,11 @@ public Response assignRoleToUser( ) { authValidator.validateAuthorizerName(authorizerName); - return resourceHandler.assignRoleToUser(authorizerName, userName, roleName); + + final Response response = resourceHandler.assignRoleToUser(authorizerName, userName, roleName); + performAuditIfSuccess(authorizerName, req, response, "Assign role[%s] to user[%s]", roleName, userName); + + return response; } /** @@ -411,7 +453,11 @@ public Response unassignRoleFromUser( ) { authValidator.validateAuthorizerName(authorizerName); - return resourceHandler.unassignRoleFromUser(authorizerName, userName, roleName); + + final Response response = resourceHandler.unassignRoleFromUser(authorizerName, userName, roleName); + performAuditIfSuccess(authorizerName, req, response, "Unassign role[%s] from user[%s]", roleName, userName); + + return response; } /** @@ -436,7 +482,12 @@ public Response assignRoleToGroupMapping( ) { authValidator.validateAuthorizerName(authorizerName); - return resourceHandler.assignRoleToGroupMapping(authorizerName, groupMappingName, roleName); + final Response response = resourceHandler.assignRoleToGroupMapping(authorizerName, groupMappingName, roleName); + + String msgFormat = "Assign role[%s] to groupMapping[%s]"; + performAuditIfSuccess(authorizerName, req, response, msgFormat, roleName, groupMappingName); + + return response; } /** @@ -461,7 +512,12 @@ public Response unassignRoleFromGroupMapping( ) { authValidator.validateAuthorizerName(authorizerName); - return resourceHandler.unassignRoleFromGroupMapping(authorizerName, groupMappingName, roleName); + + final Response response = resourceHandler.unassignRoleFromGroupMapping(authorizerName, groupMappingName, roleName); + String msgFormat = "Unassign role[%s] from groupMapping[%s]"; + performAuditIfSuccess(authorizerName, req, response, msgFormat, roleName, groupMappingName); + + return response; } /** @@ -486,7 +542,11 @@ public Response setRolePermissions( ) { authValidator.validateAuthorizerName(authorizerName); - return resourceHandler.setRolePermissions(authorizerName, roleName, permissions); + + final Response response = resourceHandler.setRolePermissions(authorizerName, roleName, permissions); + performAuditIfSuccess(authorizerName, req, response, "Set permissions[%s] for role[%s]", permissions, roleName); + + return response; } /** @@ -607,4 +667,35 @@ public Response authorizerGroupMappingUpdateListener( authValidator.validateAuthorizerName(authorizerName); return resourceHandler.authorizerGroupMappingUpdateListener(authorizerName, serializedGroupMappingAndRoleMap); } + + private boolean isSuccess(Response response) + { + if (response == null) { + return false; + } + + int responseCode = response.getStatus(); + return responseCode >= 200 && responseCode < 300; + } + + private void performAuditIfSuccess( + String authorizerName, + HttpServletRequest request, + Response response, + String msgFormat, + Object... args + ) + { + if (isSuccess(response)) { + auditManager.doAudit( + AuditEntry.builder() + .key(authorizerName) + .type("basicAuthorization") + .auditInfo(AuthorizationUtils.buildAuditInfo(request)) + .request(AuthorizationUtils.buildRequestInfo("coordinator", request)) + .payload(StringUtils.format(msgFormat, args)) + .build() + ); + } + } } diff --git a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authorization/CoordinatorBasicAuthorizerResourceTest.java b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authorization/CoordinatorBasicAuthorizerResourceTest.java index 746d3339f974..225424daf4e2 100644 --- a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authorization/CoordinatorBasicAuthorizerResourceTest.java +++ b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authorization/CoordinatorBasicAuthorizerResourceTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import org.apache.druid.audit.AuditManager; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.metadata.MetadataStorageTablesConfig; @@ -55,7 +56,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; @@ -78,15 +78,14 @@ public class CoordinatorBasicAuthorizerResourceTest private static final String AUTHORIZER_NAME2 = "test2"; private static final String AUTHORIZER_NAME3 = "test3"; - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Rule public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule = new TestDerbyConnector.DerbyConnectorRule(); @Mock private AuthValidator authValidator; @Mock private HttpServletRequest req; + @Mock + private AuditManager auditManager; private TestDerbyConnector connector; private MetadataStorageTablesConfig tablesConfig; @@ -154,7 +153,8 @@ public void setUp() authorizerMapper, new ObjectMapper(new SmileFactory()) ), - authValidator + authValidator, + auditManager ); storageUpdater.start(); diff --git a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResourceTest.java b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResourceTest.java index de0bf1db4e32..6350d3f91399 100644 --- a/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResourceTest.java +++ b/extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResourceTest.java @@ -68,7 +68,7 @@ public void setUp() .when(authValidator) .validateAuthorizerName(INVALID_AUTHORIZER_NAME); - target = new BasicAuthorizerResource(resourceHandler, authValidator); + target = new BasicAuthorizerResource(resourceHandler, authValidator, null); } @Test diff --git a/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java b/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java index 410e4a9202d2..c5e3c84cae9d 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java @@ -24,6 +24,7 @@ import org.apache.druid.java.util.http.client.CredentialedHttpClient; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.java.util.http.client.auth.BasicCredentials; +import org.apache.druid.java.util.http.client.response.StatusResponseHolder; import org.apache.druid.security.basic.authentication.entity.BasicAuthenticatorCredentialUpdate; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.ResourceAction; @@ -32,10 +33,12 @@ import org.apache.druid.testing.utils.ITRetryUtil; import org.apache.druid.tests.TestNGGroup; import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; +import java.io.IOException; import java.util.List; import java.util.Properties; @@ -80,7 +83,6 @@ public void test_druid99User_hasNodeAccess() protected void setupDatasourceOnlyUser() throws Exception { createUserAndRoleWithPermissions( - getHttpClient(User.ADMIN), "datasourceOnlyUser", "helloworld", "datasourceOnlyRole", @@ -92,7 +94,6 @@ protected void setupDatasourceOnlyUser() throws Exception protected void setupDatasourceAndContextParamsUser() throws Exception { createUserAndRoleWithPermissions( - getHttpClient(User.ADMIN), "datasourceAndContextParamsUser", "helloworld", "datasourceAndContextParamsRole", @@ -104,7 +105,6 @@ protected void setupDatasourceAndContextParamsUser() throws Exception protected void setupDatasourceAndSysTableUser() throws Exception { createUserAndRoleWithPermissions( - getHttpClient(User.ADMIN), "datasourceAndSysUser", "helloworld", "datasourceAndSysRole", @@ -116,7 +116,6 @@ protected void setupDatasourceAndSysTableUser() throws Exception protected void setupDatasourceAndSysAndStateUser() throws Exception { createUserAndRoleWithPermissions( - getHttpClient(User.ADMIN), "datasourceWithStateUser", "helloworld", "datasourceWithStateRole", @@ -128,7 +127,6 @@ protected void setupDatasourceAndSysAndStateUser() throws Exception protected void setupSysTableAndStateOnlyUser() throws Exception { createUserAndRoleWithPermissions( - getHttpClient(User.ADMIN), "stateOnlyUser", "helloworld", "stateOnlyRole", @@ -141,7 +139,6 @@ protected void setupTestSpecificHttpClients() throws Exception { // create a new user+role that can read /status createUserAndRoleWithPermissions( - getHttpClient(User.ADMIN), "druid", "helloworld", "druidrole", @@ -153,14 +150,14 @@ protected void setupTestSpecificHttpClients() throws Exception HttpUtil.makeRequest( getHttpClient(User.ADMIN), HttpMethod.POST, - config.getCoordinatorUrl() + "/druid-ext/basic-security/authentication/db/basic/users/druid" + i, + getBaseUrl() + "/authentication/db/basic/users/druid" + i, null ); HttpUtil.makeRequest( getHttpClient(User.ADMIN), HttpMethod.POST, - config.getCoordinatorUrl() + "/druid-ext/basic-security/authorization/db/basic/users/druid" + i, + getBaseUrl() + "/authorization/db/basic/users/druid" + i, null ); @@ -171,14 +168,14 @@ protected void setupTestSpecificHttpClients() throws Exception HttpUtil.makeRequest( getHttpClient(User.ADMIN), HttpMethod.POST, - config.getCoordinatorUrl() + "/druid-ext/basic-security/authentication/db/basic/users/druid99/credentials", + getBaseUrl() + "/authentication/db/basic/users/druid99/credentials", jsonMapper.writeValueAsBytes(new BasicAuthenticatorCredentialUpdate("helloworld", 5000)) ); HttpUtil.makeRequest( getHttpClient(User.ADMIN), HttpMethod.POST, - config.getCoordinatorUrl() + "/druid-ext/basic-security/authorization/db/basic/users/druid99/roles/druidrole", + getBaseUrl() + "/authorization/db/basic/users/druid99/roles/druidrole", null ); @@ -231,74 +228,51 @@ protected Properties getAvaticaConnectionPropertiesForUser(User user) } private void createUserAndRoleWithPermissions( - HttpClient adminClient, String user, String password, String role, List permissions ) throws Exception { - HttpUtil.makeRequest( - adminClient, - HttpMethod.POST, - StringUtils.format( - "%s/druid-ext/basic-security/authentication/db/basic/users/%s", - config.getCoordinatorUrl(), - user - ), - null - ); - HttpUtil.makeRequest( - adminClient, - HttpMethod.POST, - StringUtils.format( - "%s/druid-ext/basic-security/authentication/db/basic/users/%s/credentials", - config.getCoordinatorUrl(), - user - ), - jsonMapper.writeValueAsBytes(new BasicAuthenticatorCredentialUpdate(password, 5000)) - ); - HttpUtil.makeRequest( - adminClient, - HttpMethod.POST, - StringUtils.format( - "%s/druid-ext/basic-security/authorization/db/basic/users/%s", - config.getCoordinatorUrl(), - user - ), - null - ); - HttpUtil.makeRequest( - adminClient, - HttpMethod.POST, - StringUtils.format( - "%s/druid-ext/basic-security/authorization/db/basic/roles/%s", - config.getCoordinatorUrl(), - role - ), - null - ); - HttpUtil.makeRequest( - adminClient, - HttpMethod.POST, - StringUtils.format( - "%s/druid-ext/basic-security/authorization/db/basic/users/%s/roles/%s", - config.getCoordinatorUrl(), - user, - role - ), - null - ); - byte[] permissionsBytes = jsonMapper.writeValueAsBytes(permissions); - HttpUtil.makeRequest( - adminClient, - HttpMethod.POST, - StringUtils.format( - "%s/druid-ext/basic-security/authorization/db/basic/roles/%s/permissions", - config.getCoordinatorUrl(), - role - ), - permissionsBytes - ); + final HttpClient adminClient = getHttpClient(User.ADMIN); + + // Setup authentication by creating user and password + doPost(adminClient, null, "/authentication/db/basic/users/%s", user); + + final BasicAuthenticatorCredentialUpdate payload = new BasicAuthenticatorCredentialUpdate(password, 5000); + doPost(adminClient, payload, "/authentication/db/basic/users/%s/credentials", user); + + // Setup authorization by assigning a role to the user + doPost(adminClient, null, "/authorization/db/basic/users/%s", user); + doPost(adminClient, null, "/authorization/db/basic/roles/%s", role); + doPost(adminClient, null, "/authorization/db/basic/users/%s/roles/%s", user, role); + doPost(adminClient, permissions, "/authorization/db/basic/roles/%s/permissions", role); + } + + private void doPost( + HttpClient httpClient, + Object payload, + String pathFormat, + Object... pathParams + ) throws IOException + { + byte[] payloadBytes = payload == null ? null : jsonMapper.writeValueAsBytes(payload); + String url = getBaseUrl() + StringUtils.format(pathFormat, pathParams); + + StatusResponseHolder responseHolder + = HttpUtil.makeRequest(httpClient, HttpMethod.POST, url, payloadBytes); + + HttpResponseStatus status = responseHolder.getStatus(); + if (status.getCode() < 200 || status.getCode() >= 300) { + LOG.info( + "Post request to url[%s] returned status[%s], content[%s]", + url, status, responseHolder.getContent() + ); + } + } + + private String getBaseUrl() + { + return config.getCoordinatorUrl() + "/druid-ext/basic-security"; } } diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java index f99636619455..1293575d79b4 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditLogger.java @@ -30,7 +30,8 @@ public enum Level } private static final String MSG_FORMAT - = "User[%s], identity[%s], IP[%s] performed action[%s] on key[%s] with comment[%s], request[%s], payload[%s]."; + = "User[%s], identity[%s], IP[%s] performed action of type[%s] on key[%s]" + + " with comment[%s], request[%s], payload[%s]."; private final Level level; private final Logger logger = new Logger(AuditLogger.class); From ed225093fdfca799474857e1141b9dcb7bba0448 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Fri, 15 Dec 2023 10:15:16 +0530 Subject: [PATCH 25/29] More changes --- .../ITBasicAuthConfigurationTest.java | 71 ++++++------------- 1 file changed, 20 insertions(+), 51 deletions(-) diff --git a/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java b/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java index c5e3c84cae9d..17123c02352f 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java @@ -24,7 +24,6 @@ import org.apache.druid.java.util.http.client.CredentialedHttpClient; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.java.util.http.client.auth.BasicCredentials; -import org.apache.druid.java.util.http.client.response.StatusResponseHolder; import org.apache.druid.security.basic.authentication.entity.BasicAuthenticatorCredentialUpdate; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.ResourceAction; @@ -33,7 +32,6 @@ import org.apache.druid.testing.utils.ITRetryUtil; import org.apache.druid.tests.TestNGGroup; import org.jboss.netty.handler.codec.http.HttpMethod; -import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; @@ -147,37 +145,18 @@ protected void setupTestSpecificHttpClients() throws Exception // create 100 users for (int i = 0; i < 100; i++) { - HttpUtil.makeRequest( - getHttpClient(User.ADMIN), - HttpMethod.POST, - getBaseUrl() + "/authentication/db/basic/users/druid" + i, - null - ); - - HttpUtil.makeRequest( - getHttpClient(User.ADMIN), - HttpMethod.POST, - getBaseUrl() + "/authorization/db/basic/users/druid" + i, - null - ); - - LOG.info("Finished creating user druid" + i); + final String username = "druid" + i; + postAsAdmin(null, "/authentication/db/basic/users/%s", username); + postAsAdmin(null, "/authorization/db/basic/users/%s", username); + LOG.info("Created user[%s]", username); } // setup the last of 100 users and check that it works - HttpUtil.makeRequest( - getHttpClient(User.ADMIN), - HttpMethod.POST, - getBaseUrl() + "/authentication/db/basic/users/druid99/credentials", - jsonMapper.writeValueAsBytes(new BasicAuthenticatorCredentialUpdate("helloworld", 5000)) - ); - - HttpUtil.makeRequest( - getHttpClient(User.ADMIN), - HttpMethod.POST, - getBaseUrl() + "/authorization/db/basic/users/druid99/roles/druidrole", - null + postAsAdmin( + new BasicAuthenticatorCredentialUpdate("helloworld", 5000), + "/authentication/db/basic/users/druid99/credentials" ); + postAsAdmin(null, "/authorization/db/basic/users/druid99/roles/druidrole"); druid99 = new CredentialedHttpClient( new BasicCredentials("druid99", "helloworld"), @@ -234,41 +213,31 @@ private void createUserAndRoleWithPermissions( List permissions ) throws Exception { - final HttpClient adminClient = getHttpClient(User.ADMIN); - // Setup authentication by creating user and password - doPost(adminClient, null, "/authentication/db/basic/users/%s", user); + postAsAdmin(null, "/authentication/db/basic/users/%s", user); - final BasicAuthenticatorCredentialUpdate payload = new BasicAuthenticatorCredentialUpdate(password, 5000); - doPost(adminClient, payload, "/authentication/db/basic/users/%s/credentials", user); + final BasicAuthenticatorCredentialUpdate credentials + = new BasicAuthenticatorCredentialUpdate(password, 5000); + postAsAdmin(credentials, "/authentication/db/basic/users/%s/credentials", user); // Setup authorization by assigning a role to the user - doPost(adminClient, null, "/authorization/db/basic/users/%s", user); - doPost(adminClient, null, "/authorization/db/basic/roles/%s", role); - doPost(adminClient, null, "/authorization/db/basic/users/%s/roles/%s", user, role); - doPost(adminClient, permissions, "/authorization/db/basic/roles/%s/permissions", role); + postAsAdmin(null, "/authorization/db/basic/users/%s", user); + postAsAdmin(null, "/authorization/db/basic/roles/%s", role); + postAsAdmin(null, "/authorization/db/basic/users/%s/roles/%s", user, role); + postAsAdmin(permissions, "/authorization/db/basic/roles/%s/permissions", role); } - private void doPost( - HttpClient httpClient, + private void postAsAdmin( Object payload, String pathFormat, Object... pathParams ) throws IOException { + HttpClient adminClient = getHttpClient(User.ADMIN); + byte[] payloadBytes = payload == null ? null : jsonMapper.writeValueAsBytes(payload); String url = getBaseUrl() + StringUtils.format(pathFormat, pathParams); - - StatusResponseHolder responseHolder - = HttpUtil.makeRequest(httpClient, HttpMethod.POST, url, payloadBytes); - - HttpResponseStatus status = responseHolder.getStatus(); - if (status.getCode() < 200 || status.getCode() >= 300) { - LOG.info( - "Post request to url[%s] returned status[%s], content[%s]", - url, status, responseHolder.getContent() - ); - } + HttpUtil.makeRequest(adminClient, HttpMethod.POST, url, payloadBytes); } private String getBaseUrl() From 0b73b5f5f57f69e15118de5768ec51bbad546ebc Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Mon, 18 Dec 2023 17:07:35 +0530 Subject: [PATCH 26/29] Use Escalator to determine system identity --- .../org/apache/druid/audit/AuditManager.java | 14 -------------- .../druid/rpc/indexing/OverlordClientImpl.java | 4 +--- .../druid/server/audit/AuditSerdeHelper.java | 18 ++++++++++++++++++ .../server/audit/LoggingAuditManager.java | 2 +- .../druid/server/audit/SQLAuditManager.java | 2 +- .../metadata/SQLMetadataRuleManagerTest.java | 2 +- .../server/audit/SQLAuditManagerTest.java | 2 +- 7 files changed, 23 insertions(+), 21 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/audit/AuditManager.java b/processing/src/main/java/org/apache/druid/audit/AuditManager.java index 95d81dd49a0b..3ab126e93373 100644 --- a/processing/src/main/java/org/apache/druid/audit/AuditManager.java +++ b/processing/src/main/java/org/apache/druid/audit/AuditManager.java @@ -32,20 +32,6 @@ public interface AuditManager String X_DRUID_AUTHOR = "X-Druid-Author"; String X_DRUID_COMMENT = "X-Druid-Comment"; - /** - * Value of header {@link #X_DRUID_AUTHOR} used by Druid services so that they - * can be distinguished from external requests. - */ - String AUTHOR_DRUID_SYSTEM = "druid_system"; - - /** - * @return true if the audited event was initiated by the Druid system itself. - */ - default boolean isSystemRequest(AuditInfo auditInfo) - { - return AUTHOR_DRUID_SYSTEM.equals(auditInfo.getAuthor()); - } - void doAudit(AuditEntry event); /** diff --git a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java index 676848f9961e..d7fab4b75fa2 100644 --- a/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java +++ b/server/src/main/java/org/apache/druid/rpc/indexing/OverlordClientImpl.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import org.apache.druid.audit.AuditManager; import org.apache.druid.client.JsonParserIterator; import org.apache.druid.client.indexing.IndexingTotalWorkerCapacityInfo; import org.apache.druid.client.indexing.IndexingWorkerInfo; @@ -97,8 +96,7 @@ public ListenableFuture runTask(final String taskId, final Object taskObje return FutureUtils.transform( client.asyncRequest( new RequestBuilder(HttpMethod.POST, "/druid/indexer/v1/task") - .jsonContent(jsonMapper, taskObject) - .header(AuditManager.X_DRUID_AUTHOR, AuditManager.AUTHOR_DRUID_SYSTEM), + .jsonContent(jsonMapper, taskObject), new BytesFullResponseHandler() ), holder -> { diff --git a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java index 1450de66e30f..385811d055ce 100644 --- a/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java +++ b/server/src/main/java/org/apache/druid/server/audit/AuditSerdeHelper.java @@ -26,6 +26,7 @@ import org.apache.druid.guice.annotations.JsonNonNull; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.server.security.Escalator; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -50,11 +51,13 @@ public class AuditSerdeHelper private final ObjectMapper jsonMapper; private final ObjectMapper jsonMapperSkipNulls; + private final String systemIdentity; private final AuditManagerConfig config; @Inject public AuditSerdeHelper( AuditManagerConfig config, + Escalator escalator, @Json ObjectMapper jsonMapper, @JsonNonNull ObjectMapper jsonMapperSkipNulls ) @@ -62,6 +65,21 @@ public AuditSerdeHelper( this.config = config; this.jsonMapper = jsonMapper; this.jsonMapperSkipNulls = jsonMapperSkipNulls; + this.systemIdentity = escalator == null + ? null : escalator.createEscalatedAuthenticationResult().getIdentity(); + } + + /** + * Checks if the given audit event needs to be handled. + * + * @return true only if the event was not initiated by the Druid system OR if + * system requests should be audited too. + */ + public boolean shouldProcessAuditEntry(AuditEntry entry) + { + final boolean isSystemRequest = systemIdentity != null + && systemIdentity.equals(entry.getAuditInfo().getIdentity()); + return config.isAuditSystemRequests() || !isSystemRequest; } /** diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java index 68840d7a3ca0..65d3427b8ff5 100644 --- a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManager.java @@ -56,7 +56,7 @@ public LoggingAuditManager( @Override public void doAudit(AuditEntry entry) { - if (managerConfig.isAuditSystemRequests() || !isSystemRequest(entry.getAuditInfo())) { + if (serdeHelper.shouldProcessAuditEntry(entry)) { auditLogger.log(serdeHelper.processAuditEntry(entry)); } } diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java index cabbfb9251b6..13c4f167e098 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManager.java @@ -136,7 +136,7 @@ private ServiceMetricEvent.Builder createMetricEventBuilder(AuditEntry entry) @Override public void doAudit(AuditEntry event, Handle handle) throws IOException { - if (isSystemRequest(event.getAuditInfo()) && !config.isAuditSystemRequests()) { + if (!serdeHelper.shouldProcessAuditEntry(event)) { return; } diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java index 17442cf2ca04..f1b7855e3d1f 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java @@ -75,7 +75,7 @@ public void setUp() final SQLAuditManagerConfig auditManagerConfig = new SQLAuditManagerConfig(null, null, null, null, null); auditManager = new SQLAuditManager( auditManagerConfig, - new AuditSerdeHelper(auditManagerConfig, mapper, mapper), + new AuditSerdeHelper(auditManagerConfig, null, mapper, mapper), connector, Suppliers.ofInstance(tablesConfig), new NoopServiceEmitter(), diff --git a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java index 8e15aa4e2ce0..a12722088159 100644 --- a/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/SQLAuditManagerTest.java @@ -73,7 +73,7 @@ private SQLAuditManager createAuditManager(SQLAuditManagerConfig config) { return new SQLAuditManager( config, - new AuditSerdeHelper(config, mapper, mapperSkipNull), + new AuditSerdeHelper(config, null, mapper, mapperSkipNull), connector, derbyConnectorRule.metadataTablesConfigSupplier(), serviceEmitter, From c87fb1334874196681fa12618301ad184a7c9852 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 19 Dec 2023 09:09:46 +0530 Subject: [PATCH 27/29] Fix failing test --- .../endpoint/BasicAuthenticatorResource.java | 19 +++++++-------- .../endpoint/BasicAuthorizerResource.java | 2 +- .../mysql/MySQLMetadataStorageModuleTest.java | 2 ++ .../overlord/http/OverlordResource.java | 23 +++++++++++-------- .../audit/LoggingAuditManagerConfig.java | 2 +- .../server/audit/SQLAuditManagerConfig.java | 2 +- .../server/http/DataSourcesResource.java | 4 ++-- .../server/audit/AuditManagerConfigTest.java | 4 ++-- 8 files changed, 33 insertions(+), 25 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java index d77358dd7d70..fd0b029530c0 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authentication/endpoint/BasicAuthenticatorResource.java @@ -25,6 +25,7 @@ import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditManager; import org.apache.druid.guice.LazySingleton; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.security.basic.BasicSecurityResourceFilter; import org.apache.druid.security.basic.authentication.entity.BasicAuthenticatorCredentialUpdate; import org.apache.druid.server.security.AuthValidator; @@ -41,7 +42,6 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import java.util.Collections; @Path("/druid-ext/basic-security/authentication") @LazySingleton @@ -160,7 +160,7 @@ public Response createUser( authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.createUser(authenticatorName, userName); - performAuditIfSuccess(authenticatorName, userName, req, response); + performAuditIfSuccess(authenticatorName, req, response, "Create user[%s]", userName); return response; } @@ -186,7 +186,7 @@ public Response deleteUser( { authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.deleteUser(authenticatorName, userName); - performAuditIfSuccess(authenticatorName, userName, req, response); + performAuditIfSuccess(authenticatorName, req, response, "Delete user[%s]", userName); return response; } @@ -213,7 +213,7 @@ public Response updateUserCredentials( { authValidator.validateAuthenticatorName(authenticatorName); final Response response = handler.updateUserCredentials(authenticatorName, userName, update); - performAuditIfSuccess(authenticatorName, userName, req, response); + performAuditIfSuccess(authenticatorName, req, response, "Update credentials for user[%s]", userName); return response; } @@ -267,19 +267,20 @@ private boolean isSuccess(Response response) private void performAuditIfSuccess( String authenticatorName, - String updatedUser, HttpServletRequest request, - Response response + Response response, + String payloadFormat, + Object... payloadArgs ) { - if (updatedUser != null && isSuccess(response)) { + if (isSuccess(response)) { auditManager.doAudit( AuditEntry.builder() .key(authenticatorName) - .type("basicAuthentication") + .type("basic.authenticator") .auditInfo(AuthorizationUtils.buildAuditInfo(request)) .request(AuthorizationUtils.buildRequestInfo("coordinator", request)) - .payload(Collections.singletonMap("username", updatedUser)) + .payload(StringUtils.format(payloadFormat, payloadArgs)) .build() ); } diff --git a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResource.java b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResource.java index c295228149b9..700d86b3b040 100644 --- a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResource.java +++ b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/authorization/endpoint/BasicAuthorizerResource.java @@ -690,7 +690,7 @@ private void performAuditIfSuccess( auditManager.doAudit( AuditEntry.builder() .key(authorizerName) - .type("basicAuthorization") + .type("basic.authorizer") .auditInfo(AuthorizationUtils.buildAuditInfo(request)) .request(AuthorizationUtils.buildRequestInfo("coordinator", request)) .payload(StringUtils.format(msgFormat, args)) diff --git a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java index 0e2d6c359b5b..b1c4922ea643 100644 --- a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java +++ b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java @@ -32,6 +32,7 @@ import org.apache.druid.guice.LifecycleModule; import org.apache.druid.guice.MetadataConfigModule; import org.apache.druid.guice.annotations.Json; +import org.apache.druid.guice.security.EscalatorModule; import org.apache.druid.java.util.emitter.core.NoopEmitter; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.junit.Assert; @@ -111,6 +112,7 @@ private Injector createInjector() MySQLMetadataStorageModule module = new MySQLMetadataStorageModule(); Injector injector = GuiceInjectors.makeStartupInjectorWithModules( ImmutableList.of( + new EscalatorModule(), new MetadataConfigModule(), new LifecycleModule(), module, diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index 3e75b50dbbe8..4771554bca77 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -24,6 +24,7 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.inject.Inject; @@ -138,6 +139,8 @@ public class OverlordResource private AtomicReference workerConfigRef = null; private static final List API_TASK_STATES = ImmutableList.of("pending", "waiting", "running", "complete"); + private static final Set AUDITED_TASK_TYPES + = ImmutableSet.of("index", "index_parallel", "compact", "index_hadoop"); private enum TaskStateLookup { @@ -223,15 +226,17 @@ public Response taskPost( try { taskQueue.add(task); - auditManager.doAudit( - AuditEntry.builder() - .key(task.getDataSource()) - .type("task") - .request(AuthorizationUtils.buildRequestInfo("overlord", req)) - .payload(new TaskIdentifier(task.getId(), task.getGroupId(), task.getType())) - .auditInfo(AuthorizationUtils.buildAuditInfo(req)) - .build() - ); + if (AUDITED_TASK_TYPES.contains(task.getType())) { + auditManager.doAudit( + AuditEntry.builder() + .key(task.getDataSource()) + .type("task") + .request(AuthorizationUtils.buildRequestInfo("overlord", req)) + .payload(new TaskIdentifier(task.getId(), task.getGroupId(), task.getType())) + .auditInfo(AuthorizationUtils.buildAuditInfo(req)) + .build() + ); + } return Response.ok(ImmutableMap.of("task", task.getId())).build(); } diff --git a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java index a06c778b412f..35f525ba8bd4 100644 --- a/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/LoggingAuditManagerConfig.java @@ -52,7 +52,7 @@ public LoggingAuditManagerConfig( ) { this.logLevel = Configs.valueOrDefault(logLevel, AuditLogger.Level.INFO); - this.auditSystemRequests = Configs.valueOrDefault(auditSystemRequests, false); + this.auditSystemRequests = Configs.valueOrDefault(auditSystemRequests, true); this.maxPayloadSizeBytes = Configs.valueOrDefault(maxPayloadSizeBytes, HumanReadableBytes.valueOf(-1)); this.skipNullField = Configs.valueOrDefault(skipNullField, false); } diff --git a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java index c9a0bd7a70d1..e1ac78ece178 100644 --- a/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java +++ b/server/src/main/java/org/apache/druid/server/audit/SQLAuditManagerConfig.java @@ -57,7 +57,7 @@ public SQLAuditManagerConfig( @JsonProperty("includePayloadAsDimensionInMetric") Boolean includePayloadAsDimensionInMetric ) { - this.auditSystemRequests = Configs.valueOrDefault(auditSystemRequests, false); + this.auditSystemRequests = Configs.valueOrDefault(auditSystemRequests, true); this.maxPayloadSizeBytes = Configs.valueOrDefault(maxPayloadSizeBytes, HumanReadableBytes.valueOf(-1)); this.skipNullField = Configs.valueOrDefault(skipNullField, false); this.auditHistoryMillis = Configs.valueOrDefault(auditHistoryMillis, 7 * 24 * 60 * 60 * 1000L); diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index af98c886b972..51e8a3d93c3f 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -256,7 +256,7 @@ public Response markSegmentsAsUnused( auditManager.doAudit( AuditEntry.builder() .key(dataSourceName) - .type("markSegmentsAsUnused") + .type("segment.markUnused") .payload(auditPayload) .auditInfo(AuthorizationUtils.buildAuditInfo(req)) .request(AuthorizationUtils.buildRequestInfo("coordinator", req)) @@ -375,7 +375,7 @@ public Response killUnusedSegmentsInInterval( auditManager.doAudit( AuditEntry.builder() .key(dataSourceName) - .type("killUnusedSegmentsInInterval") + .type("segment.kill") .payload(ImmutableMap.of("killTaskId", killTaskId, "interval", theInterval)) .auditInfo(AuthorizationUtils.buildAuditInfo(req)) .request(AuthorizationUtils.buildRequestInfo("coordinator", req)) diff --git a/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java b/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java index 1ae61a79f6e0..2ed4d85856f6 100644 --- a/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/audit/AuditManagerConfigTest.java @@ -47,7 +47,7 @@ public void testDefaultAuditConfig() Assert.assertTrue(config instanceof SQLAuditManagerConfig); final SQLAuditManagerConfig sqlAuditConfig = (SQLAuditManagerConfig) config; - Assert.assertFalse(sqlAuditConfig.isAuditSystemRequests()); + Assert.assertTrue(sqlAuditConfig.isAuditSystemRequests()); Assert.assertFalse(sqlAuditConfig.isSkipNullField()); Assert.assertFalse(sqlAuditConfig.isIncludePayloadAsDimensionInMetric()); Assert.assertEquals(-1, sqlAuditConfig.getMaxPayloadSizeBytes()); @@ -71,7 +71,7 @@ public void testLogAuditConfigWithDefaults() Assert.assertTrue(config instanceof LoggingAuditManagerConfig); final LoggingAuditManagerConfig logAuditConfig = (LoggingAuditManagerConfig) config; - Assert.assertFalse(logAuditConfig.isAuditSystemRequests()); + Assert.assertTrue(logAuditConfig.isAuditSystemRequests()); Assert.assertFalse(logAuditConfig.isSkipNullField()); Assert.assertEquals(-1, logAuditConfig.getMaxPayloadSizeBytes()); Assert.assertEquals(AuditLogger.Level.INFO, logAuditConfig.getLogLevel()); From 33d8a227a6c0e581185fe0eeb08763982db559d6 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 19 Dec 2023 10:47:24 +0530 Subject: [PATCH 28/29] Add more tests for coverage --- .../overlord/http/OverlordResource.java | 2 +- .../overlord/http/OverlordResourceTest.java | 57 ++++++++++++++++++- .../guice/SQLMetadataStorageDruidModule.java | 32 ----------- .../org/apache/druid/guice/ServerModule.java | 33 +++++++++++ 4 files changed, 90 insertions(+), 34 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index 4771554bca77..7afaaa905264 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -140,7 +140,7 @@ public class OverlordResource private AtomicReference workerConfigRef = null; private static final List API_TASK_STATES = ImmutableList.of("pending", "waiting", "running", "complete"); private static final Set AUDITED_TASK_TYPES - = ImmutableSet.of("index", "index_parallel", "compact", "index_hadoop"); + = ImmutableSet.of("index", "index_parallel", "compact", "index_hadoop", "kill"); private enum TaskStateLookup { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java index 6eed9e32df38..561e8dd9b199 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java @@ -25,6 +25,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.apache.druid.audit.AuditEntry; +import org.apache.druid.audit.AuditManager; import org.apache.druid.common.config.JacksonConfigManager; import org.apache.druid.indexer.RunnerTaskState; import org.apache.druid.indexer.TaskInfo; @@ -32,6 +34,7 @@ import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexer.TaskStatusPlus; +import org.apache.druid.indexing.common.task.KillUnusedSegmentsTask; import org.apache.druid.indexing.common.task.NoopTask; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.indexing.overlord.ImmutableWorkerInfo; @@ -67,6 +70,7 @@ import org.apache.druid.server.security.Resource; import org.apache.druid.server.security.ResourceAction; import org.apache.druid.server.security.ResourceType; +import org.easymock.Capture; import org.easymock.EasyMock; import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.joda.time.DateTime; @@ -104,7 +108,9 @@ public class OverlordResourceTest private IndexerMetadataStorageAdapter indexerMetadataStorageAdapter; private HttpServletRequest req; private TaskRunner taskRunner; + private TaskQueue taskQueue; private WorkerTaskRunnerQueryAdapter workerTaskRunnerQueryAdapter; + private AuditManager auditManager; @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -113,6 +119,7 @@ public class OverlordResourceTest public void setUp() { taskRunner = EasyMock.createMock(TaskRunner.class); + taskQueue = EasyMock.createMock(TaskQueue.class); configManager = EasyMock.createMock(JacksonConfigManager.class); provisioningStrategy = EasyMock.createMock(ProvisioningStrategy.class); authConfig = EasyMock.createMock(AuthConfig.class); @@ -121,6 +128,7 @@ public void setUp() indexerMetadataStorageAdapter = EasyMock.createStrictMock(IndexerMetadataStorageAdapter.class); req = EasyMock.createStrictMock(HttpServletRequest.class); workerTaskRunnerQueryAdapter = EasyMock.createStrictMock(WorkerTaskRunnerQueryAdapter.class); + auditManager = EasyMock.createMock(AuditManager.class); EasyMock.expect(taskMaster.getTaskRunner()).andReturn( Optional.of(taskRunner) @@ -165,7 +173,7 @@ public Access authorize(AuthenticationResult authenticationResult, Resource reso indexerMetadataStorageAdapter, null, configManager, - null, + auditManager, authMapper, workerTaskRunnerQueryAdapter, provisioningStrategy, @@ -880,6 +888,53 @@ public void testSecuredTaskPost() overlordResource.taskPost(task, req); } + @Test + public void testKillTaskIsAudited() + { + EasyMock.expect(authConfig.isEnableInputSourceSecurity()).andReturn(false); + + final String username = Users.DRUID; + expectAuthorizationTokenCheck(username); + EasyMock.expect(req.getMethod()).andReturn("POST").once(); + EasyMock.expect(req.getRequestURI()).andReturn("/indexer/v2/task").once(); + EasyMock.expect(req.getQueryString()).andReturn("").once(); + EasyMock.expect(req.getHeader(AuditManager.X_DRUID_AUTHOR)).andReturn(username).once(); + EasyMock.expect(req.getHeader(AuditManager.X_DRUID_COMMENT)).andReturn("killing segments").once(); + EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)) + .andReturn(new AuthenticationResult(username, "druid", null, null)) + .once(); + EasyMock.expect(req.getRemoteAddr()).andReturn("127.0.0.1").once(); + + EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes(); + EasyMock.expect(taskQueue.add(EasyMock.anyObject())).andReturn(true).once(); + + final Capture auditEntryCapture = EasyMock.newCapture(); + auditManager.doAudit(EasyMock.capture(auditEntryCapture)); + EasyMock.expectLastCall().once(); + + EasyMock.replay( + taskRunner, + taskMaster, + taskQueue, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter, + authConfig, + auditManager + ); + + Task task = new KillUnusedSegmentsTask("kill_all", "allow", Intervals.ETERNITY, null, false, 10, null); + overlordResource.taskPost(task, req); + + Assert.assertTrue(auditEntryCapture.hasCaptured()); + AuditEntry auditEntry = auditEntryCapture.getValue(); + Assert.assertEquals(username, auditEntry.getAuditInfo().getAuthor()); + Assert.assertEquals("killing segments", auditEntry.getAuditInfo().getComment()); + Assert.assertEquals("druid", auditEntry.getAuditInfo().getIdentity()); + Assert.assertEquals("127.0.0.1", auditEntry.getAuditInfo().getIp()); + } + @Test public void testTaskPostDeniesDatasourceReadUser() { diff --git a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java index 836115d47a17..21d3a58a9769 100644 --- a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java +++ b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java @@ -22,8 +22,6 @@ import com.google.inject.Binder; import com.google.inject.Key; import com.google.inject.Module; -import com.google.inject.multibindings.MapBinder; -import org.apache.druid.audit.AuditManager; import org.apache.druid.indexer.MetadataStorageUpdaterJobHandler; import org.apache.druid.indexer.SQLMetadataStorageUpdaterJobHandler; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; @@ -46,10 +44,6 @@ import org.apache.druid.metadata.SegmentsMetadataManagerProvider; import org.apache.druid.metadata.SqlSegmentsMetadataManager; import org.apache.druid.metadata.SqlSegmentsMetadataManagerProvider; -import org.apache.druid.server.audit.AuditManagerConfig; -import org.apache.druid.server.audit.AuditSerdeHelper; -import org.apache.druid.server.audit.LoggingAuditManager; -import org.apache.druid.server.audit.SQLAuditManager; public class SQLMetadataStorageDruidModule implements Module { @@ -84,8 +78,6 @@ public void createBindingChoices(Binder binder, String defaultValue) PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageActionHandlerFactory.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageUpdaterJobHandler.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataSupervisorManager.class), defaultValue); - - configureAuditManager(binder); } @Override @@ -136,28 +128,4 @@ public void configure(Binder binder) .to(SQLMetadataSupervisorManager.class) .in(LazySingleton.class); } - - private void configureAuditManager(Binder binder) - { - JsonConfigProvider.bind(binder, "druid.audit.manager", AuditManagerConfig.class); - - PolyBind.createChoice( - binder, - "druid.audit.manager.type", - Key.get(AuditManager.class), - Key.get(SQLAuditManager.class) - ); - final MapBinder auditManagerBinder - = PolyBind.optionBinder(binder, Key.get(AuditManager.class)); - auditManagerBinder - .addBinding("log") - .to(LoggingAuditManager.class) - .in(LazySingleton.class); - auditManagerBinder - .addBinding("sql") - .to(SQLAuditManager.class) - .in(LazySingleton.class); - - binder.bind(AuditSerdeHelper.class).in(LazySingleton.class); - } } diff --git a/server/src/main/java/org/apache/druid/guice/ServerModule.java b/server/src/main/java/org/apache/druid/guice/ServerModule.java index 7ec37a0c4a3c..b0ae86f4a14a 100644 --- a/server/src/main/java/org/apache/druid/guice/ServerModule.java +++ b/server/src/main/java/org/apache/druid/guice/ServerModule.java @@ -23,7 +23,10 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; +import com.google.inject.Key; import com.google.inject.Provides; +import com.google.inject.multibindings.MapBinder; +import org.apache.druid.audit.AuditManager; import org.apache.druid.guice.annotations.Self; import org.apache.druid.initialization.DruidModule; import org.apache.druid.jackson.DruidServiceSerializerModifier; @@ -33,6 +36,10 @@ import org.apache.druid.java.util.common.concurrent.ScheduledExecutors; import org.apache.druid.java.util.common.lifecycle.Lifecycle; import org.apache.druid.server.DruidNode; +import org.apache.druid.server.audit.AuditManagerConfig; +import org.apache.druid.server.audit.AuditSerdeHelper; +import org.apache.druid.server.audit.LoggingAuditManager; +import org.apache.druid.server.audit.SQLAuditManager; import org.apache.druid.server.initialization.ZkPathsConfig; import java.util.List; @@ -48,6 +55,8 @@ public void configure(Binder binder) { JsonConfigProvider.bind(binder, ZK_PATHS_PROPERTY_BASE, ZkPathsConfig.class); JsonConfigProvider.bind(binder, "druid", DruidNode.class, Self.class); + + configureAuditManager(binder); } @Provides @LazySingleton @@ -65,4 +74,28 @@ public List getJacksonModules() .setSerializerModifier(new DruidServiceSerializerModifier()) ); } + + private void configureAuditManager(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.audit.manager", AuditManagerConfig.class); + + PolyBind.createChoice( + binder, + "druid.audit.manager.type", + Key.get(AuditManager.class), + Key.get(SQLAuditManager.class) + ); + final MapBinder auditManagerBinder + = PolyBind.optionBinder(binder, Key.get(AuditManager.class)); + auditManagerBinder + .addBinding("log") + .to(LoggingAuditManager.class) + .in(LazySingleton.class); + auditManagerBinder + .addBinding("sql") + .to(SQLAuditManager.class) + .in(LazySingleton.class); + + binder.bind(AuditSerdeHelper.class).in(LazySingleton.class); + } } From 1cff032b0a6d114059f09515571407fd8a2c6f6c Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 19 Dec 2023 11:26:54 +0530 Subject: [PATCH 29/29] Fix audit manager bindings --- .../guice/SQLMetadataStorageDruidModule.java | 24 ++++++++++++++ .../org/apache/druid/guice/ServerModule.java | 33 ------------------- .../druid/guice/StartupLoggingModule.java | 8 +++++ 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java index 21d3a58a9769..7ece79ad40f9 100644 --- a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java +++ b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java @@ -22,6 +22,7 @@ import com.google.inject.Binder; import com.google.inject.Key; import com.google.inject.Module; +import org.apache.druid.audit.AuditManager; import org.apache.druid.indexer.MetadataStorageUpdaterJobHandler; import org.apache.druid.indexer.SQLMetadataStorageUpdaterJobHandler; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; @@ -44,6 +45,9 @@ import org.apache.druid.metadata.SegmentsMetadataManagerProvider; import org.apache.druid.metadata.SqlSegmentsMetadataManager; import org.apache.druid.metadata.SqlSegmentsMetadataManagerProvider; +import org.apache.druid.server.audit.AuditManagerConfig; +import org.apache.druid.server.audit.AuditSerdeHelper; +import org.apache.druid.server.audit.SQLAuditManager; public class SQLMetadataStorageDruidModule implements Module { @@ -78,6 +82,8 @@ public void createBindingChoices(Binder binder, String defaultValue) PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageActionHandlerFactory.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageUpdaterJobHandler.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataSupervisorManager.class), defaultValue); + + configureAuditManager(binder); } @Override @@ -128,4 +134,22 @@ public void configure(Binder binder) .to(SQLMetadataSupervisorManager.class) .in(LazySingleton.class); } + + private void configureAuditManager(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.audit.manager", AuditManagerConfig.class); + + PolyBind.createChoice( + binder, + "druid.audit.manager.type", + Key.get(AuditManager.class), + Key.get(SQLAuditManager.class) + ); + PolyBind.optionBinder(binder, Key.get(AuditManager.class)) + .addBinding("sql") + .to(SQLAuditManager.class) + .in(LazySingleton.class); + + binder.bind(AuditSerdeHelper.class).in(LazySingleton.class); + } } diff --git a/server/src/main/java/org/apache/druid/guice/ServerModule.java b/server/src/main/java/org/apache/druid/guice/ServerModule.java index b0ae86f4a14a..7ec37a0c4a3c 100644 --- a/server/src/main/java/org/apache/druid/guice/ServerModule.java +++ b/server/src/main/java/org/apache/druid/guice/ServerModule.java @@ -23,10 +23,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; -import com.google.inject.Key; import com.google.inject.Provides; -import com.google.inject.multibindings.MapBinder; -import org.apache.druid.audit.AuditManager; import org.apache.druid.guice.annotations.Self; import org.apache.druid.initialization.DruidModule; import org.apache.druid.jackson.DruidServiceSerializerModifier; @@ -36,10 +33,6 @@ import org.apache.druid.java.util.common.concurrent.ScheduledExecutors; import org.apache.druid.java.util.common.lifecycle.Lifecycle; import org.apache.druid.server.DruidNode; -import org.apache.druid.server.audit.AuditManagerConfig; -import org.apache.druid.server.audit.AuditSerdeHelper; -import org.apache.druid.server.audit.LoggingAuditManager; -import org.apache.druid.server.audit.SQLAuditManager; import org.apache.druid.server.initialization.ZkPathsConfig; import java.util.List; @@ -55,8 +48,6 @@ public void configure(Binder binder) { JsonConfigProvider.bind(binder, ZK_PATHS_PROPERTY_BASE, ZkPathsConfig.class); JsonConfigProvider.bind(binder, "druid", DruidNode.class, Self.class); - - configureAuditManager(binder); } @Provides @LazySingleton @@ -74,28 +65,4 @@ public List getJacksonModules() .setSerializerModifier(new DruidServiceSerializerModifier()) ); } - - private void configureAuditManager(Binder binder) - { - JsonConfigProvider.bind(binder, "druid.audit.manager", AuditManagerConfig.class); - - PolyBind.createChoice( - binder, - "druid.audit.manager.type", - Key.get(AuditManager.class), - Key.get(SQLAuditManager.class) - ); - final MapBinder auditManagerBinder - = PolyBind.optionBinder(binder, Key.get(AuditManager.class)); - auditManagerBinder - .addBinding("log") - .to(LoggingAuditManager.class) - .in(LazySingleton.class); - auditManagerBinder - .addBinding("sql") - .to(SQLAuditManager.class) - .in(LazySingleton.class); - - binder.bind(AuditSerdeHelper.class).in(LazySingleton.class); - } } diff --git a/server/src/main/java/org/apache/druid/guice/StartupLoggingModule.java b/server/src/main/java/org/apache/druid/guice/StartupLoggingModule.java index c75b0901a88f..db25a0906c38 100644 --- a/server/src/main/java/org/apache/druid/guice/StartupLoggingModule.java +++ b/server/src/main/java/org/apache/druid/guice/StartupLoggingModule.java @@ -20,7 +20,10 @@ package org.apache.druid.guice; import com.google.inject.Binder; +import com.google.inject.Key; import com.google.inject.Module; +import org.apache.druid.audit.AuditManager; +import org.apache.druid.server.audit.LoggingAuditManager; import org.apache.druid.server.log.StartupLoggingConfig; public class StartupLoggingModule implements Module @@ -29,5 +32,10 @@ public class StartupLoggingModule implements Module public void configure(Binder binder) { JsonConfigProvider.bind(binder, "druid.startup.logging", StartupLoggingConfig.class); + + PolyBind.optionBinder(binder, Key.get(AuditManager.class)) + .addBinding("log") + .to(LoggingAuditManager.class) + .in(LazySingleton.class); } }