From a19c3031fff216ed36fd1710871e7c42a6f4e40a Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Fri, 9 Aug 2024 16:23:44 -0700 Subject: [PATCH] adds toggling refresh disable/enable for deactivate/activate operation while updating URL_DOWNLOAD type configs Signed-off-by: Surya Sashank Nistala --- .../action/SAGetTIFSourceConfigResponse.java | 20 +---- .../SAIndexTIFSourceConfigResponse.java | 26 ++---- .../model/SATIFSourceConfigDto.java | 19 +++-- .../SATIFSourceConfigManagementService.java | 6 +- .../SourceConfigWithoutS3RestApiIT.java | 84 +++++++++++++++++++ 5 files changed, 108 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java index 247bcd134..7bebd8fb1 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java @@ -63,25 +63,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject() .field(_ID, id) .field(_VERSION, version); - builder.startObject("source_config") - .field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName()) - .field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat()) - .field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType()) - .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes()) - .field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription()) - .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser()) - .field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt()) - .field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource()) - .field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled()) - .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime()) - .field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime()) - .field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule()) - .field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState()) - .field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType()) - .field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser()) - .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime()); - - builder.endObject(); + saTifSourceConfigDto.innerXcontent(builder); return builder.endObject(); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java index 7a1881162..209563f7c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java @@ -17,6 +17,7 @@ import java.io.IOException; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto.SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; @@ -56,40 +57,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject() .field(_ID, id) .field(_VERSION, version); - - builder.startObject("source_config") - .field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName()) - .field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat()) - .field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType()) - .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes()) - .field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription()) - .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser()) - .field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt()) - .field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource()) - .field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled()) - .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime()) - .field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime()) - .field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule()) - .field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState()) - .field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType()) - .field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser()) - .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime()); - - builder.endObject(); + saTifSourceConfigDto.innerXcontent(builder); return builder.endObject(); } + @Override public String getTIFConfigId() { return id; } + @Override public Long getVersion() { return version; } + @Override public TIFSourceConfigDto getTIFConfigDto() { return saTifSourceConfigDto; } + public RestStatus getStatus() { return status; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java index a293c881e..3ba64d47a 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java @@ -215,12 +215,18 @@ public void writeTo(final StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - builder.startObject() - .startObject(SOURCE_CONFIG_FIELD) - .field(NAME_FIELD, name) - .field(FORMAT_FIELD, format) - .field(TYPE_FIELD, type.name()) - .field(DESCRIPTION_FIELD, description); + builder.startObject(); + innerXcontent(builder); + builder.endObject(); + return builder; + } + + public XContentBuilder innerXcontent(XContentBuilder builder) throws IOException { + builder.startObject(SOURCE_CONFIG_FIELD); + builder.field(NAME_FIELD, name) + .field(FORMAT_FIELD, format) + .field(TYPE_FIELD, type.name()) + .field(DESCRIPTION_FIELD, description); if (createdByUser == null) { builder.nullField(CREATED_BY_USER_FIELD); } else { @@ -274,7 +280,6 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(ENABLED_FOR_SCAN_FIELD, enabledForScan); builder.field(IOC_TYPES_FIELD, iocTypes); builder.endObject(); - builder.endObject(); return builder; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index 2e2c0a5c6..f7869c1df 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -280,6 +280,10 @@ public void updateIocAndTIFSourceConfig( // Due to the lack of a different API to do activate/deactivate we will check if enabled_for_scan variable is changed between model and request. // If yes, we will ONLY update enabled_for_scan field and ignore any updates to the rest of the fields to simulate a dedicated activate/deactivate API. if (retrievedSaTifSourceConfig.isEnabledForScan() != saTifSourceConfigDto.isEnabledForScan()) { + // FIXME add a disable_refresh api independent of update api so that it can be supported for default configs also + boolean isEnabled = URL_DOWNLOAD.equals(retrievedSaTifSourceConfig.getType()) ? + saTifSourceConfigDto.isEnabledForScan() : + retrievedSaTifSourceConfig.isEnabled(); SATIFSourceConfig config = new SATIFSourceConfig( retrievedSaTifSourceConfig.getId(), retrievedSaTifSourceConfig.getVersion(), @@ -297,7 +301,7 @@ public void updateIocAndTIFSourceConfig( retrievedSaTifSourceConfig.getRefreshType(), Instant.now(), updatedByUser, - retrievedSaTifSourceConfig.isEnabled(), + isEnabled, retrievedSaTifSourceConfig.getIocStoreConfig(), retrievedSaTifSourceConfig.getIocTypes(), saTifSourceConfigDto.isEnabledForScan() // update only enabled_for_scan diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java index 39377262e..e79b9e989 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java @@ -35,6 +35,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -477,6 +478,14 @@ public void testActivateDeactivateIocUploadSourceConfig() throws IOException, In // update source config with hashes ioc type response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); Assert.assertEquals(RestStatus.OK, restStatus(response)); + Map updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + HashMap scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertFalse((Boolean) scr.get("enabled")); + assertTrue(scr.containsKey("enabled_for_scan")); + assertFalse((Boolean) scr.get("enabled_for_scan")); // Ensure that old ioc indices are retained (2 created from ioc upload source config + 1 from default source config) List findingIndices = getIocIndices(); @@ -494,6 +503,40 @@ public void testActivateDeactivateIocUploadSourceConfig() throws IOException, In iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); assertEquals(1, iocHits.size()); Thread.sleep(10000); + + saTifSourceConfigDto = new SATIFSourceConfigDto( + saTifSourceConfigDto.getId(), + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + Thread.sleep(10000); + // update source config with hashes ioc type + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertFalse((Boolean) scr.get("enabled")); // since its not url_download type, this flag should remain unaffected by the activate action in update source api + assertTrue(scr.containsKey("enabled_for_scan")); + assertTrue((Boolean) scr.get("enabled_for_scan")); } public void testActivateDeactivateUrlDownloadSourceConfig() throws IOException, InterruptedException { @@ -546,6 +589,14 @@ public void testActivateDeactivateUrlDownloadSourceConfig() throws IOException, // update default source config with enabled_for_scan updated Response response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); Assert.assertEquals(RestStatus.OK, restStatus(response)); + Map updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + HashMap scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertFalse((Boolean) scr.get("enabled")); + assertTrue(scr.containsKey("enabled_for_scan")); + assertFalse((Boolean) scr.get("enabled_for_scan")); // Ensure that only 1 ioc index is present from default source List findingIndices = getIocIndices(); @@ -559,6 +610,39 @@ public void testActivateDeactivateUrlDownloadSourceConfig() throws IOException, } catch (Exception e) { Assert.assertTrue(e.getMessage().contains("unsupported_operation_exception")); } + // activate source + saTifSourceConfigDto = new SATIFSourceConfigDto( + id, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + urlDownloadSource, + null, + null, + schedule, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // update default source config with enabled_for_scan updated + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertTrue((Boolean) scr.get("enabled")); + assertTrue(scr.containsKey("enabled_for_scan")); + assertTrue((Boolean) scr.get("enabled_for_scan")); } public void testDeleteIocUploadSourceConfigAndAllIocs() throws IOException {