Skip to content

Commit

Permalink
adds toggling refresh disable/enable for deactivate/activate operatio…
Browse files Browse the repository at this point in the history 
…n while updating URL_DOWNLOAD type configs

Signed-off-by: Surya Sashank Nistala <[email protected]>
  • Loading branch information
@eirsep
eirsep committed Aug 9, 2024
1 parent a9423f8 commit a19c303
Show file tree
Hide file tree
Showing 5 changed files with 108 additions and 47 deletions.
20 changes: 1 addition & 19 deletions ...ava/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,25 +63,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject()
.field(_ID, id)
.field(_VERSION, version);
builder.startObject("source_config")
.field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName())
.field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat())
.field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType())
.field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes())
.field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription())
.field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser())
.field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt())
.field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource())
.field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled())
.field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime())
.field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime())
.field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule())
.field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState())
.field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType())
.field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser())
.field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime());

builder.endObject();
saTifSourceConfigDto.innerXcontent(builder);
return builder.endObject();
}

Expand Down
26 changes: 6 additions & 20 deletions ...a/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import java.io.IOException;

import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto.SOURCE_CONFIG_FIELD;
import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID;
import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION;

Expand Down Expand Up @@ -56,40 +57,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject()
.field(_ID, id)
.field(_VERSION, version);

builder.startObject("source_config")
.field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName())
.field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat())
.field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType())
.field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes())
.field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription())
.field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser())
.field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt())
.field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource())
.field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled())
.field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime())
.field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime())
.field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule())
.field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState())
.field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType())
.field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser())
.field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime());

builder.endObject();
saTifSourceConfigDto.innerXcontent(builder);
return builder.endObject();
}

@Override
public String getTIFConfigId() {
return id;
}

@Override
public Long getVersion() {
return version;
}

@Override
public TIFSourceConfigDto getTIFConfigDto() {
return saTifSourceConfigDto;
}

public RestStatus getStatus() {
return status;
}
Expand Down
19 changes: 12 additions & 7 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,12 +215,18 @@ public void writeTo(final StreamOutput out) throws IOException {

@Override
public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
builder.startObject()
.startObject(SOURCE_CONFIG_FIELD)
.field(NAME_FIELD, name)
.field(FORMAT_FIELD, format)
.field(TYPE_FIELD, type.name())
.field(DESCRIPTION_FIELD, description);
builder.startObject();
innerXcontent(builder);
builder.endObject();
return builder;
}

public XContentBuilder innerXcontent(XContentBuilder builder) throws IOException {
builder.startObject(SOURCE_CONFIG_FIELD);
builder.field(NAME_FIELD, name)
.field(FORMAT_FIELD, format)
.field(TYPE_FIELD, type.name())
.field(DESCRIPTION_FIELD, description);
if (createdByUser == null) {
builder.nullField(CREATED_BY_USER_FIELD);
} else {
Expand Down Expand Up @@ -274,7 +280,6 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
builder.field(ENABLED_FOR_SCAN_FIELD, enabledForScan);
builder.field(IOC_TYPES_FIELD, iocTypes);
builder.endObject();
builder.endObject();
return builder;
}

Expand Down
6 changes: 5 additions & 1 deletion .../opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,10 @@ public void updateIocAndTIFSourceConfig(
// Due to the lack of a different API to do activate/deactivate we will check if enabled_for_scan variable is changed between model and request.
// If yes, we will ONLY update enabled_for_scan field and ignore any updates to the rest of the fields to simulate a dedicated activate/deactivate API.
if (retrievedSaTifSourceConfig.isEnabledForScan() != saTifSourceConfigDto.isEnabledForScan()) {
// FIXME add a disable_refresh api independent of update api so that it can be supported for default configs also
boolean isEnabled = URL_DOWNLOAD.equals(retrievedSaTifSourceConfig.getType()) ?
saTifSourceConfigDto.isEnabledForScan() :
retrievedSaTifSourceConfig.isEnabled();
SATIFSourceConfig config = new SATIFSourceConfig(
retrievedSaTifSourceConfig.getId(),
retrievedSaTifSourceConfig.getVersion(),
Expand All @@ -297,7 +301,7 @@ public void updateIocAndTIFSourceConfig(
retrievedSaTifSourceConfig.getRefreshType(),
Instant.now(),
updatedByUser,
retrievedSaTifSourceConfig.isEnabled(),
isEnabled,
retrievedSaTifSourceConfig.getIocStoreConfig(),
retrievedSaTifSourceConfig.getIocTypes(),
saTifSourceConfigDto.isEnabledForScan() // update only enabled_for_scan
Expand Down
84 changes: 84 additions & 0 deletions ...est/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
Expand Down Expand Up @@ -477,6 +478,14 @@ public void testActivateDeactivateIocUploadSourceConfig() throws IOException, In
// update source config with hashes ioc type
response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto));
Assert.assertEquals(RestStatus.OK, restStatus(response));
Map<String, Object> updateResponseAsMap = asMap(response);
assertNotNull(updateResponseAsMap);
assertTrue(updateResponseAsMap.containsKey("source_config"));
HashMap<String, Object> scr = (HashMap<String, Object>) updateResponseAsMap.get("source_config");
assertTrue(scr.containsKey("enabled"));
assertFalse((Boolean) scr.get("enabled"));
assertTrue(scr.containsKey("enabled_for_scan"));
assertFalse((Boolean) scr.get("enabled_for_scan"));

// Ensure that old ioc indices are retained (2 created from ioc upload source config + 1 from default source config)
List<String> findingIndices = getIocIndices();
Expand All @@ -494,6 +503,40 @@ public void testActivateDeactivateIocUploadSourceConfig() throws IOException, In
iocHits = (List<Map<String, Object>>) respMap.get(ListIOCsActionResponse.HITS_FIELD);
assertEquals(1, iocHits.size());
Thread.sleep(10000);

saTifSourceConfigDto = new SATIFSourceConfigDto(
saTifSourceConfigDto.getId(),
null,
feedName,
feedFormat,
sourceConfigType,
null,
null,
null,
iocUploadSource,
null,
null,
null,
null,
null,
null,
null,
enabled,
iocTypes, true
);

Thread.sleep(10000);
// update source config with hashes ioc type
response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto));
Assert.assertEquals(RestStatus.OK, restStatus(response));
updateResponseAsMap = asMap(response);
assertNotNull(updateResponseAsMap);
assertTrue(updateResponseAsMap.containsKey("source_config"));
scr = (HashMap<String, Object>) updateResponseAsMap.get("source_config");
assertTrue(scr.containsKey("enabled"));
assertFalse((Boolean) scr.get("enabled")); // since its not url_download type, this flag should remain unaffected by the activate action in update source api
assertTrue(scr.containsKey("enabled_for_scan"));
assertTrue((Boolean) scr.get("enabled_for_scan"));
}

public void testActivateDeactivateUrlDownloadSourceConfig() throws IOException, InterruptedException {
Expand Down Expand Up @@ -546,6 +589,14 @@ public void testActivateDeactivateUrlDownloadSourceConfig() throws IOException,
// update default source config with enabled_for_scan updated
Response response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto));
Assert.assertEquals(RestStatus.OK, restStatus(response));
Map<String, Object> updateResponseAsMap = asMap(response);
assertNotNull(updateResponseAsMap);
assertTrue(updateResponseAsMap.containsKey("source_config"));
HashMap<String, Object> scr = (HashMap<String, Object>) updateResponseAsMap.get("source_config");
assertTrue(scr.containsKey("enabled"));
assertFalse((Boolean) scr.get("enabled"));
assertTrue(scr.containsKey("enabled_for_scan"));
assertFalse((Boolean) scr.get("enabled_for_scan"));

// Ensure that only 1 ioc index is present from default source
List<String> findingIndices = getIocIndices();
Expand All @@ -559,6 +610,39 @@ public void testActivateDeactivateUrlDownloadSourceConfig() throws IOException,
} catch (Exception e) {
Assert.assertTrue(e.getMessage().contains("unsupported_operation_exception"));
}
// activate source
saTifSourceConfigDto = new SATIFSourceConfigDto(
id,
null,
feedName,
feedFormat,
sourceConfigType,
null,
null,
null,
urlDownloadSource,
null,
null,
schedule,
null,
null,
null,
null,
enabled,
iocTypes, true
);

// update default source config with enabled_for_scan updated
response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto));
Assert.assertEquals(RestStatus.OK, restStatus(response));
updateResponseAsMap = asMap(response);
assertNotNull(updateResponseAsMap);
assertTrue(updateResponseAsMap.containsKey("source_config"));
scr = (HashMap<String, Object>) updateResponseAsMap.get("source_config");
assertTrue(scr.containsKey("enabled"));
assertTrue((Boolean) scr.get("enabled"));
assertTrue(scr.containsKey("enabled_for_scan"));
assertTrue((Boolean) scr.get("enabled_for_scan"));
}

public void testDeleteIocUploadSourceConfigAndAllIocs() throws IOException {
Expand Down

0 comments on commit a19c303

Please sign in to comment.