diff --git a/build.gradle b/build.gradle index c21d74360..681287dbb 100644 --- a/build.gradle +++ b/build.gradle @@ -69,7 +69,7 @@ opensearchplugin { name 'opensearch-security-analytics' description 'OpenSearch Security Analytics plugin' classname 'org.opensearch.securityanalytics.SecurityAnalyticsPlugin' -// extendedPlugins = ['opensearch-job-scheduler'] + extendedPlugins = ['opensearch-job-scheduler'] } javaRestTest { @@ -143,12 +143,6 @@ repositories { sourceSets.main.java.srcDirs = ['src/main/generated','src/main/java'] configurations { zipArchive - - all { - resolutionStrategy { - force "com.google.guava:guava:32.0.1-jre" - } - } } dependencies { @@ -159,7 +153,7 @@ dependencies { api "org.opensearch:common-utils:${common_utils_version}@jar" api "org.opensearch.client:opensearch-rest-client:${opensearch_version}" implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" - implementation "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" + compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" implementation "org.apache.commons:commons-csv:1.10.0" // Needed for integ tests @@ -167,12 +161,6 @@ dependencies { zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" - - //spotless - implementation('com.google.googlejavaformat:google-java-format:1.17.0') { - exclude group: 'com.google.guava' - } - implementation 'com.google.guava:guava:32.0.1-jre' } // RPM & Debian build @@ -303,6 +291,12 @@ testClusters.integTest { } } })) + nodes.each { node -> + def plugins = node.plugins + def firstPlugin = plugins.get(0) + plugins.remove(0) + plugins.add(firstPlugin) + } } run { diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index c04189ad6..96970bac8 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -4,8 +4,12 @@ */ package org.opensearch.securityanalytics; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.function.Supplier; +import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.core.action.ActionListener; @@ -32,6 +36,9 @@ import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.mapper.Mapper; import org.opensearch.indices.SystemIndexDescriptor; +import org.opensearch.jobscheduler.spi.JobSchedulerExtension; +import org.opensearch.jobscheduler.spi.ScheduledJobParser; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; import org.opensearch.plugins.*; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; @@ -54,6 +61,7 @@ import org.opensearch.securityanalytics.threatIntel.action.*; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; @@ -68,13 +76,12 @@ import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.securityanalytics.util.RuleIndices; import org.opensearch.securityanalytics.util.RuleTopicIndices; -import org.opensearch.threadpool.ExecutorBuilder; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; -public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin { +public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension { private static final Logger log = LogManager.getLogger(SecurityAnalyticsPlugin.class); @@ -90,6 +97,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; + public static final String JOB_INDEX_NAME = ".opensearch-sap-threatintel-job"; + public static final Map TIF_JOB_INDEX_SETTING = Map.of("index.number_of_shards", 1, "index.auto_expand_replicas", "0-all", "index.hidden", true); private CorrelationRuleIndices correlationRuleIndices; @@ -117,6 +126,8 @@ public Collection getSystemIndexDescriptors(Settings sett return List.of(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); } + + @Override public Collection createComponents(Client client, ClusterService clusterService, @@ -147,7 +158,7 @@ public Collection createComponents(Client client, TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); - TIFJobRunner.getJobRunnerInstance().initialize(clusterService,tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); + TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, @@ -192,10 +203,31 @@ public List getRestHandlers(Settings settings, new RestSearchCorrelationRuleAction(), new RestIndexCustomLogTypeAction(), new RestSearchCustomLogTypeAction(), - new RestDeleteCustomLogTypeAction() + new RestDeleteCustomLogTypeAction(), + new RestPutTIFJobHandler(clusterSettings) ); } + @Override + public String getJobType() { + return "opensearch_sap_threatintel_job"; + } + + @Override + public String getJobIndex() { + return JOB_INDEX_NAME; + } + + @Override + public ScheduledJobRunner getJobRunner() { + return TIFJobRunner.getJobRunnerInstance(); + } + + @Override + public ScheduledJobParser getJobParser() { + return (parser, id, jobDocVersion) -> TIFJobParameter.PARSER.parse(parser, null); + } + @Override public List getNamedXContent() { return List.of( diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index f3e3b2f5d..d3a7b8b6f 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -122,8 +122,7 @@ public class SecurityAnalyticsSettings { // threat intel settings public static final Setting TIF_UPDATE_INTERVAL = Setting.timeSetting( "plugins.security_analytics.threatintel.tifjob.update_interval", - TimeValue.timeValueHours(24), - TimeValue.timeValueHours(1), + TimeValue.timeValueMinutes(1), Setting.Property.NodeScope, Setting.Property.Dynamic ); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java index 5ecff4b55..b82d16c20 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java @@ -52,6 +52,8 @@ import java.util.Arrays; import java.util.Optional; import java.util.concurrent.CountDownLatch; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; @@ -103,7 +105,7 @@ public void getThreatIntelFeedData( ) { try { //if index not exists - if(IndexUtils.getNewIndexByCreationDate( + if (IndexUtils.getNewIndexByCreationDate( this.clusterService.state(), this.indexNameExpressionResolver, ".opensearch-sap-threatintel*" @@ -129,7 +131,7 @@ public void getThreatIntelFeedData( listener.onFailure(e); } } - + private void createThreatIntelFeedData() throws InterruptedException { CountDownLatch countDownLatch = new CountDownLatch(1); client.execute(PutTIFJobAction.INSTANCE, new PutTIFJobRequest("feed_updater", clusterSettings.get(SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL))).actionGet(); @@ -138,7 +140,7 @@ private void createThreatIntelFeedData() throws InterruptedException { /** * Create an index for a threat intel feed - * + *

* Index setting start with single shard, zero replica, no refresh interval, and hidden. * Once the threat intel feed is indexed, do refresh and force merge. * Then, change the index setting to expand replica to all nodes, and read only allow delete. @@ -174,7 +176,7 @@ private String getIndexMapping() { * Puts threat intel feed from CSVRecord iterator into a given index in bulk * * @param indexName Index name to save the threat intel feed - * @param iterator TIF data to insert + * @param iterator TIF data to insert * @param renewLock Runnable to renew lock */ public void parseAndSaveThreatIntelFeedDataCSV( @@ -197,6 +199,10 @@ public void parseAndSaveThreatIntelFeedDataCSV( String iocType = tifMetadata.getIocType(); //todo make generic in upcoming versions Integer colNum = tifMetadata.getIocCol(); String iocValue = record.values()[colNum].split(" ")[0]; + if (iocType.equals("ip") && !isValidIp(iocValue)) { + log.info("Invalid IP address, skipping this ioc record."); + continue; + } String feedId = tifMetadata.getFeedId(); Instant timestamp = Instant.now(); ThreatIntelFeedData threatIntelFeedData = new ThreatIntelFeedData(iocType, iocValue, feedId, timestamp); @@ -218,8 +224,14 @@ public void parseAndSaveThreatIntelFeedDataCSV( freezeIndex(indexName); } - public void saveTifds(BulkRequest bulkRequest, TimeValue timeout) { + public static boolean isValidIp(String ip) { + String ipPattern = "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$"; + Pattern pattern = Pattern.compile(ipPattern); + Matcher matcher = pattern.matcher(ip); + return matcher.matches(); + } + public void saveTifds(BulkRequest bulkRequest, TimeValue timeout) { try { BulkResponse response = StashedThreadContext.run(client, () -> { return client.bulk(bulkRequest).actionGet(timeout); @@ -252,10 +264,6 @@ private void freezeIndex(final String indexName) { }); } - public void deleteThreatIntelDataIndex(final String index) { - deleteThreatIntelDataIndex(Arrays.asList(index)); - } - public void deleteThreatIntelDataIndex(final List indices) { if (indices == null || indices.isEmpty()) { return; @@ -286,10 +294,4 @@ public void deleteThreatIntelDataIndex(final List indices) { throw new OpenSearchException("failed to delete data[{}]", String.join(",", indices)); } } - public static class ThreatIntelFeedUpdateHandler implements Runnable { - - @Override - public void run() { - - } - }} +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java index fa1587a66..5f58e5529 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java @@ -45,10 +45,6 @@ public TimeValue getUpdateInterval() { return this.updateInterval; } - public void setUpdateInterval(TimeValue timeValue) { - this.updateInterval = timeValue; - } - /** * Default constructor * @param name name of a tif job diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/RestPutTIFJobHandler.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/RestPutTIFJobHandler.java new file mode 100644 index 000000000..641445a57 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/RestPutTIFJobHandler.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.opensearch.rest.RestRequest.Method.GET; +import static org.opensearch.rest.RestRequest.Method.PUT; + +/** + * Rest handler for threat intel TIFjob creation + * + * This handler handles a request of + * PUT /_plugins/security_analytics/threatintel/tifjob/{id} + * { + * "id": {id}, + * "name": {name}, + * "update_interval_in_days": 1 + * } + * + * When request is received, it will create a TIFjob + * After the creation of TIFjob is completed, it will schedule the next update task after update_interval_in_days. + * + */ +public class RestPutTIFJobHandler extends BaseRestHandler { + private static final String ACTION_NAME = "threatintel_tifjob_put"; + private final ClusterSettings clusterSettings; + + public RestPutTIFJobHandler(final ClusterSettings clusterSettings) { + this.clusterSettings = clusterSettings; + } + + @Override + public String getName() { + return ACTION_NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final PutTIFJobRequest putTIFJobRequest = new PutTIFJobRequest("jobname", + new TimeValue(1, TimeUnit.MINUTES)); + + return channel -> client.executeLocally(PutTIFJobAction.INSTANCE, putTIFJobRequest, new RestToXContentListener<>(channel)); + } + + @Override + public List routes() { + String path = "/_p/_s"; + return List.of(new Route(GET, path)); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobAction.java index 45fc037d8..3a0c68f10 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobAction.java @@ -85,7 +85,6 @@ protected void doExecute(final Task task, final DeleteTIFJobRequest request, fin return; } try { - // TODO: makes every sub-methods as async call to avoid using a thread in generic pool threadPool.generic().submit(() -> { try { deleteTIFJob(request.getName()); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java index 386fec0c3..7ec4e94f3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java @@ -5,7 +5,8 @@ package org.opensearch.securityanalytics.threatIntel.common; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobExtension.JOB_INDEX_NAME; +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; + import java.time.Instant; import java.util.Optional; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtension.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtension.java deleted file mode 100644 index 023323253..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtension.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.jobscheduler; - -import org.opensearch.jobscheduler.spi.ScheduledJobParser; -import org.opensearch.jobscheduler.spi.ScheduledJobRunner; - -import java.util.Map; - -public class TIFJobExtension implements org.opensearch.jobscheduler.spi.JobSchedulerExtension { - /** - * Job index name for a TIF job - */ - public static final String JOB_INDEX_NAME = ".scheduler-sap-threatintel-job"; - - /** - * Job index setting - * - * We want it to be single shard so that job can be run only in a single node by job scheduler. - * We want it to expand to all replicas so that querying to this index can be done locally to reduce latency. - */ - public static final Map INDEX_SETTING = Map.of("index.number_of_shards", 1, "index.auto_expand_replicas", "0-all", "index.hidden", true); - - @Override - public String getJobType() { - return "scheduler_sap_threatintel_job"; - } - - @Override - public String getJobIndex() { - return JOB_INDEX_NAME; - } - - @Override - public ScheduledJobRunner getJobRunner() { - return TIFJobRunner.getJobRunnerInstance(); - } - - @Override - public ScheduledJobParser getJobParser() { - return (parser, id, jobDocVersion) -> TIFJobParameter.PARSER.parse(parser, null); - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java index 0a24ffb75..115ed873e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java @@ -12,12 +12,14 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.xcontent.ConstructingObjectParser; -import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.*; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; +import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; import java.io.IOException; import java.time.Instant; @@ -29,35 +31,45 @@ import static org.opensearch.common.time.DateUtils.toInstant; -import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; -import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; - public class TIFJobParameter implements Writeable, ScheduledJobParameter { /** * Prefix of indices having threatIntel data */ public static final String THREAT_INTEL_DATA_INDEX_NAME_PREFIX = ".opensearch-sap-threatintel"; + + /** + * String fields for job scheduling parameters used for ParseField + */ + private static final String name_field = "name"; + private static final String enabled_field = "update_enabled"; + private static final String last_update_time_field = "last_update_time"; + private static final String last_update_time_field_readable = "last_update_time_field"; + private static final String schedule_field = "schedule"; + private static final String enabled_time_field = "enabled_time"; + private static final String enabled_time_field_readable = "enabled_time_field"; + private static final String state_field = "state"; + private static final String indices_field = "indices"; + private static final String update_stats_field = "update_stats"; + + /** * Default fields for job scheduling */ - private static final ParseField NAME_FIELD = new ParseField("name"); - private static final ParseField ENABLED_FIELD = new ParseField("update_enabled"); - private static final ParseField LAST_UPDATE_TIME_FIELD = new ParseField("last_update_time"); - private static final ParseField LAST_UPDATE_TIME_FIELD_READABLE = new ParseField("last_update_time_field"); - public static final ParseField SCHEDULE_FIELD = new ParseField("schedule"); - private static final ParseField ENABLED_TIME_FIELD = new ParseField("enabled_time"); - private static final ParseField ENABLED_TIME_FIELD_READABLE = new ParseField("enabled_time_field"); + public static final ParseField NAME_FIELD = new ParseField(name_field); + public static final ParseField ENABLED_FIELD = new ParseField(enabled_field); + public static final ParseField LAST_UPDATE_TIME_FIELD = new ParseField(last_update_time_field); + public static final ParseField LAST_UPDATE_TIME_FIELD_READABLE = new ParseField(last_update_time_field_readable); + public static final ParseField SCHEDULE_FIELD = new ParseField(schedule_field); + public static final ParseField ENABLED_TIME_FIELD = new ParseField(enabled_time_field); + public static final ParseField ENABLED_TIME_FIELD_READABLE = new ParseField(enabled_time_field_readable); /** * Additional fields for tif job */ - private static final ParseField STATE_FIELD = new ParseField("state"); - private static final ParseField INDICES_FIELD = new ParseField("indices"); - private static final ParseField UPDATE_STATS_FIELD = new ParseField("update_stats"); - + public static final ParseField STATE_FIELD = new ParseField(state_field); + public static final ParseField INDICES_FIELD = new ParseField(indices_field); + public static final ParseField UPDATE_STATS_FIELD = new ParseField(update_stats_field); /** * Default variables for job scheduling @@ -113,6 +125,61 @@ public class TIFJobParameter implements Writeable, ScheduledJobParameter { */ private UpdateStats updateStats; + public static TIFJobParameter parse(XContentParser xcp, String id, Long version) throws IOException { + String name = null; + Instant lastUpdateTime = null; + Boolean isEnabled = null; + TIFJobState state = null; + + xcp.nextToken(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case name_field: + name = xcp.text(); + break; + case last_update_time_field: + lastUpdateTime = Instant.ofEpochMilli(xcp.longValue()); + break; + case enabled_field: + isEnabled = xcp.booleanValue(); + break; + case state_field: + state = toState(xcp.text()); + break; + default: + xcp.skipChildren(); + } + } + return new TIFJobParameter(name, lastUpdateTime, isEnabled, state); + } + + public static TIFJobState toState(String stateName){ + if (stateName.equals("CREATING")){ + return TIFJobState.CREATING; + } + if (stateName.equals("AVAILABLE")){ + return TIFJobState.AVAILABLE; + } + if (stateName.equals("CREATE_FAILED")){ + return TIFJobState.CREATE_FAILED; + } + if (stateName.equals("DELETING")){ + return TIFJobState.DELETING; + } + return null; + } + + public TIFJobParameter(final String name, final Instant lastUpdateTime, final Boolean isEnabled, TIFJobState state) { + this.name = name; + this.lastUpdateTime = lastUpdateTime; + this.isEnabled = isEnabled; + this.state = state; + } + /** * tif job parser */ @@ -174,7 +241,7 @@ public TIFJobParameter(final String name, final IntervalSchedule schedule) { name, Instant.now().truncatedTo(ChronoUnit.MILLIS), null, - true, + false, schedule, TIFJobState.CREATING, new ArrayList<>(), @@ -479,8 +546,8 @@ public static TIFJobParameter build(final PutTIFJobRequest request) { String name = request.getName(); IntervalSchedule schedule = new IntervalSchedule( Instant.now().truncatedTo(ChronoUnit.MILLIS), - (int) request.getUpdateInterval().hours(), - ChronoUnit.DAYS + (int) request.getUpdateInterval().minutes(), + ChronoUnit.MINUTES ); return new TIFJobParameter(name, schedule); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java index 70f052549..b977cb4ba 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java @@ -31,6 +31,7 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexNotFoundException; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -64,12 +65,12 @@ public TIFJobParameterService(final Client client, final ClusterService clusterS * @param stepListener setup listener */ public void createJobIndexIfNotExists(final StepListener stepListener) { - if (clusterService.state().metadata().hasIndex(TIFJobExtension.JOB_INDEX_NAME) == true) { + if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == true) { stepListener.onResponse(null); return; } - final CreateIndexRequest createIndexRequest = new CreateIndexRequest(TIFJobExtension.JOB_INDEX_NAME).mapping(getIndexMapping()) - .settings(TIFJobExtension.INDEX_SETTING); + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); StashedThreadContext.run(client, () -> client.admin().indices().create(createIndexRequest, new ActionListener<>() { @Override public void onResponse(final CreateIndexResponse createIndexResponse) { @@ -79,7 +80,7 @@ public void onResponse(final CreateIndexResponse createIndexResponse) { @Override public void onFailure(final Exception e) { if (e instanceof ResourceAlreadyExistsException) { - log.info("index[{}] already exist", TIFJobExtension.JOB_INDEX_NAME); + log.info("index[{}] already exist", SecurityAnalyticsPlugin.JOB_INDEX_NAME); stepListener.onResponse(null); return; } @@ -97,7 +98,7 @@ private String getIndexMapping() { } } catch (IOException e) { log.error("Runtime exception", e); - throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO + throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); } } @@ -110,7 +111,7 @@ public IndexResponse updateJobSchedulerParameter(final TIFJobParameter jobSchedu jobSchedulerParameter.setLastUpdateTime(Instant.now()); return StashedThreadContext.run(client, () -> { try { - return client.prepareIndex(TIFJobExtension.JOB_INDEX_NAME) + return client.prepareIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) .setId(jobSchedulerParameter.getName()) .setOpType(DocWriteRequest.OpType.INDEX) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) @@ -118,7 +119,7 @@ public IndexResponse updateJobSchedulerParameter(final TIFJobParameter jobSchedu .execute() .actionGet(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT)); } catch (IOException e) { - throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO + throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); } }); } @@ -130,16 +131,16 @@ public IndexResponse updateJobSchedulerParameter(final TIFJobParameter jobSchedu * @throws IOException exception */ public TIFJobParameter getJobParameter(final String name) throws IOException { - GetRequest request = new GetRequest(TIFJobExtension.JOB_INDEX_NAME, name); + GetRequest request = new GetRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME, name); GetResponse response; try { response = StashedThreadContext.run(client, () -> client.get(request).actionGet(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT))); if (response.isExists() == false) { - log.error("TIF job[{}] does not exist in an index[{}]", name, TIFJobExtension.JOB_INDEX_NAME); + log.error("TIF job[{}] does not exist in an index[{}]", name, SecurityAnalyticsPlugin.JOB_INDEX_NAME); return null; } } catch (IndexNotFoundException e) { - log.error("Index[{}] is not found", TIFJobExtension.JOB_INDEX_NAME); + log.error("Index[{}] is not found", SecurityAnalyticsPlugin.JOB_INDEX_NAME); return null; } @@ -161,7 +162,7 @@ public void saveTIFJobParameter(final TIFJobParameter tifJobParameter, final Act tifJobParameter.setLastUpdateTime(Instant.now()); StashedThreadContext.run(client, () -> { try { - client.prepareIndex(TIFJobExtension.JOB_INDEX_NAME) + client.prepareIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) .setId(tifJobParameter.getName()) .setOpType(DocWriteRequest.OpType.CREATE) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) @@ -181,7 +182,7 @@ public void saveTIFJobParameter(final TIFJobParameter tifJobParameter, final Act */ public void deleteTIFJobParameter(final TIFJobParameter tifJobParameter) { DeleteResponse response = client.prepareDelete() - .setIndex(TIFJobExtension.JOB_INDEX_NAME) + .setIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) .setId(tifJobParameter.getName()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .execute() diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java index fdb07618d..e3500064f 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java @@ -150,12 +150,9 @@ protected void updateJobParameter(final ScheduledJobParameter jobParameter, fina } try { // create new TIF data and delete old ones - Instant startTime = Instant.now(); List oldIndices = new ArrayList<>(jobSchedulerParameter.getIndices()); List newFeedIndices = jobSchedulerUpdateService.createThreatIntelFeedData(jobSchedulerParameter, renewLock); - Instant endTime = Instant.now(); jobSchedulerUpdateService.deleteAllTifdIndices(oldIndices, newFeedIndices); - jobSchedulerUpdateService.updateJobSchedulerParameterAsSucceeded(newFeedIndices, jobSchedulerParameter, startTime, endTime); if(false == newFeedIndices.isEmpty()) { detectorThreatIntelService.updateDetectorsWithLatestThreatIntelRules(); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java index 45ad50b35..3006285ad 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java @@ -99,6 +99,8 @@ private List deleteIndices(final List indicesToDelete) { * @throws IOException */ public List createThreatIntelFeedData(final TIFJobParameter jobSchedulerParameter, final Runnable renewLock) throws IOException { + Instant startTime = Instant.now(); + List freshIndices = new ArrayList<>(); for (TIFMetadata tifMetadata : builtInTIFMetadataLoader.getTifMetadataList()) { String indexName = setupIndex(jobSchedulerParameter, tifMetadata); @@ -140,6 +142,8 @@ public List createThreatIntelFeedData(final TIFJobParameter jobScheduler } freshIndices.add(indexName); } + Instant endTime = Instant.now(); + updateJobSchedulerParameterAsSucceeded(freshIndices, jobSchedulerParameter, startTime, endTime); return freshIndices; } @@ -163,7 +167,7 @@ public void updateJobSchedulerParameterAsSucceeded( jobSchedulerParameter.setState(TIFJobState.AVAILABLE); jobSchedulerParameterService.updateJobSchedulerParameter(jobSchedulerParameter); log.info( - "threat intel feed database creation succeeded for {} and took {} seconds", + "threat intel feed data creation succeeded for {} and took {} seconds", jobSchedulerParameter.getName(), Duration.between(startTime, endTime) ); @@ -205,7 +209,7 @@ protected void waitUntilAllShardsStarted(final String indexName, final int timeo ); } catch (InterruptedException e) { log.error("runtime exception", e); - throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO + throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); } } } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java index 53ef22a76..0643b34d7 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java @@ -21,12 +21,15 @@ import org.opensearch.securityanalytics.action.SearchDetectorAction; import org.opensearch.securityanalytics.action.SearchDetectorRequest; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.TransportPutTIFJobAction; import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.threadpool.ThreadPool; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; +import java.util.concurrent.CountDownLatch; + import static org.opensearch.securityanalytics.util.DetectorUtils.getEmptySearchResponse; public class TransportSearchDetectorAction extends HandledTransportAction implements SecureTransportAction { @@ -45,11 +48,13 @@ public class TransportSearchDetectorAction extends HandledTransportAction() { @Override public void onResponse(SearchResponse response) { - actionListener.onResponse(response); + actionListener.onResponse(response); } @Override diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 55fea4224..e5d71024b 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -46,7 +46,7 @@ import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobExtension.JOB_INDEX_NAME; +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { /** diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java index a3df0c4cd..21501a796 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java @@ -11,10 +11,15 @@ import org.apache.hc.core5.http.HttpStatus; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; @@ -22,13 +27,17 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; import java.io.IOException; +import java.time.Instant; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Locale; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.opensearch.securityanalytics.TestHelpers.*; @@ -38,9 +47,9 @@ public class ThreatIntelJobRunnerIT extends SecurityAnalyticsRestTestCase { private static final Logger log = LogManager.getLogger(ThreatIntelJobRunnerIT.class); - public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatIntel() throws IOException { + public void testCreateDetector_threatIntelEnabled_testJobRunner() throws IOException, InterruptedException { - // 1. create a detector + // Create a detector updateClusterSetting(ENABLE_WORKFLOW_USAGE.getKey(), "true"); String index = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -102,46 +111,117 @@ public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatInt List iocs = getThreatIntelFeedIocs(3); assertEquals(iocs.size(),3); - // 2. delete a threat intel feed ioc index manually - List feedId = getThreatIntelFeedIds(1); - for (String feedid: feedId) { - String name = String.format(Locale.ROOT, "%s-%s%s", ".opensearch-sap-threatintel", feedid, "1"); - deleteIndex(name); + // get job runner index and verify parameters exist + List jobMetaDataList = getJobSchedulerParameter(); + assertEquals(1, jobMetaDataList.size()); + TIFJobParameter jobMetaData = jobMetaDataList.get(0); + Instant firstUpdatedTime = jobMetaData.getLastUpdateTime(); + assertNotNull("Job runner parameter index does not have metadata set", jobMetaData.getLastUpdateTime()); + assertEquals(jobMetaData.isEnabled(), true); + + // get list of first updated time for threat intel feed data + List originalFeedTimestamp = getThreatIntelFeedsTime(); + + //verify feed index exists and each feed_id exists + List feedId = getThreatIntelFeedIds(); + assertNotNull(feedId); + + // wait for job runner to run + Thread.sleep(60000); + waitUntil(() -> { + try { + return verifyJobRan(firstUpdatedTime); + } catch (IOException e) { + throw new RuntimeException("failed to verify that job ran"); + } + }, 120, TimeUnit.SECONDS); + + // verify job's last update time is different + List newJobMetaDataList = getJobSchedulerParameter(); + assertEquals(1, newJobMetaDataList.size()); + TIFJobParameter newJobMetaData = newJobMetaDataList.get(0); + Instant lastUpdatedTime = newJobMetaData.getLastUpdateTime(); + assertNotEquals(firstUpdatedTime.toString(), lastUpdatedTime.toString()); + + // verify new threat intel feed timestamp is different + List newFeedTimestamp = getThreatIntelFeedsTime(); + for (int i =0; i< newFeedTimestamp.size(); i++) { + assertNotEquals(newFeedTimestamp.get(i), originalFeedTimestamp.get(i)); } -// // 3. update the start time to a day before so it runs now -// StringEntity stringEntity = new StringEntity( -// "{\"doc\":{\"last_update_time\":{\"schedule\":{\"interval\":{\"start_time\":" + -// "\"$startTimeMillis\"}}}}}", -// ContentType.APPLICATION_JSON -// ); -// -// Response updateJobRespose = makeRequest(client(), "POST", ".scheduler-sap-threatintel-job/_update/$id" , Collections.emptyMap(), stringEntity, null, null); -// assertEquals("Updated job scheduler", RestStatus.CREATED, restStatus(updateJobRespose)); - - // 4. validate new ioc is created - List newIocs = getThreatIntelFeedIocs(1); - assertEquals(0, newIocs.size()); //TODO + // verify detector is updated by checking last updated time of detector + // TODO + + } + + protected boolean verifyJobRan(Instant firstUpdatedTime) throws IOException { + // verify job's last update time is different + List newJobMetaDataList = getJobSchedulerParameter(); + assertEquals(1, newJobMetaDataList.size()); + + TIFJobParameter newJobMetaData = newJobMetaDataList.get(0); + Instant newUpdatedTime = newJobMetaData.getLastUpdateTime(); + if (!firstUpdatedTime.toString().equals(newUpdatedTime.toString())){ + return true; + } + return false; } private List getThreatIntelFeedIocs(int num) throws IOException { - String request = getMatchAllSearchRequestString(num); + String request = getMatchNumSearchRequestString(num); SearchResponse res = executeSearchAndGetResponse(".opensearch-sap-threatintel*", request, false); return getTifdList(res, xContentRegistry()).stream().map(it -> it.getIocValue()).collect(Collectors.toList()); } - private List getThreatIntelFeedIds(int num) throws IOException { - String request = getMatchAllSearchRequestString(num); + private List getThreatIntelFeedIds() throws IOException { + String request = getMatchAllSearchRequestString(); SearchResponse res = executeSearchAndGetResponse(".opensearch-sap-threatintel*", request, false); return getTifdList(res, xContentRegistry()).stream().map(it -> it.getFeedId()).collect(Collectors.toList()); } -// private String getJobSchedulerDoc(int num) throws IOException { -// String request = getMatchAllSearchRequestString(num); -// SearchResponse res = executeSearchAndGetResponse(".scheduler-sap-threatintel-job*", request, false); -// } + private List getThreatIntelFeedsTime() throws IOException { + String request = getMatchAllSearchRequestString(); + SearchResponse res = executeSearchAndGetResponse(".opensearch-sap-threatintel*", request, false); + return getTifdList(res, xContentRegistry()).stream().map(it -> it.getTimestamp()).collect(Collectors.toList()); + } + + private List getJobSchedulerParameter() throws IOException { + String request = getMatchAllSearchRequestString(); + SearchResponse res = executeSearchAndGetResponse(".opensearch-sap-threatintel-job*", request, false); + return getTIFJobParameterList(res, xContentRegistry()).stream().collect(Collectors.toList()); + } + public static List getTIFJobParameterList(SearchResponse searchResponse, NamedXContentRegistry xContentRegistry) { + List list = new ArrayList<>(); + if (searchResponse.getHits().getHits().length != 0) { + Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> { + try { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString() + ); + list.add(TIFJobParameter.parse(xcp, hit.getId(), hit.getVersion())); + } catch (Exception e) { + log.error(() -> new ParameterizedMessage( + "Failed to parse TIF Job Parameter metadata from hit {}", hit), + e + ); + } + + }); + } + return list; + } + + private static String getMatchAllSearchRequestString() { + return "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + } - private static String getMatchAllSearchRequestString(int num) { + private static String getMatchNumSearchRequestString(int num) { return "{\n" + "\"size\" : " + num + "," + " \"query\" : {\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtensionTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtensionTests.java deleted file mode 100644 index 6096fa382..000000000 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtensionTests.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.jobscheduler; - -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobExtension.JOB_INDEX_NAME; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.jobscheduler.spi.JobDocVersion; -import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.TestHelpers; - -public class TIFJobExtensionTests extends ThreatIntelTestCase { - private static final Logger log = LogManager.getLogger(TIFJobExtensionTests.class); - - public void testBasic() { - TIFJobExtension extension = new TIFJobExtension(); - assertEquals("scheduler_sap_threatintel_job", extension.getJobType()); - assertEquals(JOB_INDEX_NAME, extension.getJobIndex()); - assertEquals(TIFJobRunner.getJobRunnerInstance(), extension.getJobRunner()); - } - - public void testParser() throws Exception { - TIFJobExtension extension = new TIFJobExtension(); - String id = TestHelpers.randomLowerCaseString(); - IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); - TIFJobParameter tifJobParameter = new TIFJobParameter(id, schedule); - - TIFJobParameter anotherTIFJobParameter = (TIFJobParameter) extension.getJobParser() - .parse( - createParser(tifJobParameter.toXContent(XContentFactory.jsonBuilder(), null)), - TestHelpers.randomLowerCaseString(), - new JobDocVersion(randomPositiveLong(), randomPositiveLong(), randomPositiveLong()) - ); - log.info("first"); - log.error(tifJobParameter); - log.error(tifJobParameter.getName()); - log.info("second"); - log.error(anotherTIFJobParameter); - log.error(anotherTIFJobParameter.getName()); - - assertTrue(tifJobParameter.getName().equals(anotherTIFJobParameter.getName())); - assertTrue(tifJobParameter.getLastUpdateTime().equals(anotherTIFJobParameter.getLastUpdateTime())); - assertTrue(tifJobParameter.getSchedule().equals(anotherTIFJobParameter.getSchedule())); - assertTrue(tifJobParameter.getState().equals(anotherTIFJobParameter.getState())); - assertTrue(tifJobParameter.getIndices().equals(anotherTIFJobParameter.getIndices())); - } - -} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterServiceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterServiceTests.java index 5b0605d79..35fd2450d 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterServiceTests.java @@ -23,6 +23,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.index.IndexNotFoundException; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; import org.opensearch.securityanalytics.TestHelpers; @@ -42,7 +43,7 @@ public void init() { } public void testcreateJobIndexIfNotExists_whenIndexExist_thenCreateRequestIsNotCalled() { - when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(true); + when(metadata.hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME)).thenReturn(true); // Verify verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { throw new RuntimeException("Shouldn't get called"); }); @@ -56,13 +57,13 @@ public void testcreateJobIndexIfNotExists_whenIndexExist_thenCreateRequestIsNotC } public void testcreateJobIndexIfNotExists_whenIndexExist_thenCreateRequestIsCalled() { - when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(false); + when(metadata.hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME)).thenReturn(false); // Verify verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { assertTrue(actionRequest instanceof CreateIndexRequest); CreateIndexRequest request = (CreateIndexRequest) actionRequest; - assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + assertEquals(SecurityAnalyticsPlugin.JOB_INDEX_NAME, request.index()); assertEquals("1", request.settings().get("index.number_of_shards")); assertEquals("0-all", request.settings().get("index.auto_expand_replicas")); assertEquals("true", request.settings().get("index.hidden")); @@ -79,9 +80,9 @@ public void testcreateJobIndexIfNotExists_whenIndexExist_thenCreateRequestIsCall } public void testcreateJobIndexIfNotExists_whenIndexCreatedAlready_thenExceptionIsIgnored() { - when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(false); + when(metadata.hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME)).thenReturn(false); verifyingClient.setExecuteVerifier( - (actionResponse, actionRequest) -> { throw new ResourceAlreadyExistsException(TIFJobExtension.JOB_INDEX_NAME); } + (actionResponse, actionRequest) -> { throw new ResourceAlreadyExistsException(SecurityAnalyticsPlugin.JOB_INDEX_NAME); } ); // Run @@ -93,7 +94,7 @@ public void testcreateJobIndexIfNotExists_whenIndexCreatedAlready_thenExceptionI } public void testcreateJobIndexIfNotExists_whenExceptionIsThrown_thenExceptionIsThrown() { - when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(false); + when(metadata.hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME)).thenReturn(false); verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { throw new RuntimeException(); }); // Run @@ -118,7 +119,7 @@ public void testUpdateTIFJobParameter_whenValidInput_thenSucceed() throws Except IndexRequest request = (IndexRequest) actionRequest; assertEquals(tifJobParameter.getName(), request.id()); assertEquals(DocWriteRequest.OpType.INDEX, request.opType()); - assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + assertEquals(SecurityAnalyticsPlugin.JOB_INDEX_NAME, request.index()); assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, request.getRefreshPolicy()); return null; }); @@ -135,7 +136,7 @@ public void testsaveTIFJobParameter_whenValidInput_thenSucceed() { verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { assertTrue(actionRequest instanceof IndexRequest); IndexRequest indexRequest = (IndexRequest) actionRequest; - assertEquals(TIFJobExtension.JOB_INDEX_NAME, indexRequest.index()); + assertEquals(SecurityAnalyticsPlugin.JOB_INDEX_NAME, indexRequest.index()); assertEquals(tifJobParameter.getName(), indexRequest.id()); assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, indexRequest.getRefreshPolicy()); assertEquals(DocWriteRequest.OpType.CREATE, indexRequest.opType()); @@ -147,7 +148,7 @@ public void testsaveTIFJobParameter_whenValidInput_thenSucceed() { } public void testGetTifJobParameter_whenException_thenNull() throws Exception { - TIFJobParameter tifJobParameter = setupClientForGetRequest(true, new IndexNotFoundException(TIFJobExtension.JOB_INDEX_NAME)); + TIFJobParameter tifJobParameter = setupClientForGetRequest(true, new IndexNotFoundException(SecurityAnalyticsPlugin.JOB_INDEX_NAME)); assertNull(tifJobParameterService.getJobParameter(tifJobParameter.getName())); } @@ -174,7 +175,7 @@ private TIFJobParameter setupClientForGetRequest(final boolean isExist, final Ru assertTrue(actionRequest instanceof GetRequest); GetRequest request = (GetRequest) actionRequest; assertEquals(tifJobParameter.getName(), request.id()); - assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + assertEquals(SecurityAnalyticsPlugin.JOB_INDEX_NAME, request.index()); GetResponse response = getMockedGetResponse(isExist ? tifJobParameter : null); if (exception != null) { throw exception; @@ -190,7 +191,7 @@ public void testDeleteTifJobParameter_whenValidInput_thenSucceed() { // Verify assertTrue(actionRequest instanceof DeleteRequest); DeleteRequest request = (DeleteRequest) actionRequest; - assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + assertEquals(SecurityAnalyticsPlugin.JOB_INDEX_NAME, request.index()); assertEquals(DocWriteRequest.OpType.DELETE, request.opType()); assertEquals(tifJobParameter.getName(), request.id()); assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, request.getRefreshPolicy()); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java index 85aeef5b9..f7b7ff8d1 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java @@ -53,7 +53,7 @@ public void testParser_whenAllValueIsFilled_thenSucceed() throws IOException { } - public void testParser_whenNullForOptionalFields_thenSucceed() throws IOException { // TODO: same issue + public void testParser_whenNullForOptionalFields_thenSucceed() throws IOException { String id = TestHelpers.randomLowerCaseString(); IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); TIFJobParameter tifJobParameter = new TIFJobParameter(id, schedule);