services,
+ @JsonProperty(TOTAL) long total) {
+ return new AutoValue_AvailableAWSServiceSummmary(services, total);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/graylog/integrations/aws/resources/responses/KinesisHealthCheckResponse.java b/src/main/java/org/graylog/integrations/aws/resources/responses/KinesisHealthCheckResponse.java
new file mode 100644
index 000000000..0d358b869
--- /dev/null
+++ b/src/main/java/org/graylog/integrations/aws/resources/responses/KinesisHealthCheckResponse.java
@@ -0,0 +1,30 @@
+package org.graylog.integrations.aws.resources.responses;
+
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+import org.graylog.autovalue.WithBeanGetter;
+
+@JsonAutoDetect
+@AutoValue
+@WithBeanGetter
+public abstract class KinesisHealthCheckResponse {
+
+ @JsonProperty
+ public abstract boolean success();
+
+ // Eg. CloudWatch, other.
+ @JsonProperty
+ public abstract String logType();
+
+ // Some specific success or error message from AWS SDK.
+ @JsonProperty
+ public abstract String message();
+
+ public static KinesisHealthCheckResponse create(@JsonProperty("success") boolean success,
+ @JsonProperty("log_type") String logType,
+ @JsonProperty("message") String message) {
+ return new AutoValue_KinesisHealthCheckResponse(success, logType, message);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/graylog/integrations/aws/resources/responses/RegionResponse.java b/src/main/java/org/graylog/integrations/aws/resources/responses/RegionResponse.java
new file mode 100644
index 000000000..1fea660ab
--- /dev/null
+++ b/src/main/java/org/graylog/integrations/aws/resources/responses/RegionResponse.java
@@ -0,0 +1,31 @@
+package org.graylog.integrations.aws.resources.responses;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+import org.graylog.autovalue.WithBeanGetter;
+
+@JsonAutoDetect
+@AutoValue
+@WithBeanGetter
+public abstract class RegionResponse {
+
+ // eu-west-2
+ @JsonProperty
+ public abstract String regionId();
+
+ // EU (London)
+ @JsonProperty
+ public abstract String regionDescription();
+
+ // The combination of both the name and description for display in the UI:
+ // EU (London): eu-west-2
+ @JsonProperty
+ public abstract String displayValue();
+
+ public static RegionResponse create(@JsonProperty("region_id") String regionId,
+ @JsonProperty("region_description") String regionDescription,
+ @JsonProperty("display_value") String displayValue ) {
+ return new AutoValue_RegionResponse(regionId, regionDescription, displayValue);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/graylog/integrations/aws/service/AWSLogMessage.java b/src/main/java/org/graylog/integrations/aws/service/AWSLogMessage.java
new file mode 100644
index 000000000..25e0de184
--- /dev/null
+++ b/src/main/java/org/graylog/integrations/aws/service/AWSLogMessage.java
@@ -0,0 +1,45 @@
+package org.graylog.integrations.aws.service;
+
+/**
+ * Supports the ability to automatically parse
+ */
+public class AWSLogMessage {
+
+ private static final String ACTION_ACCEPT = "ACCEPT";
+ private static final String ACTION_REJECT = "REJECT";
+
+ private String logMessage;
+
+ public AWSLogMessage(String logMessage) {
+ this.logMessage = logMessage;
+ }
+
+ /**
+ * Detects the type of log message.
+ *
+ * @return
+ */
+ public Type messageType() {
+
+ // AWS Flow Logs
+ // 2 123456789010 eni-abc123de 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK
+
+ // Not using a regex here, because it would be quite complicated and hard to maintain.
+ // Performance should not be an issue here, because this will only be executed once when detecting a log message.
+ if ((logMessage.contains(ACTION_ACCEPT) || logMessage.contains(ACTION_REJECT)) &&
+ logMessage.chars().filter(Character::isSpaceChar).count() == 13) {
+ return Type.FLOW_LOGS;
+ }
+
+ // Add more log message types here as needed
+
+ return Type.UNKNOWN;
+ }
+
+ // One enum value should be added for each type of log message that auto-detect is supported for.
+ public enum Type {
+
+ FLOW_LOGS, // See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html
+ UNKNOWN
+ }
+}
diff --git a/src/main/java/org/graylog/integrations/aws/service/AWSService.java b/src/main/java/org/graylog/integrations/aws/service/AWSService.java
new file mode 100644
index 000000000..ac5b98751
--- /dev/null
+++ b/src/main/java/org/graylog/integrations/aws/service/AWSService.java
@@ -0,0 +1,92 @@
+package org.graylog.integrations.aws.service;
+
+import org.graylog.integrations.aws.resources.responses.AvailableAWSService;
+import org.graylog.integrations.aws.resources.responses.AvailableAWSServiceSummmary;
+import org.graylog.integrations.aws.resources.responses.RegionResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.regions.RegionMetadata;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Service for all AWS CloudWatch business logic.
+ *
+ * This layer should not directly use the AWS SDK. All SDK operations should be performed in AWSClient.
+ */
+public class AWSService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AWSService.class);
+
+ /**
+ * @return A list of all available regions.
+ */
+ public List getAvailableRegions() {
+
+ // This stream operation is just a way to convert a list of regions to the RegionResponse object.
+ return Region.regions().stream()
+ .filter(r -> !r.isGlobalRegion()) // Ignore the global region. We don't need it.
+ .map(r -> {
+ // Build a single AWSRegionResponse with id, description, and displayValue.
+ RegionMetadata regionMetadata = r.metadata();
+ String displayValue = String.format("%s: %s", regionMetadata.description(), regionMetadata.id());
+ return RegionResponse.create(regionMetadata.id(), regionMetadata.description(), displayValue);
+ }).collect(Collectors.toList());
+ }
+
+ /**
+ * @return A list of available AWS services supported by the AWS Graylog AWS integration.
+ */
+ public AvailableAWSServiceSummmary getAvailableServices() {
+
+ ArrayList services = new ArrayList<>();
+ AvailableAWSService cloudWatchService =
+ AvailableAWSService.create("CloudWatch",
+ "Retrieve CloudWatch logs via Kinesis. Kinesis allows streaming of the logs" +
+ "in real time. Amazon CloudWatch is a monitoring and management service built" +
+ "for developers, system operators, site reliability engineers (SRE), " +
+ "and IT managers.",
+ "{\n" +
+ " \"Version\": \"2012-10-17\",\n" +
+ " \"Statement\": [\n" +
+ " {\n" +
+ " \"Sid\": \"VisualEditor0\",\n" +
+ " \"Effect\": \"Allow\",\n" +
+ " \"Action\": [\n" +
+ " \"cloudwatch:PutMetricData\",\n" +
+ " \"dynamodb:CreateTable\",\n" +
+ " \"dynamodb:DescribeTable\",\n" +
+ " \"dynamodb:GetItem\",\n" +
+ " \"dynamodb:PutItem\",\n" +
+ " \"dynamodb:Scan\",\n" +
+ " \"dynamodb:UpdateItem\",\n" +
+ " \"ec2:DescribeInstances\",\n" +
+ " \"ec2:DescribeNetworkInterfaceAttribute\",\n" +
+ " \"ec2:DescribeNetworkInterfaces\",\n" +
+ " \"elasticloadbalancing:DescribeLoadBalancerAttributes\",\n" +
+ " \"elasticloadbalancing:DescribeLoadBalancers\",\n" +
+ " \"kinesis:GetRecords\",\n" +
+ " \"kinesis:GetShardIterator\",\n" +
+ " \"kinesis:ListShards\"\n" +
+ " ],\n" +
+ " \"Resource\": \"*\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}",
+ "Requires Kinesis",
+ "https://aws.amazon.com/cloudwatch/"
+ );
+ services.add(cloudWatchService);
+
+ return AvailableAWSServiceSummmary.create(services, services.size());
+ }
+
+ //TODO Add getAWSServices List
+ //List that contains all the supported AWS services (i.e. Cloudwatch, Kinesis)
+
+ // TODO GET getUserCredentials
+
+}
\ No newline at end of file
diff --git a/src/test/java/org.graylog.integrations/aws/AWSServiceTest.java b/src/test/java/org.graylog.integrations/aws/AWSServiceTest.java
index 6ed8e55de..172a0286c 100644
--- a/src/test/java/org.graylog.integrations/aws/AWSServiceTest.java
+++ b/src/test/java/org.graylog.integrations/aws/AWSServiceTest.java
@@ -1,15 +1,58 @@
package org.graylog.integrations.aws;
+import org.graylog.integrations.aws.resources.responses.AvailableAWSServiceSummmary;
+import org.graylog.integrations.aws.resources.responses.RegionResponse;
+import org.graylog.integrations.aws.service.AWSService;
+import org.junit.Before;
import org.junit.Test;
-/**
- * Unit test for the AWSService class.
- */
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
public class AWSServiceTest {
+ private AWSService awsService;
+
+ @Before
+ public void setUp() {
+ awsService = new AWSService();
+ }
@Test
- public void name() {
+ public void regionTest() {
+
+ List availableRegions = awsService.getAvailableRegions();
+
+ // Use a loop presence check.
+ // Check format of random region.
+ boolean foundEuWestRegion = false;
+ for (RegionResponse availableRegion : availableRegions) {
+
+ if (availableRegion.regionId().equals("eu-west-2")) {
+ foundEuWestRegion = true;
+ }
+ }
+ assertTrue(foundEuWestRegion);
+
+ // Use one liner presence checks.
+ assertTrue(availableRegions.stream().anyMatch(r -> r.regionDescription().equals("EU (Stockholm)")));
+ assertTrue(availableRegions.stream().anyMatch(r -> r.displayValue().equals("EU (Stockholm): eu-north-1")));
+ assertEquals("There should be 20 total regions. This will change in future versions of the AWS SDK",
+ 20, availableRegions.size());
+ }
+
+ @Test
+ public void testAvailableServices() {
+
+ AvailableAWSServiceSummmary services = awsService.getAvailableServices();
+
+ // There should be one service.
+ assertEquals(1, services.total());
+ assertEquals(1, services.services().size());
+ // CloudWatch should be in the list of available services.
+ assertTrue(services.services().stream().anyMatch(s -> s.name().equals("CloudWatch")));
}
-}
+}
\ No newline at end of file
diff --git a/src/test/java/org.graylog.integrations/aws/CloudWatchServiceTest.java b/src/test/java/org.graylog.integrations/aws/CloudWatchServiceTest.java
new file mode 100644
index 000000000..d51df5ddd
--- /dev/null
+++ b/src/test/java/org.graylog.integrations/aws/CloudWatchServiceTest.java
@@ -0,0 +1,87 @@
+package org.graylog.integrations.aws;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnit;
+import org.mockito.junit.MockitoRule;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient;
+import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClientBuilder;
+import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsRequest;
+import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsResponse;
+import software.amazon.awssdk.services.cloudwatchlogs.model.LogGroup;
+import software.amazon.awssdk.services.cloudwatchlogs.paginators.DescribeLogGroupsIterable;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.isA;
+import static org.mockito.Mockito.when;
+
+public class CloudWatchServiceTest {
+
+ @Rule
+ public MockitoRule mockitoRule = MockitoJUnit.rule();
+
+ @Mock
+ private CloudWatchLogsClientBuilder logsClientBuilder;
+
+ @Mock
+ private CloudWatchLogsClient cloudWatchLogsClient;
+
+ @Mock
+ DescribeLogGroupsIterable logGroupsIterable;
+
+ private CloudWatchService cloudWatchService;
+
+ @Before
+ public void setUp() {
+
+ cloudWatchService = new CloudWatchService(logsClientBuilder);
+ }
+
+ @Test
+ public void testLogGroupNames() {
+
+ // Perform test setup. Return the builder and client when appropriate.
+ when(logsClientBuilder.region(isA(Region.class))).thenReturn(logsClientBuilder);
+ when(logsClientBuilder.build()).thenReturn(cloudWatchLogsClient);
+
+ // Create a fake response that contains three log groups.
+ DescribeLogGroupsResponse fakeLogGroupResponse = DescribeLogGroupsResponse
+ .builder()
+ .logGroups(LogGroup.builder().logGroupName("group-1").build(),
+ LogGroup.builder().logGroupName("group-2").build(),
+ LogGroup.builder().logGroupName("group-3").build())
+ .build();
+
+ // Mock out the response. When CloudWatchLogsClient.describeLogGroupsPaginator() is called,
+ // return two responses with six messages total.
+ List responses = Arrays.asList(fakeLogGroupResponse, fakeLogGroupResponse);
+ when(logGroupsIterable.iterator()).thenReturn(responses.iterator());
+ when(cloudWatchLogsClient.describeLogGroupsPaginator(isA(DescribeLogGroupsRequest.class))).thenReturn(logGroupsIterable);
+
+ ArrayList logGroupNames = cloudWatchService.getLogGroupNames("us-east-1");
+
+ // Inspect the log groups returned and verify the contents and size.
+ Assert.assertEquals("The number of groups should be because the two responses " +
+ "with 3 groups each were provided.", 6, logGroupNames.size());
+
+ // Loop example to verify presence of a specific log group.
+ boolean foundGroup = false;
+ for (String logGroupName : logGroupNames) {
+ if (logGroupName.equals("group-1")) {
+ foundGroup = true;
+ }
+ }
+ assertTrue(foundGroup);
+
+ // One line with stream.
+ assertTrue(logGroupNames.stream().anyMatch(logGroupName -> logGroupName.equals("group-2")));
+ }
+}
\ No newline at end of file
diff --git a/src/test/java/org.graylog.integrations/aws/KinesisServiceTest.java b/src/test/java/org.graylog.integrations/aws/KinesisServiceTest.java
new file mode 100644
index 000000000..4e8294ef9
--- /dev/null
+++ b/src/test/java/org.graylog.integrations/aws/KinesisServiceTest.java
@@ -0,0 +1,110 @@
+package org.graylog.integrations.aws;
+
+import org.graylog.integrations.aws.resources.requests.KinesisHealthCheckRequest;
+import org.graylog.integrations.aws.resources.responses.KinesisHealthCheckResponse;
+import org.graylog.integrations.aws.service.AWSLogMessage;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnit;
+import org.mockito.junit.MockitoRule;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.kinesis.KinesisClient;
+import software.amazon.awssdk.services.kinesis.KinesisClientBuilder;
+import software.amazon.awssdk.services.kinesis.model.ListStreamsRequest;
+import software.amazon.awssdk.services.kinesis.model.ListStreamsResponse;
+
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.isA;
+import static org.mockito.Mockito.when;
+
+public class KinesisServiceTest {
+
+ private static final String[] TWO_TEST_STREAMS = {"test-stream-1", "test-stream-2"};
+ private static final String TEST_REGION = Region.EU_WEST_1.id();
+
+ @Rule
+ public MockitoRule mockitoRule = MockitoJUnit.rule();
+
+ @Mock
+ private KinesisClientBuilder kinesisClientBuilder;
+
+ @Mock
+ private KinesisClient kinesisClient;
+
+ private KinesisService kinesisService;
+
+ @Before
+ public void setUp() {
+
+ // Create an AWS client with a mock KinesisClientBuilder
+ kinesisService = new KinesisService(kinesisClientBuilder);
+ }
+
+ @Test
+ public void testLogIdentification() {
+
+ // Verify that an ACCEPT flow log us detected as a flow log.
+ AWSLogMessage logMessage = new AWSLogMessage("2 123456789010 eni-abc123de 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK");
+ assertEquals(AWSLogMessage.Type.FLOW_LOGS, logMessage.messageType());
+
+ // Verify that an ACCEPT flow log us detected as a flow log.
+ logMessage = new AWSLogMessage("2 123456789010 eni-abc123de 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 REJECT OK");
+ assertEquals(AWSLogMessage.Type.FLOW_LOGS, logMessage.messageType());
+
+ // Verify that it's detected as unknown
+ logMessage = new AWSLogMessage("haha this is not a real log message");
+ assertEquals(AWSLogMessage.Type.UNKNOWN, logMessage.messageType());
+ }
+
+ @Test
+ public void healthCheck() {
+
+ KinesisHealthCheckRequest request = KinesisHealthCheckRequest.create("us-east-1", "some-group", "", "");
+ KinesisHealthCheckResponse healthCheckResponse = kinesisService.healthCheck(request);
+
+ // Hard-coded to flow logs for now. This will be mocked out with a real message at some point
+ assertEquals(AWSLogMessage.Type.FLOW_LOGS.toString(), healthCheckResponse.logType());
+ }
+
+ @Test
+ public void testGetStreams() throws ExecutionException {
+
+ // Test with two streams and one page. This is the most common case for most AWS accounts.
+ when(kinesisClientBuilder.region(Region.EU_WEST_1)).thenReturn(kinesisClientBuilder);
+ when(kinesisClientBuilder.build()).thenReturn(kinesisClient);
+
+ when(kinesisClient.listStreams(isA(ListStreamsRequest.class)))
+ .thenReturn(ListStreamsResponse.builder()
+ .streamNames(TWO_TEST_STREAMS)
+ .hasMoreStreams(false).build());
+
+
+ List kinesisStreams = kinesisService.getKinesisStreams(TEST_REGION, null, null);
+ assertEquals(2, kinesisStreams.size());
+
+ // Test with stream paging functionality. This will be the case when a large number of Kinesis streams
+ // are present on a particular AWS account.
+ when(kinesisClientBuilder.region(Region.EU_WEST_1)).thenReturn(kinesisClientBuilder);
+ when(kinesisClientBuilder.build()).thenReturn(kinesisClient);
+
+ when(kinesisClient.listStreams(isA(ListStreamsRequest.class)))
+ // First return a response with two streams indicating that there are more.
+ .thenReturn(ListStreamsResponse.builder()
+ .streamNames(TWO_TEST_STREAMS)
+ .hasMoreStreams(true).build())
+ // Then return a response with two streams and indicate that all have been retrieved.
+ .thenReturn(ListStreamsResponse.builder()
+ .streamNames(TWO_TEST_STREAMS)
+ .hasMoreStreams(false).build()); // Indicate no more streams.
+
+ kinesisStreams = kinesisService.getKinesisStreams(TEST_REGION, null, null);
+
+ // There should be 4 total streams (two from each page).
+ assertEquals(4, kinesisStreams.size());
+ }
+}
\ No newline at end of file
diff --git a/src/test/java/org.graylog.integrations/aws/resources/AWSResourceTest.java b/src/test/java/org.graylog.integrations/aws/resources/AWSResourceTest.java
new file mode 100644
index 000000000..8abd8e2d6
--- /dev/null
+++ b/src/test/java/org.graylog.integrations/aws/resources/AWSResourceTest.java
@@ -0,0 +1,44 @@
+package org.graylog.integrations.aws.resources;
+
+import org.graylog.integrations.aws.CloudWatchService;
+import org.graylog.integrations.aws.KinesisService;
+import org.graylog.integrations.aws.service.AWSService;
+import org.junit.Before;
+import org.junit.Rule;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnit;
+import org.mockito.junit.MockitoRule;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClientBuilder;
+import software.amazon.awssdk.services.kinesis.KinesisClient;
+import software.amazon.awssdk.services.kinesis.KinesisClientBuilder;
+
+import static org.mockito.Mockito.when;
+
+public class AWSResourceTest {
+
+ @Rule
+ public MockitoRule mockitoRule = MockitoJUnit.rule();
+
+ private AWSResource awsResource;
+
+ @Mock
+ private KinesisClientBuilder kinesisClientBuilder;
+
+ @Mock
+ private CloudWatchLogsClientBuilder logsClientBuilder;
+
+ @Mock
+ KinesisClient kinesisClient;
+
+ @Before
+ public void setUp() {
+
+ // Make sure that the actual KinesisClient is not used.
+ when(kinesisClientBuilder.region(Region.EU_WEST_1)).thenReturn(kinesisClientBuilder);
+ when(kinesisClientBuilder.build()).thenReturn(kinesisClient);
+
+ // Set up the chain of mocks.
+ awsResource = new AWSResource(new AWSService(), new KinesisService(kinesisClientBuilder), new CloudWatchService(logsClientBuilder));
+ }
+}
\ No newline at end of file