PeriodicShardSyncManager Changes Needed for DynamoDBStreamsKinesisAda…

…pter Compatibility (#970) * Interface and Rename for PeriodicShardSyncManager * Removed Automatic Indents * More Auto Indent Fixes
awslabs · Sep 9, 2022 · 251b331 · 251b331
1 parent 8dceb3f
commit 251b331
Show file tree

Hide file tree

Showing 9 changed files with 459 additions and 20 deletions.
diff --git a/...va/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IPeriodicShardSyncManager.java b/...va/com/amazonaws/services/kinesis/clientlibrary/lib/worker/IPeriodicShardSyncManager.java
@@ -0,0 +1,28 @@
+package com.amazonaws.services.kinesis.clientlibrary.lib.worker;
+
+import com.google.common.annotations.VisibleForTesting;
+import lombok.Value;
+import lombok.experimental.Accessors;
+
+public interface IPeriodicShardSyncManager {
+
+    TaskResult start();
+
+    /**
+     * Runs ShardSync once, without scheduling further periodic ShardSyncs.
+     * @return TaskResult from shard sync
+     */
+    TaskResult syncShardsOnce();
+
+    void stop();
+
+    @Value
+    @Accessors(fluent = true)
+    @VisibleForTesting
+    class ShardSyncResponse {
+        private final boolean shouldDoShardSync;
+        private final boolean isHoleDetected;
+        private final String reasonForDecision;
+    }
+}
+
diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ITask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ITask.java
@@ -20,7 +20,7 @@
  * Interface for shard processing tasks.
  * A task may execute an application callback (e.g. initialize, process, shutdown).
  */
-interface ITask extends Callable<TaskResult> {
+public interface ITask extends Callable<TaskResult> {
 
     /**
      * Perform task logic.

diff --git a/.../amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisPeriodicShardSyncManager.java b/.../amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisPeriodicShardSyncManager.java
diff --git a/...m/amazonaws/services/kinesis/clientlibrary/lib/worker/MetricsCollectingTaskDecorator.java b/...m/amazonaws/services/kinesis/clientlibrary/lib/worker/MetricsCollectingTaskDecorator.java
@@ -21,7 +21,7 @@
 /**
  * Decorates an ITask and reports metrics about its timing and success/failure.
  */
-class MetricsCollectingTaskDecorator implements ITask {
+public class MetricsCollectingTaskDecorator implements ITask {
 
     private final ITask other;
     private final IMetricsFactory factory;

diff --git a/...va/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncStrategy.java b/...va/com/amazonaws/services/kinesis/clientlibrary/lib/worker/PeriodicShardSyncStrategy.java
@@ -6,9 +6,9 @@
  */
 class PeriodicShardSyncStrategy implements ShardSyncStrategy {
 
-    private PeriodicShardSyncManager periodicShardSyncManager;
+    private IPeriodicShardSyncManager periodicShardSyncManager;
 
-    PeriodicShardSyncStrategy(PeriodicShardSyncManager periodicShardSyncManager) {
+    PeriodicShardSyncStrategy(IPeriodicShardSyncManager periodicShardSyncManager) {
         this.periodicShardSyncManager = periodicShardSyncManager;
     }
 

diff --git a/...va/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardEndShardSyncStrategy.java b/...va/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardEndShardSyncStrategy.java
@@ -17,10 +17,10 @@ class ShardEndShardSyncStrategy implements ShardSyncStrategy {
     private ShardSyncTaskManager shardSyncTaskManager;
 
     /** Runs periodic shard sync jobs in the background as an auditor process for shard-end syncs. */
-    private PeriodicShardSyncManager periodicShardSyncManager;
+    private IPeriodicShardSyncManager periodicShardSyncManager;
 
     ShardEndShardSyncStrategy(ShardSyncTaskManager shardSyncTaskManager,
-                              PeriodicShardSyncManager periodicShardSyncManager) {
+                              IPeriodicShardSyncManager periodicShardSyncManager) {
         this.shardSyncTaskManager = shardSyncTaskManager;
         this.periodicShardSyncManager = periodicShardSyncManager;
     }

diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardSyncTask.java
@@ -30,7 +30,7 @@
  * It will clean up leases/activities for shards that have been completely processed (if
  * cleanupLeasesUponShardCompletion is true).
  */
-class ShardSyncTask implements ITask {
+public class ShardSyncTask implements ITask {
 
     private static final Log LOG = LogFactory.getLog(ShardSyncTask.class);
 
@@ -56,7 +56,7 @@ class ShardSyncTask implements ITask {
      * @param shardSyncer shardSyncer instance used to check and create new leases
      * @param latestShards latest snapshot of shards to reuse
      */
-    ShardSyncTask(IKinesisProxy kinesisProxy,
+    public ShardSyncTask(IKinesisProxy kinesisProxy,
             ILeaseManager<KinesisClientLease> leaseManager,
             InitialPositionInStreamExtended initialPositionInStream,
             boolean cleanupLeasesUponShardCompletion,

diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/TaskResult.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/TaskResult.java
@@ -22,7 +22,7 @@
  * Used to capture information from a task that we want to communicate back to the higher layer.
  * E.g. exception thrown when executing the task, if we reach end of a shard.
  */
-class TaskResult {
+public class TaskResult {
 
     // Did we reach the end of the shard while processing this task.
     private boolean shardEndReached;
@@ -38,7 +38,7 @@ class TaskResult {
     /**
      * @return the shardEndReached
      */
-    protected boolean isShardEndReached() {
+    public boolean isShardEndReached() {
         return shardEndReached;
     }
 
@@ -77,7 +77,7 @@ public Exception getException() {
     /**
      * @param e Any exception encountered when running the process task.
      */
-    TaskResult(Exception e) {
+    public TaskResult(Exception e) {
         this(e, false);
     }
 

diff --git a/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java b/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java
@@ -156,7 +156,7 @@ public class Worker implements Runnable {
     // Periodic Shard Sync related fields
     private LeaderDecider leaderDecider;
     private ShardSyncStrategy shardSyncStrategy;
-    private PeriodicShardSyncManager leaderElectedPeriodicShardSyncManager;
+    private IPeriodicShardSyncManager leaderElectedPeriodicShardSyncManager;
 
     private final LeaseCleanupManager leaseCleanupManager;
 
@@ -533,7 +533,7 @@ config, getStreamConfig(config, kinesisClient),
             IMetricsFactory metricsFactory, long taskBackoffTimeMillis, long failoverTimeMillis,
             boolean skipShardSyncAtWorkerInitializationIfLeasesExist, ShardPrioritization shardPrioritization,
             Optional<Integer> retryGetRecordsInSeconds, Optional<Integer> maxGetRecordsThreadPool, WorkerStateChangeListener workerStateChangeListener,
-            LeaseCleanupValidator leaseCleanupValidator, LeaderDecider leaderDecider, PeriodicShardSyncManager periodicShardSyncManager) {
+            LeaseCleanupValidator leaseCleanupValidator, LeaderDecider leaderDecider, IPeriodicShardSyncManager periodicShardSyncManager) {
         this(applicationName, recordProcessorFactory, config, streamConfig, initialPositionInStream,
                 parentShardPollIntervalMillis, shardSyncIdleTimeMillis, cleanupLeasesUponShardCompletion, checkpoint,
                 leaseCoordinator, execService, metricsFactory, taskBackoffTimeMillis, failoverTimeMillis,
@@ -550,7 +550,7 @@ maxGetRecordsThreadPool, workerStateChangeListener, new KinesisShardSyncer(lease
             boolean skipShardSyncAtWorkerInitializationIfLeasesExist, ShardPrioritization shardPrioritization,
             Optional<Integer> retryGetRecordsInSeconds, Optional<Integer> maxGetRecordsThreadPool,
             WorkerStateChangeListener workerStateChangeListener, ShardSyncer shardSyncer, LeaderDecider leaderDecider,
-            PeriodicShardSyncManager periodicShardSyncManager) {
+            IPeriodicShardSyncManager periodicShardSyncManager) {
         this.applicationName = applicationName;
         this.recordProcessorFactory = recordProcessorFactory;
         this.config = config;
@@ -590,7 +590,7 @@ maxGetRecordsThreadPool, workerStateChangeListener, new KinesisShardSyncer(lease
      */
     private void createShardSyncStrategy(ShardSyncStrategyType strategyType,
                                          LeaderDecider leaderDecider,
-                                         PeriodicShardSyncManager periodicShardSyncManager) {
+                                         IPeriodicShardSyncManager periodicShardSyncManager) {
         switch (strategyType) {
             case PERIODIC:
                 this.leaderDecider = getOrCreateLeaderDecider(leaderDecider);
@@ -652,7 +652,7 @@ LeaderDecider getLeaderDecider() {
     /**
      * @return the leaderElectedPeriodicShardSyncManager
      */
-    PeriodicShardSyncManager getPeriodicShardSyncManager() {
+    IPeriodicShardSyncManager getPeriodicShardSyncManager() {
         return leaderElectedPeriodicShardSyncManager;
     }
 
@@ -1224,7 +1224,7 @@ StreamConfig getStreamConfig() {
      *            KinesisClientLibConfiguration
      * @return Returns metrics factory based on the config.
      */
-    private static IMetricsFactory getMetricsFactory(AmazonCloudWatch cloudWatchClient,
+    public static IMetricsFactory getMetricsFactory(AmazonCloudWatch cloudWatchClient,
             KinesisClientLibConfiguration config) {
         IMetricsFactory metricsFactory;
         if (config.getMetricsLevel() == MetricsLevel.NONE) {
@@ -1278,13 +1278,13 @@ private LeaderDecider getOrCreateLeaderDecider(LeaderDecider leaderDecider) {
 
     /** A non-null PeriodicShardSyncManager can only provided from unit tests. Any application code will create the
      * PeriodicShardSyncManager for the first time here. */
-    private PeriodicShardSyncManager getOrCreatePeriodicShardSyncManager(PeriodicShardSyncManager periodicShardSyncManager,
+    private IPeriodicShardSyncManager getOrCreatePeriodicShardSyncManager(IPeriodicShardSyncManager periodicShardSyncManager,
                                                                          boolean isAuditorMode) {
         if (periodicShardSyncManager != null) {
             return periodicShardSyncManager;
         }
 
-        return new PeriodicShardSyncManager(config.getWorkerIdentifier(),
+        return new KinesisPeriodicShardSyncManager(config.getWorkerIdentifier(),
                                             leaderDecider,
                                             new ShardSyncTask(streamConfig.getStreamProxy(),
                                                     leaseCoordinator.getLeaseManager(),
@@ -1353,6 +1353,8 @@ public static class Builder {
         @Setter @Accessors(fluent = true)
         private IKinesisProxy kinesisProxy;
         @Setter @Accessors(fluent = true)
+        private IPeriodicShardSyncManager periodicShardSyncManager;
+        @Setter @Accessors(fluent = true)
         private WorkerStateChangeListener workerStateChangeListener;
         @Setter @Accessors(fluent = true)
         private LeaseCleanupValidator leaseCleanupValidator;
@@ -1421,6 +1423,12 @@ public Worker build() {
                 throw new IllegalArgumentException(
                         "Kinesis Client Library configuration needs to be provided to build Worker");
             }
+            if (periodicShardSyncManager != null) {
+                if (leaseManager == null || shardSyncer == null || metricsFactory == null || leaderDecider == null) {
+
+                    throw new IllegalArgumentException("LeaseManager, ShardSyncer, MetricsFactory, and LeaderDecider must be provided if PeriodicShardSyncManager is provided");
+                }
+            }
             if (recordProcessorFactory == null) {
                 throw new IllegalArgumentException("A Record Processor Factory needs to be provided to build Worker");
             }
@@ -1546,7 +1554,7 @@ public Worker build() {
                     workerStateChangeListener,
                     shardSyncer,
                     leaderDecider,
-                    null /* PeriodicShardSyncManager */);
+                    periodicShardSyncManager);
         }
 
         <R, T extends AwsClientBuilder<T, R>> R createClient(final T builder,