From bc0e568429dffbcc73a27debca993be2a98605a1 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Tue, 6 Oct 2020 18:53:43 +0300 Subject: [PATCH] add redis SSL support to the offline-to-online ingestion job Signed-off-by: Oleg Avdeev --- .../src/main/scala/feast/ingestion/BasePipeline.scala | 3 ++- .../src/main/scala/feast/ingestion/IngestionJobConfig.scala | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala b/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala index 34c667b7e6..4bd8f527a6 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala @@ -31,10 +31,11 @@ trait BasePipeline { .setMaster("local") jobConfig.store match { - case RedisConfig(host, port) => + case RedisConfig(host, port, ssl) => conf .set("spark.redis.host", host) .set("spark.redis.port", port.toString) + .set("spark.redis.ssl", ssl.toString) } jobConfig.metrics match { diff --git a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala index 880f00f37f..72844ce937 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala @@ -26,7 +26,7 @@ object Modes extends Enumeration { abstract class StoreConfig -case class RedisConfig(host: String, port: Int) extends StoreConfig +case class RedisConfig(host: String, port: Int, ssl: Boolean) extends StoreConfig abstract class MetricConfig @@ -84,7 +84,7 @@ case class IngestionJobConfig( source: Source = null, startTime: DateTime = DateTime.now(), endTime: DateTime = DateTime.now(), - store: StoreConfig = RedisConfig("localhost", 6379), + store: StoreConfig = RedisConfig("localhost", 6379, false), metrics: Option[MetricConfig] = Some(StatsDConfig("localhost", 9125)), deadLetterPath: Option[String] = None )