feat(aws): add https support (#2061)

dragonflydb · Oct 24, 2023 · 1a813ce · 1a813ce
1 parent dc025e5
commit 1a813ce
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 8 deletions.
diff --git a/helio b/helio
diff --git a/src/server/detail/snapshot_storage.cc b/src/server/detail/snapshot_storage.cc
@@ -175,24 +175,25 @@ io::Result<std::vector<std::string>, GenericError> FileSnapshotStorage::LoadPath
   return paths;
 }
 
-AwsS3SnapshotStorage::AwsS3SnapshotStorage(const std::string& endpoint, bool ec2_metadata,
-                                           bool sign_payload) {
+AwsS3SnapshotStorage::AwsS3SnapshotStorage(const std::string& endpoint, bool https,
+                                           bool ec2_metadata, bool sign_payload) {
   shard_set->pool()->GetNextProactor()->Await([&] {
     if (!ec2_metadata) {
       setenv("AWS_EC2_METADATA_DISABLED", "true", 0);
     }
     // S3ClientConfiguration may request configuration and credentials from
     // EC2 metadata so must be run in a proactor thread.
     Aws::S3::S3ClientConfiguration s3_conf{};
-    LOG(INFO) << "Creating AWS S3 client; region=" << s3_conf.region << "; endpoint=" << endpoint;
+    LOG(INFO) << "Creating AWS S3 client; region=" << s3_conf.region << "; https=" << std::boolalpha
+              << https << "; endpoint=" << endpoint;
     if (!sign_payload) {
       s3_conf.payloadSigningPolicy = Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::ForceNever;
     }
     std::shared_ptr<Aws::Auth::AWSCredentialsProvider> credentials_provider =
         std::make_shared<util::aws::CredentialsProviderChain>();
     // Pass a custom endpoint. If empty uses the S3 endpoint.
     std::shared_ptr<Aws::S3::S3EndpointProviderBase> endpoint_provider =
-        std::make_shared<util::aws::S3EndpointProvider>(endpoint);
+        std::make_shared<util::aws::S3EndpointProvider>(endpoint, https);
     s3_ = std::make_shared<Aws::S3::S3Client>(credentials_provider, endpoint_provider, s3_conf);
   });
 }

diff --git a/src/server/detail/snapshot_storage.h b/src/server/detail/snapshot_storage.h
@@ -72,7 +72,8 @@ class FileSnapshotStorage : public SnapshotStorage {
 
 class AwsS3SnapshotStorage : public SnapshotStorage {
  public:
-  AwsS3SnapshotStorage(const std::string& endpoint, bool ec2_metadata, bool sign_payload);
+  AwsS3SnapshotStorage(const std::string& endpoint, bool https, bool ec2_metadata,
+                       bool sign_payload);
 
   io::Result<std::pair<io::Sink*, uint8_t>, GenericError> OpenWriteFile(
       const std::string& path) override;

diff --git a/src/server/server_family.cc b/src/server/server_family.cc
@@ -97,6 +97,7 @@ ABSL_FLAG(int32_t, slowlog_log_slower_than, 10000,
 ABSL_FLAG(uint32_t, slowlog_max_len, 20, "Slow log maximum length.");
 
 ABSL_FLAG(string, s3_endpoint, "", "endpoint for s3 snapshots, default uses aws regional endpoint");
+ABSL_FLAG(bool, s3_use_https, true, "whether to use https for s3 endpoints");
 // Disable EC2 metadata by default, or if a users credentials are invalid the
 // AWS client will spent 30s trying to connect to inaccessable EC2 endpoints
 // to load the credentials.
@@ -534,8 +535,8 @@ void ServerFamily::Init(util::AcceptServer* acceptor, std::vector<facade::Listen
   if (IsCloudPath(flag_dir)) {
     shard_set->pool()->GetNextProactor()->Await([&] { util::aws::Init(); });
     snapshot_storage_ = std::make_shared<detail::AwsS3SnapshotStorage>(
-        absl::GetFlag(FLAGS_s3_endpoint), absl::GetFlag(FLAGS_s3_ec2_metadata),
-        absl::GetFlag(FLAGS_s3_sign_payload));
+        absl::GetFlag(FLAGS_s3_endpoint), absl::GetFlag(FLAGS_s3_use_https),
+        absl::GetFlag(FLAGS_s3_ec2_metadata), absl::GetFlag(FLAGS_s3_sign_payload));
   } else if (fq_threadpool_) {
     snapshot_storage_ = std::make_shared<detail::FileSnapshotStorage>(fq_threadpool_.get());
   } else {
+9 −9		README.md
+3 −1		examples/s3_demo.cc
+29 −0		util/aws/http_client.cc
+11 −0		util/aws/http_client.h
+6 −6		util/aws/s3_endpoint_provider.cc
+3 −1		util/aws/s3_endpoint_provider.h
+1 −1		util/http/http_client.cc