Skip to content

Commit

Permalink
Add support for gcp fog storage options
Browse files Browse the repository at this point in the history
  • Loading branch information
@johha
johha committed Oct 10, 2023
1 parent 9b08822 commit ce3ee7b
Show file tree
Hide file tree
Showing 11 changed files with 190 additions and 42 deletions.
4 changes: 4 additions & 0 deletions config/cloud_controller.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ resource_pool:
blobstore_timeout: 5
provider: "Local"
fog_aws_storage_options: {}
fog_gcp_storage_options: {}

packages:
app_package_directory_key: "cc-packages"
Expand All @@ -185,6 +186,7 @@ packages:
blobstore_timeout: 5
provider: "Local"
fog_aws_storage_options: {}
fog_gcp_storage_options: {}

droplets:
droplet_directory_key: cc-droplets
Expand All @@ -193,13 +195,15 @@ droplets:
blobstore_timeout: 5
provider: "Local"
fog_aws_storage_options: {}
fog_gcp_storage_options: {}

buildpacks:
buildpack_directory_key: cc-buildpacks
fog_connection:
blobstore_timeout: 5
provider: "Local"
fog_aws_storage_options: {}
fog_gcp_storage_options: {}

registry_buddy:
host: "127.0.0.1"
Expand Down
3 changes: 2 additions & 1 deletion lib/cloud_controller/blobstore/client_provider.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ def provide_fog(options, directory_key, root_dir)
root_dir: root_dir,
min_size: options[:minimum_size],
max_size: options[:maximum_size],
storage_options: options[:fog_aws_storage_options]
aws_storage_options: options[:fog_aws_storage_options],
gcp_storage_options: options[:fog_gcp_storage_options]
)

logger = Steno.logger('cc.blobstore')
Expand Down
27 changes: 20 additions & 7 deletions lib/cloud_controller/blobstore/fog/fog_client.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,16 @@ def initialize(connection_config:,
root_dir: nil,
min_size: nil,
max_size: nil,
storage_options: nil)
aws_storage_options: nil,
gcp_storage_options: nil)
@root_dir = root_dir
@connection_config = connection_config
@directory_key = directory_key
@cdn = cdn
@min_size = min_size || 0
@max_size = max_size
@storage_options = storage_options
@aws_storage_options = aws_storage_options
@gcp_storage_options = gcp_storage_options
end

def local?
Expand Down Expand Up @@ -136,12 +138,23 @@ def files
end

def formatted_storage_options
return {} unless @storage_options && @storage_options[:encryption]
if connection.service.equal?(Fog::AWS::Storage)
return {} unless @aws_storage_options && @aws_storage_options[:encryption]

opts = @storage_options.dup
encrypt_opt = opts.delete(:encryption)
opts['x-amz-server-side-encryption'] = encrypt_opt
opts
opts = @aws_storage_options.dup
encrypt_opt = opts.delete(:encryption)
opts['x-amz-server-side-encryption'] = encrypt_opt
opts

elsif [Fog::Storage::GoogleJSON, Fog::Storage::GoogleXML].include?(connection.service)
return {} unless @gcp_storage_options

@gcp_storage_options

else
{}

end
end

def delete_file(file)
Expand Down
10 changes: 7 additions & 3 deletions lib/cloud_controller/config_schemas/base/api_schema.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,13 +144,15 @@ class ApiSchema < VCAP::Config
minimum_size: Integer,
resource_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

buildpacks: {
buildpack_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

packages: {
Expand All @@ -159,6 +161,7 @@ class ApiSchema < VCAP::Config
app_package_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash,
optional(:image_registry) => {
base_path: String
}
Expand All @@ -168,7 +171,8 @@ class ApiSchema < VCAP::Config
droplet_directory_key: String,
max_staged_droplets_stored: Integer,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

optional(:registry_buddy) => {
Expand Down
10 changes: 7 additions & 3 deletions lib/cloud_controller/config_schemas/base/clock_schema.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,20 +77,23 @@ class ClockSchema < VCAP::Config
minimum_size: Integer,
resource_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

buildpacks: {
buildpack_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

packages: {
max_package_size: Integer,
app_package_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash,
optional(:image_registry) => {
base_path: String
}
Expand All @@ -99,7 +102,8 @@ class ClockSchema < VCAP::Config
droplets: {
droplet_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

db_encryption_key: enum(String, NilClass),
Expand Down
10 changes: 7 additions & 3 deletions lib/cloud_controller/config_schemas/base/deployment_updater_schema.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,20 +80,23 @@ class DeploymentUpdaterSchema < VCAP::Config
minimum_size: Integer,
resource_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

buildpacks: {
buildpack_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

packages: {
max_package_size: Integer,
app_package_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash,
optional(:image_registry) => {
base_path: String
}
Expand All @@ -102,7 +105,8 @@ class DeploymentUpdaterSchema < VCAP::Config
droplets: {
droplet_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

stacks_file: String,
Expand Down
10 changes: 7 additions & 3 deletions lib/cloud_controller/config_schemas/base/worker_schema.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,20 +68,23 @@ class WorkerSchema < VCAP::Config
minimum_size: Integer,
resource_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

buildpacks: {
buildpack_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

packages: {
max_package_size: Integer,
app_package_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash,
optional(:image_registry) => {
base_path: String
}
Expand All @@ -90,7 +93,8 @@ class WorkerSchema < VCAP::Config
droplets: {
droplet_directory_key: String,
fog_connection: Hash,
fog_aws_storage_options: Hash
fog_aws_storage_options: Hash,
fog_gcp_storage_options: Hash
},

optional(:registry_buddy) => {
Expand Down
8 changes: 8 additions & 0 deletions spec/fixtures/config/port_8181_config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,18 +140,22 @@ resource_pool:
maximum_size: 42
minimum_size: 1
fog_aws_storage_options: {}
fog_gcp_storage_options: {}

buildpacks:
fog_aws_storage_options: {}
fog_gcp_storage_options: {}

packages:
max_package_size: 42
max_valid_packages_stored: 42
fog_aws_storage_options: {}
fog_gcp_storage_options: {}

droplets:
max_staged_droplets_stored: 42
fog_aws_storage_options: {}
fog_gcp_storage_options: {}

request_timeout_in_seconds: 600
skip_cert_verify: true
Expand All @@ -175,24 +179,28 @@ resource_pool:
minimum_size: 1
fog_connection: {}
fog_aws_storage_options: {}
fog_gcp_storage_options: {}

buildpacks:
buildpack_directory_key: ''
fog_connection: {}
fog_aws_storage_options: {}
fog_gcp_storage_options: {}

packages:
app_package_directory_key: ''
max_package_size: 42
max_valid_packages_stored: 42
fog_connection: {}
fog_aws_storage_options: {}
fog_gcp_storage_options: {}

droplets:
droplet_directory_key: ''
max_staged_droplets_stored: 42
fog_connection: {}
fog_aws_storage_options: {}
fog_gcp_storage_options: {}



Expand Down
25 changes: 23 additions & 2 deletions spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ module Blobstore
root_dir: anything,
min_size: anything,
max_size: anything,
storage_options: { encryption: 'my organic algo' })
aws_storage_options: { encryption: 'my organic algo' },
gcp_storage_options: anything)
end

context 'fog methods' do
Expand All @@ -62,6 +63,25 @@ module Blobstore
end
end

context 'when a gcp uniform option is requested' do
before do
options.merge!(fog_gcp_storage_options: { uniform: false })
end

it 'passes the specified uniform option to the fog client' do
allow(FogClient).to receive(:new).and_call_original
ClientProvider.provide(options: options, directory_key: 'key')
expect(FogClient).to have_received(:new).with(connection_config: anything,
directory_key: anything,
cdn: anything,
root_dir: anything,
min_size: anything,
max_size: anything,
aws_storage_options: anything,
gcp_storage_options: { uniform: false })
end
end

context 'when a cdn is requested in the options' do
before do
options.merge!(cdn: { uri: 'http://cdn.com' })
Expand All @@ -76,7 +96,8 @@ module Blobstore
root_dir: anything,
min_size: anything,
max_size: anything,
storage_options: anything)
aws_storage_options: anything,
gcp_storage_options: anything)
end
end

Expand Down
Loading

0 comments on commit ce3ee7b

Please sign in to comment.