diff --git a/config/cloud_controller.yml b/config/cloud_controller.yml index 1dae3ab3102..7562432b9c9 100644 --- a/config/cloud_controller.yml +++ b/config/cloud_controller.yml @@ -176,6 +176,7 @@ resource_pool: blobstore_timeout: 5 provider: "Local" fog_aws_storage_options: {} + fog_gcp_storage_options: {} packages: app_package_directory_key: "cc-packages" @@ -185,6 +186,7 @@ packages: blobstore_timeout: 5 provider: "Local" fog_aws_storage_options: {} + fog_gcp_storage_options: {} droplets: droplet_directory_key: cc-droplets @@ -193,6 +195,7 @@ droplets: blobstore_timeout: 5 provider: "Local" fog_aws_storage_options: {} + fog_gcp_storage_options: {} buildpacks: buildpack_directory_key: cc-buildpacks @@ -200,6 +203,7 @@ buildpacks: blobstore_timeout: 5 provider: "Local" fog_aws_storage_options: {} + fog_gcp_storage_options: {} registry_buddy: host: "127.0.0.1" diff --git a/lib/cloud_controller/blobstore/client_provider.rb b/lib/cloud_controller/blobstore/client_provider.rb index 4fe6a85e742..635f3491439 100644 --- a/lib/cloud_controller/blobstore/client_provider.rb +++ b/lib/cloud_controller/blobstore/client_provider.rb @@ -31,7 +31,8 @@ def provide_fog(options, directory_key, root_dir) root_dir: root_dir, min_size: options[:minimum_size], max_size: options[:maximum_size], - storage_options: options[:fog_aws_storage_options] + aws_storage_options: options[:fog_aws_storage_options], + gcp_storage_options: options[:fog_gcp_storage_options] ) logger = Steno.logger('cc.blobstore') diff --git a/lib/cloud_controller/blobstore/fog/fog_client.rb b/lib/cloud_controller/blobstore/fog/fog_client.rb index 66b73ac2542..67bce45ed63 100644 --- a/lib/cloud_controller/blobstore/fog/fog_client.rb +++ b/lib/cloud_controller/blobstore/fog/fog_client.rb @@ -20,14 +20,16 @@ def initialize(connection_config:, root_dir: nil, min_size: nil, max_size: nil, - storage_options: nil) + aws_storage_options: nil, + gcp_storage_options: nil) @root_dir = root_dir @connection_config = connection_config @directory_key = directory_key @cdn = cdn @min_size = min_size || 0 @max_size = max_size - @storage_options = storage_options + @aws_storage_options = aws_storage_options + @gcp_storage_options = gcp_storage_options end def local? @@ -136,12 +138,23 @@ def files end def formatted_storage_options - return {} unless @storage_options && @storage_options[:encryption] + if connection.service.equal?(Fog::AWS::Storage) + return {} unless @aws_storage_options && @aws_storage_options[:encryption] - opts = @storage_options.dup - encrypt_opt = opts.delete(:encryption) - opts['x-amz-server-side-encryption'] = encrypt_opt - opts + opts = @aws_storage_options.dup + encrypt_opt = opts.delete(:encryption) + opts['x-amz-server-side-encryption'] = encrypt_opt + opts + + elsif [Fog::Storage::GoogleJSON, Fog::Storage::GoogleXML].include?(connection.service) + return {} unless @gcp_storage_options + + @gcp_storage_options + + else + {} + + end end def delete_file(file) diff --git a/lib/cloud_controller/config_schemas/base/api_schema.rb b/lib/cloud_controller/config_schemas/base/api_schema.rb index 0b72de1965e..f28590ad2ed 100644 --- a/lib/cloud_controller/config_schemas/base/api_schema.rb +++ b/lib/cloud_controller/config_schemas/base/api_schema.rb @@ -144,13 +144,15 @@ class ApiSchema < VCAP::Config minimum_size: Integer, resource_directory_key: String, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, buildpacks: { buildpack_directory_key: String, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, packages: { @@ -159,6 +161,7 @@ class ApiSchema < VCAP::Config app_package_directory_key: String, fog_connection: Hash, fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash, optional(:image_registry) => { base_path: String } @@ -168,7 +171,8 @@ class ApiSchema < VCAP::Config droplet_directory_key: String, max_staged_droplets_stored: Integer, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, optional(:registry_buddy) => { diff --git a/lib/cloud_controller/config_schemas/base/clock_schema.rb b/lib/cloud_controller/config_schemas/base/clock_schema.rb index 1f0297ed4ab..c028fc44576 100644 --- a/lib/cloud_controller/config_schemas/base/clock_schema.rb +++ b/lib/cloud_controller/config_schemas/base/clock_schema.rb @@ -77,13 +77,15 @@ class ClockSchema < VCAP::Config minimum_size: Integer, resource_directory_key: String, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, buildpacks: { buildpack_directory_key: String, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, packages: { @@ -91,6 +93,7 @@ class ClockSchema < VCAP::Config app_package_directory_key: String, fog_connection: Hash, fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash, optional(:image_registry) => { base_path: String } @@ -99,7 +102,8 @@ class ClockSchema < VCAP::Config droplets: { droplet_directory_key: String, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, db_encryption_key: enum(String, NilClass), diff --git a/lib/cloud_controller/config_schemas/base/deployment_updater_schema.rb b/lib/cloud_controller/config_schemas/base/deployment_updater_schema.rb index 51043b5ccb5..98bb1d0db8d 100644 --- a/lib/cloud_controller/config_schemas/base/deployment_updater_schema.rb +++ b/lib/cloud_controller/config_schemas/base/deployment_updater_schema.rb @@ -80,13 +80,15 @@ class DeploymentUpdaterSchema < VCAP::Config minimum_size: Integer, resource_directory_key: String, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, buildpacks: { buildpack_directory_key: String, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, packages: { @@ -94,6 +96,7 @@ class DeploymentUpdaterSchema < VCAP::Config app_package_directory_key: String, fog_connection: Hash, fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash, optional(:image_registry) => { base_path: String } @@ -102,7 +105,8 @@ class DeploymentUpdaterSchema < VCAP::Config droplets: { droplet_directory_key: String, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, stacks_file: String, diff --git a/lib/cloud_controller/config_schemas/base/worker_schema.rb b/lib/cloud_controller/config_schemas/base/worker_schema.rb index b7962c44d09..53a60813e20 100644 --- a/lib/cloud_controller/config_schemas/base/worker_schema.rb +++ b/lib/cloud_controller/config_schemas/base/worker_schema.rb @@ -68,13 +68,15 @@ class WorkerSchema < VCAP::Config minimum_size: Integer, resource_directory_key: String, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, buildpacks: { buildpack_directory_key: String, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, packages: { @@ -82,6 +84,7 @@ class WorkerSchema < VCAP::Config app_package_directory_key: String, fog_connection: Hash, fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash, optional(:image_registry) => { base_path: String } @@ -90,7 +93,8 @@ class WorkerSchema < VCAP::Config droplets: { droplet_directory_key: String, fog_connection: Hash, - fog_aws_storage_options: Hash + fog_aws_storage_options: Hash, + fog_gcp_storage_options: Hash }, optional(:registry_buddy) => { diff --git a/spec/fixtures/config/port_8181_config.yml b/spec/fixtures/config/port_8181_config.yml index ab050201c08..ddec9eeea1c 100644 --- a/spec/fixtures/config/port_8181_config.yml +++ b/spec/fixtures/config/port_8181_config.yml @@ -140,18 +140,22 @@ resource_pool: maximum_size: 42 minimum_size: 1 fog_aws_storage_options: {} + fog_gcp_storage_options: {} buildpacks: fog_aws_storage_options: {} + fog_gcp_storage_options: {} packages: max_package_size: 42 max_valid_packages_stored: 42 fog_aws_storage_options: {} + fog_gcp_storage_options: {} droplets: max_staged_droplets_stored: 42 fog_aws_storage_options: {} + fog_gcp_storage_options: {} request_timeout_in_seconds: 600 skip_cert_verify: true @@ -175,11 +179,13 @@ resource_pool: minimum_size: 1 fog_connection: {} fog_aws_storage_options: {} + fog_gcp_storage_options: {} buildpacks: buildpack_directory_key: '' fog_connection: {} fog_aws_storage_options: {} + fog_gcp_storage_options: {} packages: app_package_directory_key: '' @@ -187,12 +193,14 @@ packages: max_valid_packages_stored: 42 fog_connection: {} fog_aws_storage_options: {} + fog_gcp_storage_options: {} droplets: droplet_directory_key: '' max_staged_droplets_stored: 42 fog_connection: {} fog_aws_storage_options: {} + fog_gcp_storage_options: {} diff --git a/spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb b/spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb index 65af0338dd0..713fe60f726 100644 --- a/spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb +++ b/spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb @@ -46,7 +46,8 @@ module Blobstore root_dir: anything, min_size: anything, max_size: anything, - storage_options: { encryption: 'my organic algo' }) + aws_storage_options: { encryption: 'my organic algo' }, + gcp_storage_options: anything) end context 'fog methods' do @@ -62,6 +63,25 @@ module Blobstore end end + context 'when a gcp uniform option is requested' do + before do + options.merge!(fog_gcp_storage_options: { uniform: false }) + end + + it 'passes the specified uniform option to the fog client' do + allow(FogClient).to receive(:new).and_call_original + ClientProvider.provide(options: options, directory_key: 'key') + expect(FogClient).to have_received(:new).with(connection_config: anything, + directory_key: anything, + cdn: anything, + root_dir: anything, + min_size: anything, + max_size: anything, + aws_storage_options: anything, + gcp_storage_options: { uniform: false }) + end + end + context 'when a cdn is requested in the options' do before do options.merge!(cdn: { uri: 'http://cdn.com' }) @@ -76,7 +96,8 @@ module Blobstore root_dir: anything, min_size: anything, max_size: anything, - storage_options: anything) + aws_storage_options: anything, + gcp_storage_options: anything) end end diff --git a/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb b/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb index c79ca1cb79c..92dc3492517 100644 --- a/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb +++ b/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb @@ -17,6 +17,7 @@ module Blobstore aws_secret_access_key: 'fake_secret_access_key' } end + let(:directory_key) { 'a-directory-key' } subject(:client) do @@ -355,41 +356,122 @@ def upload_tmpfile(client, key='abcdef') end end - context 'encryption' do + context 'storage options' do let(:files) { double(:files, create: true) } before do allow_any_instance_of(FogClient).to receive(:files).and_return(files) end - context 'when encryption type is specified' do - let(:client_with_encryption) do - FogClient.new(connection_config: connection_config, - directory_key: directory_key, - storage_options: { encryption: 'my-algo' }) + context 'aws' do + context 'when encryption type is specified' do + let(:client_with_encryption) do + FogClient.new(connection_config: connection_config, + directory_key: directory_key, + aws_storage_options: { encryption: 'my-algo' }) + end + + it 'passes the encryption options to aws' do + path = File.join(local_dir, 'empty_file.png') + FileUtils.touch(path) + + client_with_encryption.cp_to_blobstore(path, 'abcdef123456') + + expect(files).to have_received(:create).with(key: anything, + body: anything, + content_type: anything, + public: anything, + 'x-amz-server-side-encryption' => 'my-algo') + end end - it 'passes the encryption options to aws' do - path = File.join(local_dir, 'empty_file.png') - FileUtils.touch(path) + context 'when encryption type is not specified' do + let(:client_with_encryption) do + FogClient.new(connection_config:, + directory_key:) + end - client_with_encryption.cp_to_blobstore(path, 'abcdef123456') + it 'does not pass the encryption options to aws' do + path = File.join(local_dir, 'empty_file.png') + FileUtils.touch(path) - expect(files).to have_received(:create).with(key: anything, - body: anything, - content_type: anything, - public: anything, - 'x-amz-server-side-encryption' => 'my-algo') + client_with_encryption.cp_to_blobstore(path, 'abcdef123456') + + expect(files).to have_received(:create).with(key: anything, + body: anything, + content_type: anything, + public: anything) + end end end - context 'when encryption type is not specified' do + context 'gcp' do + let(:gcp_connection_config) do + { + provider: 'Google', + google_project: 'gcs_project', + google_client_email: 'gcs_service_account_email', + google_json_key_string: 'gcs_service_account_json_key' + } + end + + context 'when uniform flag is specified' do + let(:client_with_encryption) do + FogClient.new(connection_config: gcp_connection_config, + directory_key: directory_key, + gcp_storage_options: { uniform: false, other: 'thing' }) + end + + it 'passes the storage options to gcp' do + path = File.join(local_dir, 'empty_file.png') + FileUtils.touch(path) + + client_with_encryption.cp_to_blobstore(path, 'abcdef123456') + + expect(files).to have_received(:create).with(key: anything, + body: anything, + content_type: anything, + public: anything, + uniform: false, + other: 'thing') + end + end + + context 'when gcp uniform flag is not specified' do + let(:client_with_encryption) do + FogClient.new(connection_config: gcp_connection_config, + directory_key: directory_key) + end + + it 'has empty storage options' do + path = File.join(local_dir, 'empty_file.png') + FileUtils.touch(path) + + client_with_encryption.cp_to_blobstore(path, 'abcdef123456') + + expect(files).to have_received(:create).with(key: anything, + body: anything, + content_type: anything, + public: anything) + end + end + end + + context 'AzureRM' do + let(:az_connection_config) do + { + provider: 'AzureRM', + environment: 'env', + azure_storage_account_name: 'account', + azure_storage_access_key: 'key' + } + end let(:client_with_encryption) do - FogClient.new(connection_config:, - directory_key:) + FogClient.new(connection_config: az_connection_config, + directory_key: directory_key) end - it 'does not pass the encryption options to aws' do + it 'has empty storage options' do path = File.join(local_dir, 'empty_file.png') FileUtils.touch(path) @@ -451,7 +533,7 @@ def upload_tmpfile(client, key='abcdef') let(:client) do FogClient.new(connection_config: connection_config, directory_key: directory_key, - storage_options: { encryption: encryption, other: 'thing' }) + aws_storage_options: { encryption: encryption, other: 'thing' }) end let(:dest_file) { double(:file, copy: true, save: true, nil?: false) } let(:src_file) { double(:file, copy: true, nil?: false) } diff --git a/spec/unit/lib/cloud_controller/config_spec.rb b/spec/unit/lib/cloud_controller/config_spec.rb index 4b863bf7b0d..1ed8189e493 100644 --- a/spec/unit/lib/cloud_controller/config_spec.rb +++ b/spec/unit/lib/cloud_controller/config_spec.rb @@ -9,6 +9,7 @@ module VCAP::CloudController fog_aws_storage_options: { encryption: 'AES256' }, + fog_gcp_storage_options: {}, app_package_directory_key: 'app_key' }, droplets: { @@ -327,6 +328,7 @@ module VCAP::CloudController fog_aws_storage_options: { encryption: 'AES256' }, + fog_gcp_storage_options: {}, app_package_directory_key: 'app_key' }, droplets: { @@ -489,6 +491,7 @@ module VCAP::CloudController fog_aws_storage_options: { encryption: 'AES256' }, + fog_gcp_storage_options: {}, app_package_directory_key: 'app_key' }) expect(config_instance.get(:packages, :fog_aws_storage_options)).to eq(encryption: 'AES256')