Merge pull request #6 from igosneves/add-configuration-storage-host

Add support to storage_blob_host configuration

.gitignore

@@ -23,4 +23,7 @@ devenv.local.nix
 terraform.tfstate
 terraform.tfstate.backup
 .terraform.tfstate.lock.info
-*.tfvars
+*.tfvars
+
+__azurite_db*
+__blobstorage__/

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## [Unreleased]
 
+- Allow creating public container
+- Add Azurite support
 
 ## [0.5.3] 2024-10-31
 

README.md

@@ -46,6 +46,30 @@ prod:
   principal_id: 71b34410-4c50-451d-b456-95ead1b18cce
 ```
 
+### Azurite
+
+To use Azurite, pass the `storage_blob_host` config key with the Azurite URL (`http://127.0.0.1:10000/devstoreaccount1` by default)
+and the Azurite credentials (`devstoreaccount1` and `Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==` by default).
+
+Example:
+
+```
+dev:
+  service: AzureBlob
+  container: container_name
+  storage_account_name: devstoreaccount1
+  storage_access_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+  storage_blob_host: http://127.0.0.1:10000/devstoreaccount1
+```
+
+You'll have to create the container before you can start uploading files.
+You can do so using Azure CLI, Azure Storage Explorer, or by running:
+
+`bin/rails runner "ActiveStorage::Blob.service.client.tap{|client| client.create_container unless client.get_container_properties.present?}.tap { |client| puts 'done!' if client.get_container_properties.present?}"`
+
+Make sure that `config.active_storage.service = :dev` is set to your azurite configuration.
+Container names can't have any special characters, or you'll get an error.
+
 ## Standalone
 
 Instantiate a client with your account name, an access key and the container name:

Rakefile

@@ -5,6 +5,7 @@ require "minitest/test_task"
 require "azure_blob"
 require_relative "test/support/app_service_vpn"
 require_relative "test/support/azure_vm_vpn"
+require_relative "test/support/azurite"
 
 Minitest::TestTask.create(:test_rails) do
   self.test_globs = [ "test/rails/**/test_*.rb",
@@ -39,6 +40,28 @@ ensure
   vpn.kill
 end
 
+task :test_azurite do |t|
+  azurite = Azurite.new
+  # Azurite well-known credentials
+  # https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio%2Cblob-storage#well-known-storage-account-and-key
+  account_name = ENV["AZURE_ACCOUNT_NAME"] = "devstoreaccount1"
+  access_key = ENV["AZURE_ACCESS_KEY"] = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+  host = ENV["STORAGE_BLOB_HOST"] = "http://127.0.0.1:10000/devstoreaccount1"
+  ENV["TESTING_AZURITE"] = "true"
+
+  # Create containers
+  private_container = AzureBlob::Client.new(account_name:, access_key:, host:, container: ENV["AZURE_PRIVATE_CONTAINER"])
+  public_container = AzureBlob::Client.new(account_name:, access_key:, host:, container: ENV["AZURE_PUBLIC_CONTAINER"])
+  # public_container.delete_container
+  private_container.create_container unless private_container.get_container_properties.present?
+  public_container.create_container(public_access: true) unless public_container.get_container_properties.present?
+
+  Rake::Task["test_client"].execute
+  Rake::Task["test_rails"].execute
+ensure
+  azurite.kill
+end
+
 task :test_entra_id do |t|
   ENV["AZURE_ACCESS_KEY"] = nil
   Rake::Task["test"].execute

devenv.nix

@@ -15,6 +15,7 @@
     sshuttle
     sshpass
     rsync
+    azurite
   ];
 
   languages.ruby.enable = true;

lib/active_storage/service/azure_blob_service.rb

@@ -35,13 +35,14 @@ module ActiveStorage
   class Service::AzureBlobService < Service
     attr_reader :client, :container, :signer
 
-    def initialize(storage_account_name:, storage_access_key: nil, container:, public: false, **options)
+    def initialize(storage_account_name:, storage_access_key: nil, container:, storage_blob_host: nil, public: false, **options)
       @container = container
       @public = public
       @client = AzureBlob::Client.new(
         account_name: storage_account_name,
         access_key: storage_access_key,
         container: container,
+        host: storage_blob_host,
         **options)
     end
 

lib/azure_blob/client.rb

@@ -15,9 +15,10 @@ module AzureBlob
   # AzureBlob Client class. You interact with the Azure Blob api
   # through an instance of this class.
   class Client
-    def initialize(account_name:, access_key: nil, principal_id: nil, container:, **options)
+    def initialize(account_name:, access_key: nil, principal_id: nil, container:, host: nil, **options)
       @account_name = account_name
       @container = container
+      @host = host
       @cloud_regions = options[:cloud_regions]&.to_sym || :global
 
       no_access_key = access_key.nil? || access_key&.empty?
@@ -29,8 +30,8 @@ def initialize(account_name:, access_key: nil, principal_id: nil, container:, **
         )
       end
       @signer = using_managed_identities ?
-        AzureBlob::EntraIdSigner.new(account_name:, host:, principal_id:) :
-        AzureBlob::SharedKeySigner.new(account_name:, access_key:)
+        AzureBlob::EntraIdSigner.new(account_name:, host: self.host, principal_id:) :
+        AzureBlob::SharedKeySigner.new(account_name:, access_key:, host: self.host)
     end
 
     # Create a blob of type block. Will automatically split the the blob in multiple block and send the blob in pieces (blocks) if the blob is too big.
@@ -190,8 +191,12 @@ def get_container_properties(options = {})
     # Calls to {Create Container}[https://learn.microsoft.com/en-us/rest/api/storageservices/create-container]
     def create_container(options = {})
       uri = generate_uri(container)
+      headers = {}
+      headers[:"x-ms-blob-public-access"] = "blob" if options[:public_access]
+      headers[:"x-ms-blob-public-access"] = options[:public_access] if ["container","blob"].include?(options[:public_access])
+
       uri.query = URI.encode_www_form(restype: "container")
-      response = Http.new(uri, signer:).put
+      response = Http.new(uri, headers, signer:).put
     end
 
     # Delete the container

lib/azure_blob/shared_key_signer.rb

@@ -7,9 +7,11 @@
 
 module AzureBlob
   class SharedKeySigner # :nodoc:
-    def initialize(account_name:, access_key:)
+    def initialize(account_name:, access_key:, host:)
       @account_name = account_name
       @access_key = Base64.decode64(access_key)
+      @host = host
+      @remove_prefix = @host.include?("/#{@account_name}")
     end
 
     def authorization_header(uri:, verb:, headers: {})
@@ -39,6 +41,11 @@ def authorization_header(uri:, verb:, headers: {})
     end
 
     def sas_token(uri, options = {})
+      if remove_prefix
+        uri = uri.clone
+        uri.path = uri.path.delete_prefix("/#{account_name}")
+      end
+
       to_sign = [
         options[:permissions],
         options[:start],
@@ -99,6 +106,6 @@ module Resources # :nodoc:
       end
     end
 
-    attr_reader :access_key, :account_name
+    attr_reader :access_key, :account_name, :remove_prefix
   end
 end

test/client/test_client.rb

@@ -13,11 +13,13 @@ def setup
     @access_key = ENV["AZURE_ACCESS_KEY"]
     @container = ENV["AZURE_PRIVATE_CONTAINER"]
     @principal_id = ENV["AZURE_PRINCIPAL_ID"]
+    @host = ENV["STORAGE_BLOB_HOST"]
     @client = AzureBlob::Client.new(
       account_name: @account_name,
       access_key: @access_key,
       container: @container,
       principal_id: @principal_id,
+      host: @host,
     )
     @key = "test client##{name}"
     @content = "Some random content #{Random.rand(200)}"
@@ -104,11 +106,13 @@ def test_upload_integrity_block
   end
 
   def test_upload_raise_on_invalid_checksum_blob
+    skip if ENV["TESTING_AZURITE"]
     checksum = OpenSSL::Digest::MD5.base64digest(content + "a")
     assert_raises(AzureBlob::Http::IntegrityError) { client.create_block_blob(key, content, content_md5: checksum) }
   end
 
   def test_upload_raise_on_invalid_checksum_block
+    skip if ENV["TESTING_AZURITE"]
     checksum = OpenSSL::Digest::MD5.base64digest(content + "a")
     assert_raises(AzureBlob::Http::IntegrityError) { client.put_blob_block(key, 0, content, content_md5: checksum) }
   end
@@ -338,6 +342,7 @@ def test_create_container
       access_key: @access_key,
       container: Random.alphanumeric(20).tr("0-9", "").downcase,
       principal_id: @principal_id,
+      host: @host,
     )
     container = client.get_container_properties
     refute container.present?

test/rails/controllers/direct_uploads_controller_test.rb

@@ -23,14 +23,16 @@ class ActiveStorage::AzureBlobDirectUploadsControllerTest < ActionDispatch::Inte
     post rails_direct_uploads_url, params: { blob: {
       filename: "hello.txt", byte_size: 6, checksum: checksum, content_type: "text/plain", metadata: metadata, } }
 
+    host = @config[:host] || "https://#{@config[:storage_account_name]}.blob.core.windows.net"
+
     response.parsed_body.tap do |details|
       assert_equal ActiveStorage::Blob.find(details["id"]), ActiveStorage::Blob.find_signed!(details["signed_id"])
       assert_equal "hello.txt", details["filename"]
       assert_equal 6, details["byte_size"]
       assert_equal checksum, details["checksum"]
       assert_equal metadata, details["metadata"]
       assert_equal "text/plain", details["content_type"]
-      assert_match %r{#{@config[:storage_account_name]}\.blob\.core\.windows\.net/#{@config[:container]}}, details["direct_upload"]["url"]
+      assert details["direct_upload"]["url"].start_with?("#{host}/#{@config[:container]}")
       assert_equal({ "Content-Type" => "text/plain", "Content-MD5" => checksum, "x-ms-blob-content-disposition" => "inline; filename=\"hello.txt\"; filename*=UTF-8''hello.txt", "x-ms-blob-type" => "BlockBlob" }, details["direct_upload"]["headers"])
     end
   end

test/rails/service/azure_blob_public_service_test.rb

@@ -8,10 +8,15 @@ class ActiveStorage::Service::AzureBlobPublicServiceTest < ActiveSupport::TestCa
 
   include ActiveStorage::Service::SharedServiceTests
 
+  setup do
+    @config = SERVICE_CONFIGURATIONS[:azure_public]
+  end
+
   test "public URL generation" do
     url = @service.url(@key, filename: ActiveStorage::Filename.new("avatar.png"))
+    host = @config[:host] || "https://#{@config[:storage_account_name]}.blob.core.windows.net"
 
-    assert_match(/.*\.blob\.core\.windows\.net\/.*\/#{@key}/, url)
+    assert url.start_with?("#{host}/#{@config[:container]}/#{@key}")
 
     response = Net::HTTP.get_response(URI(url))
     assert_equal "200", response.code
@@ -30,7 +35,7 @@ class ActiveStorage::Service::AzureBlobPublicServiceTest < ActiveSupport::TestCa
     @service.headers_for_direct_upload(key, checksum: checksum, content_type: content_type, filename: ActiveStorage::Filename.new("test.txt")).each do |k, v|
       request.add_field k, v
     end
-    Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+    Net::HTTP.start(uri.host, uri.port, use_ssl: uri.port == 443) do |http|
       http.request request
     end
 

test/rails/service/azure_blob_service_test.rb

@@ -21,7 +21,8 @@ class ActiveStorage::Service::AzureBlobServiceTest < ActiveSupport::TestCase
     @service.headers_for_direct_upload(key, checksum: checksum, content_type: content_type, filename: ActiveStorage::Filename.new("test.txt")).each do |k, v|
       request.add_field k, v
     end
-    Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+
+    Net::HTTP.start(uri.host, uri.port, use_ssl: uri.port == 443) do |http|
       http.request request
     end
 
@@ -42,7 +43,7 @@ class ActiveStorage::Service::AzureBlobServiceTest < ActiveSupport::TestCase
     @service.headers_for_direct_upload(key, checksum: checksum, content_type: "text/plain", filename: ActiveStorage::Filename.new("test.txt"), disposition: :attachment).each do |k, v|
       request.add_field k, v
     end
-    Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+    Net::HTTP.start(uri.host, uri.port, use_ssl: uri.port == 443) do |http|
       http.request request
     end
 

test/rails/service/configurations.yml

@@ -3,6 +3,7 @@ DEFAULT: &default
   storage_account_name: <%= ENV["AZURE_ACCOUNT_NAME"] %>
   storage_access_key: <%= ENV["AZURE_ACCESS_KEY"] %>
   principal_id: <%= ENV["AZURE_PRINCIPAL_ID"]%>
+  storage_blob_host: <%= ENV["STORAGE_BLOB_HOST"] %>
 
 azure:
   <<: *default

test/rails/service/shared_service_tests.rb

@@ -30,6 +30,7 @@ module ActiveStorage::Service::SharedServiceTests
     end
 
     test "uploading without integrity" do
+      skip if ENV["TESTING_AZURITE"]
       key  = SecureRandom.base58(24)
       data = "Something else entirely!"
 
@@ -39,7 +40,7 @@ module ActiveStorage::Service::SharedServiceTests
 
       assert_not @service.exist?(key)
     ensure
-      @service.delete key
+      @service.delete key unless ENV["TESTING_AZURITE"]
     end
 
     test "uploading with integrity and multiple keys" do

test/support/azurite.rb

@@ -0,0 +1,20 @@
+require "open3"
+require "shellwords"
+
+class Azurite
+  def initialize(verbose: false)
+    @verbose = verbose
+    stdin, stdout, @wait_thread = Open3.popen2e("azurite")
+    stdout.each do |line|
+      break if line.include?("Azurite Blob service is successfully listening at http://127.0.0.1:10000")
+    end
+  end
+
+  def kill
+    Process.kill("INT", wait_thread.pid)
+  end
+
+  private
+
+  attr_reader :wait_thread
+end