From 814494dca82c87b517b31e7d2c3024b41440919e Mon Sep 17 00:00:00 2001 From: Daniel Pierce Date: Mon, 9 Dec 2024 16:15:53 -0500 Subject: [PATCH] Make storage adapters report their protocol string (#974) * Make storage adapters report their protocol string Having this available will ease writing tests for systems that use multiple storage adapters. The fedora storage adapter already does this. * Refactor to use protocol instance method --- .../specs/shared_specs/storage_adapter.rb | 1 + lib/valkyrie/storage/disk.rb | 13 +++++++--- lib/valkyrie/storage/fedora.rb | 15 +++++++---- lib/valkyrie/storage/memory.rb | 11 ++++++-- lib/valkyrie/storage/versioned_disk.rb | 25 +++++++++++++------ 5 files changed, 48 insertions(+), 17 deletions(-) diff --git a/lib/valkyrie/specs/shared_specs/storage_adapter.rb b/lib/valkyrie/specs/shared_specs/storage_adapter.rb index 4a2e9867..db1dc2cb 100644 --- a/lib/valkyrie/specs/shared_specs/storage_adapter.rb +++ b/lib/valkyrie/specs/shared_specs/storage_adapter.rb @@ -12,6 +12,7 @@ class Valkyrie::Specs::CustomResource < Valkyrie::Resource Valkyrie::Specs.send(:remove_const, :CustomResource) end subject { storage_adapter } + it { is_expected.to respond_to(:protocol) } it { is_expected.to respond_to(:handles?).with_keywords(:id) } it { is_expected.to respond_to(:find_by).with_keywords(:id) } it { is_expected.to respond_to(:delete).with_keywords(:id) } diff --git a/lib/valkyrie/storage/disk.rb b/lib/valkyrie/storage/disk.rb index 92922ed0..45ed1b71 100644 --- a/lib/valkyrie/storage/disk.rb +++ b/lib/valkyrie/storage/disk.rb @@ -3,6 +3,8 @@ module Valkyrie::Storage # Implements the DataMapper Pattern to store binary data on disk class Disk attr_reader :base_path, :path_generator, :file_mover + PROTOCOL = 'disk://' + def initialize(base_path:, path_generator: BucketedStorage, file_mover: FileUtils.method(:mv)) @base_path = Pathname.new(base_path.to_s) @path_generator = path_generator.new(base_path: base_path) @@ -18,13 +20,13 @@ def upload(file:, original_filename:, resource: nil, **_extra_arguments) new_path = path_generator.generate(resource: resource, file: file, original_filename: original_filename) FileUtils.mkdir_p(new_path.parent) file_mover.call(file.path, new_path) - find_by(id: Valkyrie::ID.new("disk://#{new_path}")) + find_by(id: Valkyrie::ID.new("#{protocol}#{new_path}")) end # @param id [Valkyrie::ID] # @return [Boolean] true if this adapter can handle this type of identifer def handles?(id:) - id.to_s.start_with?("disk://#{base_path}") + id.to_s.start_with?("#{protocol}#{base_path}") end # @param feature [Symbol] Feature to test for. @@ -33,8 +35,13 @@ def supports?(_feature) false end + # @return [String] identifier prefix + def protocol + PROTOCOL + end + def file_path(id) - id.to_s.gsub(/^disk:\/\//, '') + id.to_s.gsub(/^#{Regexp.escape(protocol)}/, '') end # Return the file associated with the given identifier diff --git a/lib/valkyrie/storage/fedora.rb b/lib/valkyrie/storage/fedora.rb index bb1c7009..d10bd5ca 100644 --- a/lib/valkyrie/storage/fedora.rb +++ b/lib/valkyrie/storage/fedora.rb @@ -27,7 +27,7 @@ def initialize(connection:, base_path: "/", fedora_version: Valkyrie::Persistenc # @param id [Valkyrie::ID] # @return [Boolean] true if this adapter can handle this type of identifer def handles?(id:) - id.to_s.start_with?(PROTOCOL) + id.to_s.start_with?(protocol) end # @param feature [Symbol] Feature to test for. @@ -41,6 +41,11 @@ def supports?(feature) false end + # @return [String] identifier prefix + def protocol + PROTOCOL + end + # Return the file associated with the given identifier # @param id [Valkyrie::ID] # @return [Valkyrie::StorageAdapter::StreamFile] @@ -63,7 +68,7 @@ def upload(file:, original_filename:, resource:, content_type: "application/octe # Fedora 6 auto versions, so check to see if there's a version for this # initial upload. If not, then mint one (fedora 4/5) version_id = current_version_id(id: valkyrie_identifier(uri: identifier)) || mint_version(identifier, latest_version(identifier)) - perform_find(id: Valkyrie::ID.new(identifier.to_s.sub(/^.+\/\//, PROTOCOL)), version_id: version_id) + perform_find(id: Valkyrie::ID.new(identifier.to_s.sub(/^.+\/\//, protocol)), version_id: version_id) end # @param id [Valkyrie::ID] ID of the Valkyrie::StorageAdapter::StreamFile to @@ -79,7 +84,7 @@ def upload_version(id:, file:) end upload_file(fedora_uri: uri, io: file) version_id = mint_version(uri, latest_version(uri)) - perform_find(id: Valkyrie::ID.new(uri.to_s.sub(/^.+\/\//, PROTOCOL)), version_id: version_id) + perform_find(id: Valkyrie::ID.new(uri.to_s.sub(/^.+\/\//, protocol)), version_id: version_id) end # @param id [Valkyrie::ID] @@ -201,12 +206,12 @@ def io # Translate the Valkrie ID into a URL for the fedora file # @return [RDF::URI] def fedora_identifier(id:) - identifier = id.to_s.sub(PROTOCOL, "#{connection.http.scheme}://") + identifier = id.to_s.sub(protocol, "#{connection.http.scheme}://") RDF::URI(identifier) end def valkyrie_identifier(uri:) - id = uri.to_s.sub("http://", PROTOCOL) + id = uri.to_s.sub("http://", protocol) Valkyrie::ID.new(id) end diff --git a/lib/valkyrie/storage/memory.rb b/lib/valkyrie/storage/memory.rb index 91411911..6ac7adee 100644 --- a/lib/valkyrie/storage/memory.rb +++ b/lib/valkyrie/storage/memory.rb @@ -6,6 +6,8 @@ module Valkyrie::Storage # in cases where you want to preserve real data class Memory attr_reader :cache + PROTOCOL = 'memory://' + def initialize @cache = {} end @@ -16,7 +18,7 @@ def initialize # @param _extra_arguments [Hash] additional arguments which may be passed to other adapters # @return [Valkyrie::StorageAdapter::StreamFile] def upload(file:, original_filename:, resource: nil, **_extra_arguments) - identifier = Valkyrie::ID.new("memory://#{resource.id}") + identifier = Valkyrie::ID.new("#{protocol}#{resource.id}") version_id = Valkyrie::ID.new("#{identifier}##{SecureRandom.uuid}") cache[identifier] ||= {} cache[identifier][:current] = Valkyrie::StorageAdapter::StreamFile.new(id: identifier, io: file, version_id: version_id) @@ -67,7 +69,7 @@ def find_by(id:) # @param id [Valkyrie::ID] # @return [Boolean] true if this adapter can handle this type of identifer def handles?(id:) - id.to_s.start_with?("memory://") + id.to_s.start_with?(protocol) end # @param feature [Symbol] Feature to test for. @@ -83,6 +85,11 @@ def supports?(feature) end end + # @return [String] identifier prefix + def protocol + PROTOCOL + end + def id_and_version(id) id, version = id.to_s.split("#") [Valkyrie::ID.new(id), version] diff --git a/lib/valkyrie/storage/versioned_disk.rb b/lib/valkyrie/storage/versioned_disk.rb index e7751d25..9ba759bf 100644 --- a/lib/valkyrie/storage/versioned_disk.rb +++ b/lib/valkyrie/storage/versioned_disk.rb @@ -7,6 +7,8 @@ module Valkyrie::Storage # with "deletionmarker" in the name of the file. class VersionedDisk attr_reader :base_path, :path_generator, :file_mover + PROTOCOL = 'versiondisk://' + def initialize(base_path:, path_generator: ::Valkyrie::Storage::Disk::BucketedStorage, file_mover: FileUtils.method(:cp)) @base_path = Pathname.new(base_path.to_s) @path_generator = path_generator.new(base_path: base_path) @@ -26,7 +28,7 @@ def upload(file:, original_filename:, resource: nil, paused: false, **extra_argu return sleep(0.001) && upload(file: file, original_filename: original_filename, resource: resource, paused: true, **extra_arguments) if !paused && File.exist?(new_path) FileUtils.mkdir_p(new_path.parent) file_mover.call(file.try(:path) || file.try(:disk_path), new_path) - find_by(id: Valkyrie::ID.new("versiondisk://#{new_path}")) + find_by(id: Valkyrie::ID.new("#{protocol}#{new_path}")) end def current_timestamp @@ -50,13 +52,13 @@ def upload_version(id:, file:, paused: false) return sleep(0.001) && upload_version(id: id, file: file, paused: true) if !paused && File.exist?(new_path) FileUtils.mkdir_p(new_path.parent) file_mover.call(file.try(:path) || file.try(:disk_path), new_path) - find_by(id: Valkyrie::ID.new("versiondisk://#{new_path}")) + find_by(id: Valkyrie::ID.new("#{protocol}#{new_path}")) end # @param id [Valkyrie::ID] # @return [Boolean] true if this adapter can handle this type of identifer def handles?(id:) - id.to_s.start_with?("versiondisk://#{base_path}") + id.to_s.start_with?("#{protocol}#{base_path}") end # @param feature [Symbol] Feature to test for. @@ -66,6 +68,11 @@ def supports?(feature) false end + # @return [String] identifier prefix + def protocol + PROTOCOL + end + # Return the file associated with the given identifier # @param id [Valkyrie::ID] # @return [Valkyrie::StorageAdapter::File] @@ -95,7 +102,7 @@ def delete(id:) # @return [Array] def find_versions(id:) version_files(id: id).select { |x| !x.to_s.include?("deletionmarker") }.map do |file| - find_by(id: Valkyrie::ID.new("versiondisk://#{file}")) + find_by(id: Valkyrie::ID.new("#{protocol}#{file}")) end end @@ -106,7 +113,7 @@ def version_files(id:) end def file_path(version_id) - version_id.to_s.gsub(/^versiondisk:\/\//, '') + version_id.to_s.gsub(/^#{Regexp.escape(protocol)}/, '') end # @return VersionId A VersionId value that's resolved a current reference, @@ -128,6 +135,10 @@ def initialize(id) @id = id end + def protocol + PROTOCOL + end + def current_reference_id self.class.new(Valkyrie::ID.new(string_id.gsub(version, "current"))) end @@ -139,13 +150,13 @@ def resolve_current end def file_path - @file_path ||= string_id.gsub(/^versiondisk:\/\//, '') + @file_path ||= string_id.gsub(/^#{Regexp.escape(protocol)}/, '') end def version_files root = Pathname.new(file_path) root.parent.children.select { |file| file.basename.to_s.end_with?(filename) }.sort.reverse.map do |file| - VersionId.new(Valkyrie::ID.new("versiondisk://#{file}")) + VersionId.new(Valkyrie::ID.new("#{protocol}#{file}")) end end