diff --git a/app/jobs/upload_job.rb b/app/jobs/upload_job.rb index 33cbc658..a131ab5c 100644 --- a/app/jobs/upload_job.rb +++ b/app/jobs/upload_job.rb @@ -35,6 +35,8 @@ class UploadJob < ApplicationJob } GRAPHQL + PROJECT_CONFIG = 'project_config.yml' + def perform(payload) modified_locales(payload).each do |locale| projects_data = load_projects_data(locale, repository(payload), owner(payload)) @@ -86,30 +88,53 @@ def load_projects_data(locale, repository, owner) end def format_project(project_dir, locale, repository, owner) - components = [] - images = [] - proj_config = {} - data = project_dir.object - raise InvalidDirectoryStructureError, "The directory structure is incorrect and the job can't be processed." unless data.respond_to?(:entries) + validate_directory_structure(data) - data.entries.each do |file| - if file.name == 'project_config.yml' - proj_config = YAML.safe_load(file.object.text, symbolize_names: true) + proj_config_file = data.entries.find { |file| file.name == PROJECT_CONFIG } + proj_config = YAML.safe_load(proj_config_file.object.text, symbolize_names: true) - # Skip if build is set to false (for backwards compatibility the build must happen if the key is not present) - if proj_config[:build] == false - @skip_job = true - break - end - elsif file.object.text - components << component(file) + if proj_config[:build] == false + @skip_job = true + return proj_config + end + + files = data.entries.reject { |file| file.name == PROJECT_CONFIG } + categorized_files = categorize_files(files, project_dir, locale, repository, owner) + + { **proj_config, locale:, **categorized_files } + end + + def validate_directory_structure(data) + raise InvalidDirectoryStructureError, 'The directory structure is incorrect and the job can\'t be processed.' unless data.respond_to?(:entries) + end + + def categorize_files(files, project_dir, locale, repository, owner) + categories = { + components: [], + images: [], + videos: [], + audio: [] + } + + files.each do |file| + mime_type = file_mime_type(file) + + case mime_type + when /text/ + categories[:components] << component(file) + when /image/ + categories[:images] << media(file, project_dir, locale, repository, owner) + when /video/ + categories[:videos] << media(file, project_dir, locale, repository, owner) + when /audio/ + categories[:audio] << media(file, project_dir, locale, repository, owner) else - images << image(file, project_dir, locale, repository, owner) + raise "Unsupported file type: #{mime_type}" end end - { **proj_config, locale:, components:, images: } + categories end def component(file) @@ -120,7 +145,7 @@ def component(file) { name:, extension:, content:, default: } end - def image(file, project_dir, locale, repository, owner) + def media(file, project_dir, locale, repository, owner) filename = file.name directory = project_dir.name url = "https://github.com/#{owner}/#{repository}/raw/#{ENV.fetch('GITHUB_WEBHOOK_REF')}/#{locale}/code/#{directory}/#{filename}" @@ -134,6 +159,10 @@ def repository(payload) def owner(payload) payload[:repository][:owner][:name] end + + def file_mime_type(file) + Marcel::MimeType.for(file.object, name: file.name) + end end class InvalidDirectoryStructureError < StandardError; end diff --git a/app/models/filesystem_project.rb b/app/models/filesystem_project.rb index c2622367..5a73be00 100644 --- a/app/models/filesystem_project.rb +++ b/app/models/filesystem_project.rb @@ -4,33 +4,55 @@ class FilesystemProject CODE_FORMATS = ['.py', '.csv', '.txt', '.html', '.css'].freeze - IMAGE_FORMATS = ['.png', '.jpg', '.jpeg', '.webp'].freeze PROJECTS_ROOT = Rails.root.join('lib/tasks/project_components') + PROJECT_CONFIG = 'project_config.yml' def self.import_all! PROJECTS_ROOT.each_child do |dir| - proj_config = YAML.safe_load_file(dir.join('project_config.yml').to_s) - files = dir.children - code_files = files.filter { |file| CODE_FORMATS.include? File.extname(file) } - image_files = files.filter { |file| IMAGE_FORMATS.include? File.extname(file) } - - components = [] - code_files.each do |file| - components << component(file, dir) - end + proj_config = YAML.safe_load_file(dir.join(PROJECT_CONFIG).to_s) - images = [] - image_files.each do |file| - images << image(file, dir) - end + files = dir.children.reject { |file| file.basename.to_s == 'project_config.yml' } + categorized_files = categorize_files(files, dir) project_importer = ProjectImporter.new(name: proj_config['NAME'], identifier: proj_config['IDENTIFIER'], type: proj_config['TYPE'] || 'python', - locale: proj_config['LOCALE'] || 'en', components:, images:) + locale: proj_config['LOCALE'] || 'en', **categorized_files) project_importer.import! end end + def self.categorize_files(files, dir) + categories = { + components: [], + images: [], + videos: [], + audio: [] + } + + files.each do |file| + if CODE_FORMATS.include? File.extname(file) + categories[:components] << component(file, dir) + else + mime_type = file_mime_type(file) + + case mime_type + when /text/ + categories[:components] << component(file, dir) + when /image/ + categories[:images] << media(file, dir) + when /video/ + categories[:videos] << media(file, dir) + when /audio/ + categories[:audio] << media(file, dir) + else + raise "Unsupported file type: #{mime_type}" + end + end + end + + categories + end + def self.component(file, dir) name = File.basename(file, '.*') extension = File.extname(file).delete('.') @@ -39,7 +61,11 @@ def self.component(file, dir) { name:, extension:, content: code, default: } end - def self.image(file, dir) + def self.file_mime_type(file) + Marcel::MimeType.for(File.open(file), name: File.basename(file)) + end + + def self.media(file, dir) filename = File.basename(file) io = File.open(dir.join(filename).to_s) { filename:, io: } diff --git a/app/models/project.rb b/app/models/project.rb index 83ce43eb..c7cea3d6 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -8,6 +8,8 @@ class Project < ApplicationRecord has_many :components, -> { order(default: :desc, name: :asc) }, dependent: :destroy, inverse_of: :project has_many :project_errors, dependent: :nullify has_many_attached :images + has_many_attached :videos + has_many_attached :audio accepts_nested_attributes_for :components @@ -58,6 +60,10 @@ def last_edited_at [updated_at, components.maximum(:updated_at)].compact.max end + def media + images + videos + audio + end + private def check_unique_not_null diff --git a/app/views/api/projects/show.json.jbuilder b/app/views/api/projects/show.json.jbuilder index 7e760cb9..7ff4b103 100644 --- a/app/views/api/projects/show.json.jbuilder +++ b/app/views/api/projects/show.json.jbuilder @@ -31,6 +31,16 @@ json.image_list(@project.images) do |image| json.url(rails_blob_url(image)) end +json.videos(@project.videos) do |video| + json.filename(video.filename) + json.url(rails_blob_url(video)) +end + +json.audio(@project.audio) do |audio_file| + json.filename(audio_file.filename) + json.url(rails_blob_url(audio_file)) +end + json.user_name(@user&.name) if @user.present? && @project.parent json.finished(@project.finished) if @project.school.present? && @project.remixed_from_id.present? diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 306571f8..3047b2ac 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -1,13 +1,16 @@ # frozen_string_literal: true class ProjectImporter - attr_reader :name, :identifier, :images, :components, :type, :locale + attr_reader :name, :identifier, :images, :videos, :audio, :media, :components, :type, :locale def initialize(**kwargs) @name = kwargs[:name] @identifier = kwargs[:identifier] @components = kwargs[:components] @images = kwargs[:images] + @videos = kwargs[:videos] + @audio = kwargs[:audio] + @media = Array(images) + Array(videos) + Array(audio) @type = kwargs[:type] @locale = kwargs[:locale] end @@ -17,8 +20,8 @@ def import! setup_project delete_components create_components - delete_removed_images - attach_images_if_needed + delete_removed_media + attach_media_if_needed project.save! end @@ -46,37 +49,45 @@ def create_components end end - def delete_removed_images - return if removed_image_names.empty? + def delete_removed_media + return if removed_media_names.empty? - removed_image_names.each do |filename| - img = project.images.find { |i| i.blob.filename == filename } - img.purge + removed_media_names.each do |filename| + media_file = project.media.find { |i| i.blob.filename == filename } + media_file.purge end end - def removed_image_names - existing_images = project.images.map { |x| x.blob.filename.to_s } - existing_images - images.pluck(:filename) + def removed_media_names + existing_media = project.media.map { |x| x.blob.filename.to_s } + existing_media - media.pluck(:filename) end - def attach_images_if_needed - images.each do |image| - existing_image = find_existing_image(image[:filename]) - if existing_image - next if existing_image.blob.checksum == image_checksum(image[:io]) + def attach_media_if_needed + media.each do |media_file| + existing_media_file = find_existing_media_file(media_file[:filename]) + if existing_media_file + next if existing_media_file.blob.checksum == media_checksum(media_file[:io]) - existing_image.purge + existing_media.purge + end + if images.include?(media_file) + project.images.attach(**media_file) + elsif videos.include?(media_file) + project.videos.attach(**media_file) + elsif audio.include?(media_file) + project.audio.attach(**media_file) + else + raise "Unsupported media file: #{media_file[:filename]}" end - project.images.attach(**image) end end - def find_existing_image(filename) - project.images.find { |i| i.blob.filename == filename } + def find_existing_media_file(filename) + project.media.find { |i| i.blob.filename == filename } end - def image_checksum(io) + def media_checksum(io) OpenSSL::Digest.new('MD5').tap do |checksum| while (chunk = io.read(5.megabytes)) checksum << chunk diff --git a/lib/tasks/project_components/audio-video/hello_world.m4a b/lib/tasks/project_components/audio-video/hello_world.m4a new file mode 100644 index 00000000..05e64888 Binary files /dev/null and b/lib/tasks/project_components/audio-video/hello_world.m4a differ diff --git a/lib/tasks/project_components/audio-video/highlandCow.mp4 b/lib/tasks/project_components/audio-video/highlandCow.mp4 new file mode 100644 index 00000000..163aff31 Binary files /dev/null and b/lib/tasks/project_components/audio-video/highlandCow.mp4 differ diff --git a/lib/tasks/project_components/audio-video/main.py b/lib/tasks/project_components/audio-video/main.py new file mode 100644 index 00000000..00950d9a --- /dev/null +++ b/lib/tasks/project_components/audio-video/main.py @@ -0,0 +1 @@ +print('hello world') \ No newline at end of file diff --git a/lib/tasks/project_components/audio-video/music.mp3 b/lib/tasks/project_components/audio-video/music.mp3 new file mode 100644 index 00000000..5b7a89f9 Binary files /dev/null and b/lib/tasks/project_components/audio-video/music.mp3 differ diff --git a/lib/tasks/project_components/audio-video/project_config.yml b/lib/tasks/project_components/audio-video/project_config.yml new file mode 100644 index 00000000..22362501 --- /dev/null +++ b/lib/tasks/project_components/audio-video/project_config.yml @@ -0,0 +1,3 @@ +NAME: 'Audio Video Test' +IDENTIFIER: 'audio-video' +TYPE: 'python' diff --git a/lib/tasks/project_components/audio-video/toy.mov b/lib/tasks/project_components/audio-video/toy.mov new file mode 100644 index 00000000..c9810a13 Binary files /dev/null and b/lib/tasks/project_components/audio-video/toy.mov differ diff --git a/spec/factories/project.rb b/spec/factories/project.rb index ad414f08..1054311e 100644 --- a/spec/factories/project.rb +++ b/spec/factories/project.rb @@ -34,6 +34,22 @@ end end + trait :with_attached_video do + after(:build) do |object| + object.videos.attach(io: Rails.root.join('spec/fixtures/files/test_video_1.mp4').open, + filename: 'test_video', + content_type: 'video/mp4') + end + end + + trait :with_attached_audio do + after(:build) do |object| + object.audio.attach(io: Rails.root.join('spec/fixtures/files/test_audio_1.mp3').open, + filename: 'test_audio', + content_type: 'audio/mp3') + end + end + trait :with_instructions do instructions { Faker::Lorem.paragraph } school { create(:school) } diff --git a/spec/fixtures/files/test_audio_1.mp3 b/spec/fixtures/files/test_audio_1.mp3 new file mode 100644 index 00000000..5b7a89f9 Binary files /dev/null and b/spec/fixtures/files/test_audio_1.mp3 differ diff --git a/spec/fixtures/files/test_video_1.mp4 b/spec/fixtures/files/test_video_1.mp4 new file mode 100644 index 00000000..163aff31 Binary files /dev/null and b/spec/fixtures/files/test_video_1.mp4 differ diff --git a/spec/jobs/upload_job_spec.rb b/spec/jobs/upload_job_spec.rb index 28357d28..42657dce 100644 --- a/spec/jobs/upload_job_spec.rb +++ b/spec/jobs/upload_job_spec.rb @@ -41,6 +41,24 @@ isBinary: true } }, + { + name: 'music.mp3', + extension: '.mp3', + object: { + __typename: 'Blob', + text: nil, + isBinary: true + } + }, + { + name: 'video.mp4', + extension: '.mp4', + object: { + __typename: 'Blob', + text: nil, + isBinary: true + } + }, { name: 'main.py', extension: '.py', @@ -88,12 +106,26 @@ filename: 'astronaut1.png', io: instance_of(StringIO) } + ], + videos: [ + { + filename: 'video.mp4', + io: instance_of(StringIO) + } + ], + audio: [ + { + filename: 'music.mp3', + io: instance_of(StringIO) + } ] } end before do stub_request(:get, 'https://github.com/me/my-amazing-repo/raw/branches/whatever/ja-JP/code/dont-collide-starter/astronaut1.png').to_return(status: 200, body: '', headers: {}) + stub_request(:get, 'https://github.com/me/my-amazing-repo/raw/branches/whatever/ja-JP/code/dont-collide-starter/music.mp3').to_return(status: 200, body: '', headers: {}) + stub_request(:get, 'https://github.com/me/my-amazing-repo/raw/branches/whatever/ja-JP/code/dont-collide-starter/video.mp4').to_return(status: 200, body: '', headers: {}) end context 'with the build flag undefined' do diff --git a/spec/lib/project_importer_spec.rb b/spec/lib/project_importer_spec.rb index 8fab6e50..b64a8416 100644 --- a/spec/lib/project_importer_spec.rb +++ b/spec/lib/project_importer_spec.rb @@ -15,6 +15,12 @@ ], images: [ { filename: 'my-amazing-image.png', io: File.open('spec/fixtures/files/test_image_1.png') } + ], + videos: [ + { filename: 'my-amazing-video.mp4', io: File.open('spec/fixtures/files/test_video_1.mp4') } + ], + audio: [ + { filename: 'my-amazing-audio.mp3', io: File.open('spec/fixtures/files/test_audio_1.mp3') } ] ) end @@ -49,6 +55,16 @@ importer.import! expect(project.images.count).to eq(1) end + + it 'creates the project videos' do + importer.import! + expect(project.videos.count).to eq(1) + end + + it 'creates the project audio' do + importer.import! + expect(project.audio.count).to eq(1) + end end context 'when the project already exists in the database' do @@ -58,6 +74,8 @@ :with_default_component, :with_components, :with_attached_image, + :with_attached_video, + :with_attached_audio, component_count: 2, identifier: 'my-amazing-project', locale: 'ja-JP' @@ -87,5 +105,13 @@ it 'updates images' do expect { importer.import! }.to change { project.reload.images[0].filename.to_s }.to('my-amazing-image.png') end + + it 'updates videos' do + expect { importer.import! }.to change { project.reload.videos[0].filename.to_s }.to('my-amazing-video.mp4') + end + + it 'updates audio' do + expect { importer.import! }.to change { project.reload.audio[0].filename.to_s }.to('my-amazing-audio.mp3') + end end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 80dc3c7e..864a6d4f 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -13,10 +13,20 @@ it { is_expected.to have_many(:components) } it { is_expected.to have_many(:project_errors).dependent(:nullify) } it { is_expected.to have_many_attached(:images) } + it { is_expected.to have_many_attached(:videos) } + it { is_expected.to have_many_attached(:audio) } it 'purges attached images' do expect(described_class.reflect_on_attachment(:images).options[:dependent]).to eq(:purge_later) end + + it 'purges attached videos' do + expect(described_class.reflect_on_attachment(:videos).options[:dependent]).to eq(:purge_later) + end + + it 'purges attached audio' do + expect(described_class.reflect_on_attachment(:audio).options[:dependent]).to eq(:purge_later) + end end describe 'validations' do @@ -241,6 +251,14 @@ end end + describe '#media' do + let(:project) { create(:project, :with_attached_image, :with_attached_video, :with_attached_audio) } + + it 'returns all media files' do + expect(project.media).to eq(project.images + project.videos + project.audio) + end + end + describe 'auditing' do let(:school) { create(:school) } let(:teacher) { create(:teacher, school:) } diff --git a/spec/requests/projects/show_spec.rb b/spec/requests/projects/show_spec.rb index 19d2ddc1..782cc829 100644 --- a/spec/requests/projects/show_spec.rb +++ b/spec/requests/projects/show_spec.rb @@ -26,7 +26,9 @@ user_id: project.user_id, instructions: project.instructions, components: [], - image_list: [] + image_list: [], + videos: [], + audio: [] }.to_json end @@ -71,6 +73,8 @@ }, components: [], image_list: [], + videos: [], + audio: [], user_name: 'Joe Bloggs', finished: student_project.finished }.to_json @@ -97,7 +101,9 @@ locale: another_project.locale, user_id: another_project.user_id, components: [], - image_list: [] + image_list: [], + videos: [], + audio: [] }.to_json end @@ -126,7 +132,9 @@ user_id: starter_project.user_id, instructions: nil, components: [], - image_list: [] + image_list: [], + videos: [], + audio: [] }.to_json end @@ -168,7 +176,9 @@ name: project.name, user_id: project.user_id, components: [], - image_list: [] + image_list: [], + videos: [], + audio: [] }.to_json end