From 2dadc30db2ed7515eb3938aa6695c077337c8f09 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Fri, 5 Jun 2026 13:55:56 +0100 Subject: [PATCH 01/19] add starter yml --- .../scratch-integration-test-starter/project_config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 lib/tasks/project_components/scratch-integration-test-starter/project_config.yml diff --git a/lib/tasks/project_components/scratch-integration-test-starter/project_config.yml b/lib/tasks/project_components/scratch-integration-test-starter/project_config.yml new file mode 100644 index 000000000..a1a0201ed --- /dev/null +++ b/lib/tasks/project_components/scratch-integration-test-starter/project_config.yml @@ -0,0 +1,9 @@ +NAME: "scratch integration test" +IDENTIFIER: "editor-scratch-testing-starter" +TYPE: "code_editor_scratch" +COMPONENTS: + - name: "main" + extension: "sb3" + location: "main.sb3" + index: 0 + default: true From dfe4b87aa921fc049c1555b4e1dc870ef8a17d70 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Fri, 5 Jun 2026 13:57:14 +0100 Subject: [PATCH 02/19] allow processing sb3 files --- app/models/filesystem_project.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/filesystem_project.rb b/app/models/filesystem_project.rb index 3632ba3e2..29848710b 100644 --- a/app/models/filesystem_project.rb +++ b/app/models/filesystem_project.rb @@ -3,7 +3,7 @@ require 'yaml' class FilesystemProject - CODE_FORMATS = ['.py', '.csv', '.txt', '.html', '.css'].freeze + CODE_FORMATS = ['.py', '.csv', '.txt', '.html', '.css', '.sb3'].freeze PROJECTS_ROOT = Rails.root.join('lib/tasks/project_components') PROJECT_CONFIG = 'project_config.yml' From 2c317825195b90ca382b4aa282c519d3aceaa1a7 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Fri, 5 Jun 2026 13:57:32 +0100 Subject: [PATCH 03/19] initial parser attempt --- lib/sb3_parser.rb | 68 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 lib/sb3_parser.rb diff --git a/lib/sb3_parser.rb b/lib/sb3_parser.rb new file mode 100644 index 000000000..c81127d42 --- /dev/null +++ b/lib/sb3_parser.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'json' +require 'marcel' +require 'stringio' +require 'zip' + +class Sb3Parser + class MissingProjectJsonError < StandardError; end + class MissingAssetError < StandardError; end + + attr_reader :file_path + + def initialize(file_path:) + @file_path = file_path + end + + def parse + Zip::File.open(file_path) do |zip_file| + project_json = project_json_entry(zip_file) + content = JSON.parse(project_json.get_input_stream.read) + + output = { + scratch_component: { content: } + # assets: assets(zip_file, extract_asset_names(content)) + } + pp output + output + end + end + + private + + def project_json_entry(zip_file) + zip_file.find_entry('project.json') || raise(MissingProjectJsonError, 'project.json not found in SB3 archive') + end + + # def extract_asset_names(value) + # case value + # when Hash + # names = [] + # names << value['md5ext'] if value['md5ext'].is_a?(String) + # value.each_value { |item| names.concat(extract_asset_names(item)) } + # names.uniq + # when Array + # value.flat_map { |item| extract_asset_names(item) }.uniq + # else + # [] + # end + # end + + # def assets(zip_file, asset_names) + # entries_by_name = zip_file.each.reject(&:directory?).index_by { |entry| entry.name } + + # asset_names.map do |asset_name| + # entry = entries_by_name[asset_name] || raise(MissingAssetError, "asset #{asset_name} not found in SB3 archive") + # asset(entry) + # end + # end + + # def asset(entry) + # io = StringIO.new(entry.get_input_stream.read) + # content_type = Marcel::MimeType.for(io, name: entry.name) + # io.rewind + + # { filename: entry.name, io:, content_type: } + # end +end From 2277ac07a31654c87916f8a78b7dba3ac8652ad4 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Mon, 8 Jun 2026 11:21:44 +0100 Subject: [PATCH 04/19] update parser to work with GH webhook --- lib/sb3_parser.rb | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/sb3_parser.rb b/lib/sb3_parser.rb index c81127d42..173f2a888 100644 --- a/lib/sb3_parser.rb +++ b/lib/sb3_parser.rb @@ -9,14 +9,16 @@ class Sb3Parser class MissingProjectJsonError < StandardError; end class MissingAssetError < StandardError; end - attr_reader :file_path + attr_reader :component, :file_path, :io - def initialize(file_path:) - @file_path = file_path + def initialize(component: nil, file_path: nil) + @component = component + @file_path = component&.fetch(:file_path, nil) || file_path + @io = component&.fetch(:io, nil) end def parse - Zip::File.open(file_path) do |zip_file| + open_zip do |zip_file| project_json = project_json_entry(zip_file) content = JSON.parse(project_json.get_input_stream.read) @@ -31,6 +33,15 @@ def parse private + def open_zip + return Zip::File.open(file_path) { |zip_file| yield zip_file } if file_path + + io.rewind if io.respond_to?(:rewind) + result = nil + Zip::File.open_buffer(io.read) { |zip_file| result = yield zip_file } + result + end + def project_json_entry(zip_file) zip_file.find_entry('project.json') || raise(MissingProjectJsonError, 'project.json not found in SB3 archive') end From e539c07ecc04ab5c5c4968d8f19d86e426ef866c Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Mon, 8 Jun 2026 11:22:21 +0100 Subject: [PATCH 05/19] implement scratch importing in project importer --- lib/project_importer.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 2669336ba..c16ef51cc 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -20,6 +20,7 @@ def import! setup_project delete_components create_components + create_scratch_component delete_removed_media attach_media_if_needed @@ -39,16 +40,31 @@ def setup_project end def delete_components + return unless project.project_type != 'code_editor_scratch' + project.components.each(&:destroy) end def create_components + return unless project.project_type != 'code_editor_scratch' + components.each do |component| project_component = Component.new(**component) project.components << project_component end end + def create_scratch_component + return unless project.project_type == 'code_editor_scratch' + + components.each do |component| + next unless component[:extension] == 'sb3' + + parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content) + project.scratch_component = ScratchComponent.new(content: parsed_content) if parsed_content + end + end + def delete_removed_media return if removed_media_names.empty? From f0584784f41867c37244d027673897c7308339f7 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Mon, 8 Jun 2026 11:26:38 +0100 Subject: [PATCH 06/19] update upload job to accept and process sb3 files --- app/jobs/upload_job.rb | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/app/jobs/upload_job.rb b/app/jobs/upload_job.rb index a72de493f..65fa4bd6c 100644 --- a/app/jobs/upload_job.rb +++ b/app/jobs/upload_job.rb @@ -131,6 +131,11 @@ def categorize_files(files, project_dir, locale, repository, owner) } files.each do |file| + if file.extension == '.sb3' + categories[:components] << component(file, project_dir, locale, repository, owner) + next + end + mime_type = file_mime_type(file) case mime_type @@ -150,9 +155,11 @@ def categorize_files(files, project_dir, locale, repository, owner) categories end - def component(file) + def component(file, project_dir = nil, locale = nil, repository = nil, owner = nil) name = file.name.chomp(file.extension) extension = file.extension[1..] + return { name:, extension:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open } if extension == 'sb3' + content = file.object.text default = file.name == 'main.py' { name:, extension:, content:, default: } @@ -160,9 +167,12 @@ def component(file) def media(file, project_dir, locale, repository, owner) filename = file.name + { filename:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open } + end + + def file_url(file, project_dir, locale, repository, owner) directory = project_dir.name - url = "https://github.com/#{owner}/#{repository}/raw/#{ENV.fetch('GITHUB_WEBHOOK_REF')}/#{locale}/code/#{directory}/#{filename}" - { filename:, io: URI.parse(url).open } + "https://github.com/#{owner}/#{repository}/raw/#{ENV.fetch('GITHUB_WEBHOOK_REF')}/#{locale}/code/#{directory}/#{file.name}" end def repository(payload) From 29d0a7a09b8ce4e49e44f72de7250a4c78d9a5ba Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Mon, 8 Jun 2026 11:43:20 +0100 Subject: [PATCH 07/19] update filesystem_project for sb3 support --- app/models/filesystem_project.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/models/filesystem_project.rb b/app/models/filesystem_project.rb index 29848710b..906e43a3e 100644 --- a/app/models/filesystem_project.rb +++ b/app/models/filesystem_project.rb @@ -11,7 +11,8 @@ def self.import_all! PROJECTS_ROOT.each_child do |dir| proj_config = YAML.safe_load_file(dir.join(PROJECT_CONFIG).to_s) - files = dir.children.reject { |file| file.basename.to_s == 'project_config.yml' } + files = dir.children.reject { |file| file.basename.to_s == PROJECT_CONFIG } + files = configured_scratch_files(files, dir, proj_config) if proj_config['TYPE'] == Project::Types::CODE_EDITOR_SCRATCH categorized_files = categorize_files(files, dir) project_importer = ProjectImporter.new(name: proj_config['NAME'], identifier: proj_config['IDENTIFIER'], @@ -53,9 +54,18 @@ def self.categorize_files(files, dir) categories end + def self.configured_scratch_files(files, dir, proj_config) + configured_locations = Array(proj_config['COMPONENTS']).pluck('location') + return files if configured_locations.empty? + + files.reject { |file| File.extname(file) == '.sb3' && configured_locations.exclude?(file.basename.to_s) } + end + def self.component(file, dir) name = File.basename(file, '.*') extension = File.extname(file).delete('.') + return { name:, extension:, file_path: dir.join(File.basename(file)).to_s } if extension == 'sb3' + code = File.read(dir.join(File.basename(file)).to_s) default = (File.basename(file) == 'main.py') { name:, extension:, content: code, default: } From fdc972801481ba65dc01dc6f9e7209e85664b031 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Mon, 8 Jun 2026 15:15:40 +0100 Subject: [PATCH 08/19] tiny clean up on sb3 parser --- lib/sb3_parser.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/sb3_parser.rb b/lib/sb3_parser.rb index 173f2a888..aa0c5ffd2 100644 --- a/lib/sb3_parser.rb +++ b/lib/sb3_parser.rb @@ -26,15 +26,14 @@ def parse scratch_component: { content: } # assets: assets(zip_file, extract_asset_names(content)) } - pp output output end end private - def open_zip - return Zip::File.open(file_path) { |zip_file| yield zip_file } if file_path + def open_zip(&) + return Zip::File.open(file_path, &) if file_path io.rewind if io.respond_to?(:rewind) result = nil From bddaab629958eb40585ddf18c4cc6d0af923b986 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Tue, 9 Jun 2026 11:42:58 +0100 Subject: [PATCH 09/19] add asset importing for sb3 files --- lib/scratch_asset_importer.rb | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb index 632fba835..0d3694ea4 100644 --- a/lib/scratch_asset_importer.rb +++ b/lib/scratch_asset_importer.rb @@ -22,7 +22,11 @@ def show_progress? end end - attr_reader :asset_base_url, :asset_name + def self.import_from_sb3_assets(assets, asset_base_url) + new(nil, asset_base_url).import_from_sb3_assets(assets) + end + + attr_reader :asset_base_url, :asset_names ASSET_FETCHING_DELAY = 0.2 @@ -45,7 +49,18 @@ def asset end end - def create_scratch_asset + def import_from_sb3_assets(assets) + bar = ProgressBar.create(format: '%t: |%B| %c of %C %E', total: assets.count) if show_progress? + + assets.each do |asset| + bar.increment if show_progress? + import_sb3_asset(asset.fetch(:filename), asset.fetch(:io).read) + end + end + + private + + def create_scratch_asset(asset_names) return if ScratchAsset.global_assets.exists?(filename: asset_name) io = StringIO.new(asset.body) @@ -99,6 +114,17 @@ def s3_client ) end + def import_sb3_asset(asset_name, content) + return if ScratchAsset.global_assets.exists?(filename: asset_name) + + sleep(ASSET_FETCHING_DELAY) + ScratchAsset.create!(filename: asset_name, project_id: nil, uploaded_user_id: nil) + .file + .attach(io: StringIO.new(content), filename: asset_name) + rescue StandardError => e + Rails.logger.error("Failed to import SB3 asset #{asset_name}: #{e.message}") + end + def connection @connection ||= Faraday.new(url: asset_base_url) do |faraday| faraday.response :raise_error From 42c2ef2d9da663da316e40ece538d80691be6dd7 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Tue, 9 Jun 2026 11:43:21 +0100 Subject: [PATCH 10/19] update sb3 parser to pull files from zip --- lib/sb3_parser.rb | 56 +++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/lib/sb3_parser.rb b/lib/sb3_parser.rb index aa0c5ffd2..0dd14b4e9 100644 --- a/lib/sb3_parser.rb +++ b/lib/sb3_parser.rb @@ -23,8 +23,8 @@ def parse content = JSON.parse(project_json.get_input_stream.read) output = { - scratch_component: { content: } - # assets: assets(zip_file, extract_asset_names(content)) + scratch_component: { content: }, + assets: assets(zip_file, extract_asset_names(content)) } output end @@ -45,34 +45,32 @@ def project_json_entry(zip_file) zip_file.find_entry('project.json') || raise(MissingProjectJsonError, 'project.json not found in SB3 archive') end - # def extract_asset_names(value) - # case value - # when Hash - # names = [] - # names << value['md5ext'] if value['md5ext'].is_a?(String) - # value.each_value { |item| names.concat(extract_asset_names(item)) } - # names.uniq - # when Array - # value.flat_map { |item| extract_asset_names(item) }.uniq - # else - # [] - # end - # end - - # def assets(zip_file, asset_names) - # entries_by_name = zip_file.each.reject(&:directory?).index_by { |entry| entry.name } + def extract_asset_names(value) + case value + when Hash + names = [] + names << value['md5ext'] if value['md5ext'].is_a?(String) + value.each_value { |item| names.concat(extract_asset_names(item)) } + names.uniq + when Array + value.flat_map { |item| extract_asset_names(item) }.uniq + else + [] + end + end - # asset_names.map do |asset_name| - # entry = entries_by_name[asset_name] || raise(MissingAssetError, "asset #{asset_name} not found in SB3 archive") - # asset(entry) - # end - # end + def assets(zip_file, asset_names) + asset_names.map do |asset_name| + entry = zip_file.find_entry(asset_name) || raise(MissingAssetError, "asset #{asset_name} not found in SB3 archive") + asset(entry) + end + end - # def asset(entry) - # io = StringIO.new(entry.get_input_stream.read) - # content_type = Marcel::MimeType.for(io, name: entry.name) - # io.rewind + def asset(entry) + io = StringIO.new(entry.get_input_stream.read) + content_type = Marcel::MimeType.for(io, name: entry.name) + io.rewind - # { filename: entry.name, io:, content_type: } - # end + { filename: entry.name, io:, content_type: } + end end From e63e015abfdc0e64f89ecfd2ffeb81b2765d7771 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Tue, 9 Jun 2026 11:43:34 +0100 Subject: [PATCH 11/19] call asset importer in project importer --- lib/project_importer.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index c16ef51cc..ea0994e4d 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -21,6 +21,7 @@ def import! delete_components create_components create_scratch_component + create_scratch_assets delete_removed_media attach_media_if_needed @@ -60,11 +61,22 @@ def create_scratch_component components.each do |component| next unless component[:extension] == 'sb3' - parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content) + parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component) project.scratch_component = ScratchComponent.new(content: parsed_content) if parsed_content end end + def create_scratch_assets + return unless project.project_type == 'code_editor_scratch' + + components.each do |component| + next unless component[:extension] == 'sb3' + + parsed_assets = Sb3Parser.new(component: component).parse.fetch(:assets) + ScratchAssetImporter.import_from_sb3_assets(parsed_assets, 'test.com') + end + end + def delete_removed_media return if removed_media_names.empty? From 5a4c68bf3d99dbb79403443f70288591c327e012 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Tue, 9 Jun 2026 15:45:26 +0100 Subject: [PATCH 12/19] fix import bug --- lib/project_importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index ea0994e4d..88c72c4b9 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -61,7 +61,7 @@ def create_scratch_component components.each do |component| next unless component[:extension] == 'sb3' - parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component) + parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content) project.scratch_component = ScratchComponent.new(content: parsed_content) if parsed_content end end From 676f9fe77d42a1f2a4641f6720c6ba148fc23df5 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 11 Jun 2026 11:44:22 +0100 Subject: [PATCH 13/19] refactor asset importer for consistency with original importer --- lib/project_importer.rb | 2 +- lib/scratch_asset_importer.rb | 43 +++++++++++++++++------------------ 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 88c72c4b9..73f8b2c2d 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -73,7 +73,7 @@ def create_scratch_assets next unless component[:extension] == 'sb3' parsed_assets = Sb3Parser.new(component: component).parse.fetch(:assets) - ScratchAssetImporter.import_from_sb3_assets(parsed_assets, 'test.com') + ScratchAssetImporter.import_from_sb3(parsed_assets) end end diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb index 0d3694ea4..aea7cc0d5 100644 --- a/lib/scratch_asset_importer.rb +++ b/lib/scratch_asset_importer.rb @@ -14,6 +14,13 @@ def import_all(asset_names, asset_base_url) new(asset_name, asset_base_url).import end end + + def import_from_sb3(assets) + + assets.each do |asset| + new(nil, nil).import_from_sb3(asset) + end + end private @@ -22,11 +29,7 @@ def show_progress? end end - def self.import_from_sb3_assets(assets, asset_base_url) - new(nil, asset_base_url).import_from_sb3_assets(assets) - end - - attr_reader :asset_base_url, :asset_names + attr_reader :asset_base_url, :asset_name ASSET_FETCHING_DELAY = 0.2 @@ -49,13 +52,8 @@ def asset end end - def import_from_sb3_assets(assets) - bar = ProgressBar.create(format: '%t: |%B| %c of %C %E', total: assets.count) if show_progress? - - assets.each do |asset| - bar.increment if show_progress? - import_sb3_asset(asset.fetch(:filename), asset.fetch(:io).read) - end + def import_from_sb3(asset) + create_sb3_asset(asset.fetch(:filename), asset.fetch(:io).read) end private @@ -70,6 +68,17 @@ def create_scratch_asset(asset_names) .attach(io:, filename: asset_name) end + def create_sb3_asset(asset_name, content) + return if ScratchAsset.global_assets.exists?(filename: asset_name) + + sleep(ASSET_FETCHING_DELAY) + ScratchAsset.create!(filename: asset_name, project_id: nil, uploaded_user_id: nil) + .file + .attach(io: StringIO.new(content), filename: asset_name) + rescue StandardError => e + Rails.logger.error("Failed to import SB3 asset #{asset_name}: #{e.message}") + end + def save_to_editor_asset_bucket return unless save_to_editor_asset_bucket? @@ -114,16 +123,6 @@ def s3_client ) end - def import_sb3_asset(asset_name, content) - return if ScratchAsset.global_assets.exists?(filename: asset_name) - - sleep(ASSET_FETCHING_DELAY) - ScratchAsset.create!(filename: asset_name, project_id: nil, uploaded_user_id: nil) - .file - .attach(io: StringIO.new(content), filename: asset_name) - rescue StandardError => e - Rails.logger.error("Failed to import SB3 asset #{asset_name}: #{e.message}") - end def connection @connection ||= Faraday.new(url: asset_base_url) do |faraday| From 888151ca19082c52e045013819b726fc058a98ce Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 11 Jun 2026 12:10:24 +0100 Subject: [PATCH 14/19] clean up --- lib/scratch_asset_importer.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb index aea7cc0d5..0abe6e699 100644 --- a/lib/scratch_asset_importer.rb +++ b/lib/scratch_asset_importer.rb @@ -14,9 +14,8 @@ def import_all(asset_names, asset_base_url) new(asset_name, asset_base_url).import end end - + def import_from_sb3(assets) - assets.each do |asset| new(nil, nil).import_from_sb3(asset) end @@ -58,7 +57,7 @@ def import_from_sb3(asset) private - def create_scratch_asset(asset_names) + def create_scratch_asset return if ScratchAsset.global_assets.exists?(filename: asset_name) io = StringIO.new(asset.body) @@ -123,7 +122,6 @@ def s3_client ) end - def connection @connection ||= Faraday.new(url: asset_base_url) do |faraday| faraday.response :raise_error From d9eed537009ff499b69f15a7bf2aba3e49e4a696 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 11 Jun 2026 14:12:56 +0100 Subject: [PATCH 15/19] separate scratch component in upload job --- app/jobs/upload_job.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/jobs/upload_job.rb b/app/jobs/upload_job.rb index 65fa4bd6c..53eb178b8 100644 --- a/app/jobs/upload_job.rb +++ b/app/jobs/upload_job.rb @@ -132,7 +132,7 @@ def categorize_files(files, project_dir, locale, repository, owner) files.each do |file| if file.extension == '.sb3' - categories[:components] << component(file, project_dir, locale, repository, owner) + categories[:components] << scratch_file_component(file, project_dir, locale, repository, owner) next end @@ -155,16 +155,20 @@ def categorize_files(files, project_dir, locale, repository, owner) categories end - def component(file, project_dir = nil, locale = nil, repository = nil, owner = nil) + def component(file) name = file.name.chomp(file.extension) extension = file.extension[1..] - return { name:, extension:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open } if extension == 'sb3' - content = file.object.text default = file.name == 'main.py' { name:, extension:, content:, default: } end + def scratch_file_component(file, project_dir, locale, repository, owner) + name = file.name.chomp(file.extension) + extension = file.extension[1..] + { name:, extension:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open } + end + def media(file, project_dir, locale, repository, owner) filename = file.name { filename:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open } From a6ac0ac2bd1d939770719b66d7e32079407d866b Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 11 Jun 2026 15:40:55 +0100 Subject: [PATCH 16/19] rubocop --- app/models/filesystem_project.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/filesystem_project.rb b/app/models/filesystem_project.rb index 906e43a3e..aa5ce5753 100644 --- a/app/models/filesystem_project.rb +++ b/app/models/filesystem_project.rb @@ -12,7 +12,7 @@ def self.import_all! proj_config = YAML.safe_load_file(dir.join(PROJECT_CONFIG).to_s) files = dir.children.reject { |file| file.basename.to_s == PROJECT_CONFIG } - files = configured_scratch_files(files, dir, proj_config) if proj_config['TYPE'] == Project::Types::CODE_EDITOR_SCRATCH + files = configured_scratch_files(files, proj_config) if proj_config['TYPE'] == Project::Types::CODE_EDITOR_SCRATCH categorized_files = categorize_files(files, dir) project_importer = ProjectImporter.new(name: proj_config['NAME'], identifier: proj_config['IDENTIFIER'], @@ -54,7 +54,7 @@ def self.categorize_files(files, dir) categories end - def self.configured_scratch_files(files, dir, proj_config) + def self.configured_scratch_files(files, proj_config) configured_locations = Array(proj_config['COMPONENTS']).pluck('location') return files if configured_locations.empty? From 5c0c65a401e8364315b73b2f3285d1659df16165 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 11 Jun 2026 16:20:59 +0100 Subject: [PATCH 17/19] add sb3 parser tests --- spec/lib/sb3_parser_spec.rb | 84 ++++++++++++++++++++++++++++++ spec/support/sb3_archive_helper.rb | 24 +++++++++ 2 files changed, 108 insertions(+) create mode 100644 spec/lib/sb3_parser_spec.rb create mode 100644 spec/support/sb3_archive_helper.rb diff --git a/spec/lib/sb3_parser_spec.rb b/spec/lib/sb3_parser_spec.rb new file mode 100644 index 000000000..492172d96 --- /dev/null +++ b/spec/lib/sb3_parser_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'sb3_parser' + +RSpec.describe Sb3Parser do + describe '#parse' do + let(:png_content) { sb3_fixture_content('test_image_1.png') } + let(:mp3_content) { sb3_fixture_content('test_audio_1.mp3') } + let(:project_json) do + { + targets: [ + { + costumes: [ + { name: 'cat', md5ext: 'abc123.png' }, + { name: 'duplicate cat', md5ext: 'abc123.png' } + ], + sounds: [ + { name: 'meow', md5ext: 'def456.mp3' } + ] + } + ] + } + end + let(:entries) do + { + 'project.json' => project_json.to_json, + 'abc123.png' => png_content, + 'def456.mp3' => mp3_content + } + end + + it 'parses project.json and referenced assets from component io' do + result = described_class.new(component: { io: sb3_archive(entries) }).parse + + expect(result.fetch(:scratch_component).fetch(:content)).to eq(JSON.parse(project_json.to_json)) + + assets = result.fetch(:assets) + expect(assets.map { |asset| asset.fetch(:filename) }).to contain_exactly('abc123.png', 'def456.mp3') + + png_asset = assets.find { |asset| asset.fetch(:filename) == 'abc123.png' } + expect(png_asset.fetch(:content_type)).to eq('image/png') + expect(png_asset.fetch(:io).read).to eq(png_content) + end + + it 'parses an archive from a file path' do + Tempfile.create(['scratch-project', '.sb3']) do |file| + archive = sb3_archive(entries) + file.binmode + file.write(archive.read) + file.flush + + result = described_class.new(file_path: file.path).parse + + expect(result.fetch(:scratch_component).fetch(:content)).to eq(JSON.parse(project_json.to_json)) + expect(result.fetch(:assets).map { |asset| asset.fetch(:filename) }).to contain_exactly('abc123.png', 'def456.mp3') + end + end + + it 'returns no assets when project.json does not reference any md5ext values' do + archive = sb3_archive('project.json' => { targets: [] }.to_json) + + result = described_class.new(component: { io: archive }).parse + + expect(result.fetch(:assets)).to eq([]) + end + + it 'raises when project.json is missing' do + archive = sb3_archive('abc123.png' => png_content) + + expect do + described_class.new(component: { io: archive }).parse + end.to raise_error(described_class::MissingProjectJsonError, 'project.json not found in SB3 archive') + end + + it 'raises when a referenced asset is missing' do + archive = sb3_archive('project.json' => { targets: [{ costumes: [{ md5ext: 'missing.png' }] }] }.to_json) + + expect do + described_class.new(component: { io: archive }).parse + end.to raise_error(described_class::MissingAssetError, 'asset missing.png not found in SB3 archive') + end + end +end diff --git a/spec/support/sb3_archive_helper.rb b/spec/support/sb3_archive_helper.rb new file mode 100644 index 000000000..4c920b642 --- /dev/null +++ b/spec/support/sb3_archive_helper.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Sb3ArchiveHelper + def sb3_archive(entries) + Zip::OutputStream.write_buffer do |zip| + entries.each do |name, content| + zip.put_next_entry(name) + zip.write(content) + end + end.tap(&:rewind) + end + + def sb3_archive_string(entries) + sb3_archive(entries).string + end + + def sb3_fixture_content(filename) + Rails.root.join('spec/fixtures/files', filename).binread + end +end + +RSpec.configure do |config| + config.include Sb3ArchiveHelper +end From 5ec7c7227db9edc94cd2c732b8dd61124db0b0e0 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Fri, 12 Jun 2026 08:38:33 +0100 Subject: [PATCH 18/19] clear up confusing naming --- lib/project_importer.rb | 2 +- lib/scratch_asset_importer.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 73f8b2c2d..d04216ee0 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -73,7 +73,7 @@ def create_scratch_assets next unless component[:extension] == 'sb3' parsed_assets = Sb3Parser.new(component: component).parse.fetch(:assets) - ScratchAssetImporter.import_from_sb3(parsed_assets) + ScratchAssetImporter.import_all_from_sb3(parsed_assets) end end diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb index 0abe6e699..e9a98449d 100644 --- a/lib/scratch_asset_importer.rb +++ b/lib/scratch_asset_importer.rb @@ -15,7 +15,7 @@ def import_all(asset_names, asset_base_url) end end - def import_from_sb3(assets) + def import_all_from_sb3(assets) assets.each do |asset| new(nil, nil).import_from_sb3(asset) end From 6ae3fd725751d0c825e963a35f3da3f499ad8c1c Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Fri, 12 Jun 2026 08:39:43 +0100 Subject: [PATCH 19/19] update tests for sb3 import functionality --- spec/jobs/upload_job_spec.rb | 110 ++++++++++++++++++++++++ spec/lib/project_importer_spec.rb | 101 ++++++++++++++++++++++ spec/lib/scratch_asset_importer_spec.rb | 59 +++++++++++++ 3 files changed, 270 insertions(+) diff --git a/spec/jobs/upload_job_spec.rb b/spec/jobs/upload_job_spec.rb index 61d5bb86a..be69fe875 100644 --- a/spec/jobs/upload_job_spec.rb +++ b/spec/jobs/upload_job_spec.rb @@ -269,6 +269,116 @@ end end + context 'when a scratch project is uploaded' do + let(:scratch_payload) do + { + repository: { name: 'my-amazing-repo', owner: { name: 'me' } }, + commits: [{ added: ['ja-JP/code/scratch-integration-test-starter/main.sb3'], modified: [], removed: [] }] + } + end + let(:scratch_project_json) do + { + targets: [ + { + costumes: [{ md5ext: 'test_image_1.png' }], + sounds: [{ md5ext: 'test_audio_1.mp3' }] + } + ] + } + end + let(:scratch_sb3_body) do + sb3_archive_string( + 'project.json' => scratch_project_json.to_json, + 'test_image_1.png' => sb3_fixture_content('test_image_1.png'), + 'test_audio_1.mp3' => sb3_fixture_content('test_audio_1.mp3') + ) + end + let(:raw_response) do + { + data: { + repository: { + object: { + __typename: 'Tree', + entries: [ + { + name: 'scratch-integration-test-starter', + object: { + __typename: 'Tree', + entries: [ + { + name: 'main.sb3', + extension: '.sb3', + object: { + __typename: 'Blob', + text: nil, + isBinary: true + } + }, + { + name: 'project_config.yml', + extension: '.yml', + object: { + __typename: 'Blob', + text: "name: \"Scratch Integration Test\"\nidentifier: \"scratch-integration-test-starter\"\ntype: \"code_editor_scratch\"\n", + isBinary: false + } + } + ] + } + } + ] + } + } + } + }.deep_stringify_keys + end + + before do + allow(GithubApi::Client).to receive(:query).and_return(graphql_response) + allow(ProjectImporter).to receive(:new).and_call_original + + stub_request(:get, 'https://github.com/me/my-amazing-repo/raw/branches/whatever/ja-JP/code/scratch-integration-test-starter/main.sb3') + .to_return(status: 200, body: scratch_sb3_body, headers: {}) + end + + it 'imports the Scratch project with the sb3 component as io' do + described_class.perform_now(scratch_payload) + + expect(ProjectImporter).to have_received(:new).with( + hash_including( + name: 'Scratch Integration Test', + identifier: 'scratch-integration-test-starter', + type: Project::Types::CODE_EDITOR_SCRATCH, + locale: 'ja-JP', + images: [], + videos: [], + audio: [], + components: [ + hash_including( + name: 'main', + extension: 'sb3', + io: an_object_responding_to(:read) + ) + ] + ) + ) + end + + it 'requests the sb3 file from the correct URL' do + described_class.perform_now(scratch_payload) + + expect(WebMock).to have_requested(:get, 'https://github.com/me/my-amazing-repo/raw/branches/whatever/ja-JP/code/scratch-integration-test-starter/main.sb3').once + end + + it 'saves the Scratch project to the database' do + expect { described_class.perform_now(scratch_payload) }.to change(Project, :count).by(1) + + project = Project.find_by(identifier: 'scratch-integration-test-starter', locale: 'ja-JP') + expect(project.project_type).to eq(Project::Types::CODE_EDITOR_SCRATCH) + expect(project.scratch_component.content).to eq(JSON.parse(scratch_project_json.to_json)) + end + end + context 'when locale is unsupported' do let(:raw_response) { { data: { repository: nil } } } let(:bad_payload) do diff --git a/spec/lib/project_importer_spec.rb b/spec/lib/project_importer_spec.rb index 7d847dc54..f38380543 100644 --- a/spec/lib/project_importer_spec.rb +++ b/spec/lib/project_importer_spec.rb @@ -114,4 +114,105 @@ expect { importer.import! }.to change { project.reload.audio[0].filename.to_s }.to('my-amazing-audio.mp3') end end + + context 'when the project has type code_editor_scratch' do + let(:scratch_project_file) { Tempfile.new(['test_scratch_project', '.sb3']) } + let(:importer) do + described_class.new( + name: 'My amazing Scratch project', + identifier: 'my-amazing-scratch-project', + type: Project::Types::CODE_EDITOR_SCRATCH, + locale: 'en', + components: [ + { name: 'main', extension: 'sb3', file_path: scratch_project_file.path } + ] + ) + end + let(:scratch_project_content) do + { + targets: [ + { + costumes: [{ md5ext: 'test_image_1.png' }], + sounds: [{ md5ext: 'test_audio_1.mp3' }], + videos: [{ md5ext: 'test_video_1.mp4' }] + } + ] + } + end + + let(:project) { Project.find_by(identifier: importer.identifier, user_id: nil, locale: importer.locale) } + + before do + scratch_project_file.binmode + scratch_project_file.write( + sb3_archive( + 'project.json' => scratch_project_content.to_json, + 'test_image_1.png' => sb3_fixture_content('test_image_1.png'), + 'test_video_1.mp4' => sb3_fixture_content('test_video_1.mp4'), + 'test_audio_1.mp3' => sb3_fixture_content('test_audio_1.mp3') + ).read + ) + scratch_project_file.flush + end + + after do + scratch_project_file.close + scratch_project_file.unlink + end + + context 'when importing a new scratch project' do + it 'imports the Scratch component content' do + importer.import! + + expect(project.components.count).to eq(0) + expect(project.scratch_component.content).to eq(JSON.parse(scratch_project_content.to_json)) + end + + it 'imports the project assets' do + importer.import! + expect(ScratchAsset.global_assets.where(filename: ['test_image_1.png', 'test_video_1.mp4', 'test_audio_1.mp3']).count).to eq(3) + end + end + + context 'when the scratch project already exists in the database' do + let(:original_scratch_content) do + { targets: ['old target'], monitors: [], extensions: [], meta: {} } + end + let!(:project) do + create( + :project, + identifier: 'my-amazing-scratch-project', + locale: 'en', + project_type: Project::Types::CODE_EDITOR_SCRATCH, + name: 'Old Scratch project name' + ) + end + + before do + create(:scratch_component, project:, content: original_scratch_content) + end + + it 'does not create a new project' do + expect { importer.import! }.not_to change(Project, :count) + end + + it 'updates the project name' do + expect { importer.import! }.to change { project.reload.name }.to(importer.name) + end + + it 'updates the scratch component content' do + importer.import! + + expect(project.reload.scratch_component.content).to eq(JSON.parse(scratch_project_content.to_json)) + end + + it 'imports any new assets without duplicating existing ones' do + create(:scratch_asset, :with_file, filename: 'test_image_1.png') + + importer.import! + + expect(ScratchAsset.global_assets.where(filename: ['test_image_1.png', 'test_video_1.mp4', 'test_audio_1.mp3']).count).to eq(3) + end + end + end end diff --git a/spec/lib/scratch_asset_importer_spec.rb b/spec/lib/scratch_asset_importer_spec.rb index d8e209c2c..9cfedd2c6 100644 --- a/spec/lib/scratch_asset_importer_spec.rb +++ b/spec/lib/scratch_asset_importer_spec.rb @@ -118,4 +118,63 @@ end end end + + describe '.import_all_from_sb3' do + def sb3_asset(filename, content = sb3_fixture_content(filename)) + { filename:, io: StringIO.new(content) } + end + + it 'imports assets from SB3 archive content' do + png_content = sb3_fixture_content('test_image_1.png') + + described_class.import_all_from_sb3([sb3_asset('test_image_1.png', png_content)]) + + scratch_asset = ScratchAsset.find_by(filename: 'test_image_1.png') + expect(scratch_asset).to be_present + expect(scratch_asset).to be_global + expect(scratch_asset.file.download).to eq(png_content) + end + + it 'does nothing if global asset already exists' do + create(:scratch_asset, :with_file, filename: 'test_image_1.png') + + expect do + described_class.import_all_from_sb3([sb3_asset('test_image_1.png')]) + end.not_to change(ScratchAsset, :count) + end + + it 'can import multiple assets' do + described_class.import_all_from_sb3([ + sb3_asset('test_image_1.png'), + sb3_asset('test_audio_1.mp3', sb3_fixture_content('test_audio_1.mp3')) + ]) + + expect(ScratchAsset.find_by(filename: 'test_image_1.png')).to be_present + expect(ScratchAsset.find_by(filename: 'test_audio_1.mp3')).to be_present + end + + it 'still imports a global asset when a project asset already uses the filename' do + project = create(:project, project_type: Project::Types::CODE_EDITOR_SCRATCH, locale: nil, user_id: SecureRandom.uuid) + create(:scratch_component, project:) + create(:scratch_asset, :with_file, filename: 'test_image_1.png', project:) + + expect do + described_class.import_all_from_sb3([sb3_asset('test_image_1.png')]) + end.to change { ScratchAsset.global_assets.where(filename: 'test_image_1.png').count }.by(1) + end + + it 'skips assets that fail to import' do + allow(ScratchAsset).to receive(:create!).and_call_original + allow(ScratchAsset).to receive(:create!).with(filename: 'failing.png', project_id: nil, uploaded_user_id: nil) + .and_raise(StandardError, 'attach failed') + + described_class.import_all_from_sb3([ + sb3_asset('failing.png', 'bad'), + sb3_asset('test_image_1.png') + ]) + + expect(ScratchAsset.find_by(filename: 'failing.png')).not_to be_present + expect(ScratchAsset.find_by(filename: 'test_image_1.png')).to be_present + end + end end