Importing projects site projects from .sb3 files

app/jobs/upload_job.rb

@@ -131,6 +131,11 @@ def categorize_files(files, project_dir, locale, repository, owner)
     }
 
     files.each do |file|
+      if file.extension == '.sb3'
+        categories[:components] << scratch_file_component(file, project_dir, locale, repository, owner)
+        next
+      end
+
       mime_type = file_mime_type(file)
 
       case mime_type
@@ -158,11 +163,18 @@ def component(file)
     { name:, extension:, content:, default: }
   end
 
+  def scratch_file_component(file, project_dir, locale, repository, owner)
+    name = file.name.chomp(file.extension)
+    extension = file.extension[1..]
+    { name:, extension:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open }
+  end
+
   def media(file, project_dir, locale, repository, owner)
-    filename = file.name
-    directory = project_dir.name
-    url = "https://github.com/#{owner}/#{repository}/raw/#{ENV.fetch('GITHUB_WEBHOOK_REF')}/#{locale}/code/#{directory}/#{filename}"
-    { filename:, io: URI.parse(url).open }
+    { filename: file.name, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open }
+  end
+
+  def file_url(file, project_dir, locale, repository, owner)
+    "https://github.com/#{owner}/#{repository}/raw/#{ENV.fetch('GITHUB_WEBHOOK_REF')}/#{locale}/code/#{project_dir.name}/#{file.name}"
   end
 
   def repository(payload)

app/models/filesystem_project.rb

@@ -3,15 +3,15 @@
 require 'yaml'
 
 class FilesystemProject
-  CODE_FORMATS = ['.py', '.csv', '.txt', '.html', '.css'].freeze
+  CODE_FORMATS = ['.py', '.csv', '.txt', '.html', '.css', '.sb3'].freeze
   PROJECTS_ROOT = Rails.root.join('lib/tasks/project_components')
   PROJECT_CONFIG = 'project_config.yml'
 
   def self.import_all!
     PROJECTS_ROOT.each_child do |dir|
       proj_config = YAML.safe_load_file(dir.join(PROJECT_CONFIG).to_s)
 
-      files = dir.children.reject { |file| file.basename.to_s == 'project_config.yml' }
+      files = dir.children.reject { |file| file.basename.to_s == PROJECT_CONFIG }
       categorized_files = categorize_files(files, dir)
 
       project_importer = ProjectImporter.new(name: proj_config['NAME'], identifier: proj_config['IDENTIFIER'],
@@ -56,6 +56,8 @@ def self.categorize_files(files, dir)
   def self.component(file, dir)
     name = File.basename(file, '.*')
     extension = File.extname(file).delete('.')
+    return { name:, extension:, file_path: dir.join(File.basename(file)).to_s } if extension.casecmp?('sb3')
+
     code = File.read(dir.join(File.basename(file)).to_s)
     default = (File.basename(file) == 'main.py')
     { name:, extension:, content: code, default: }

app/models/scratch_asset.rb

@@ -5,8 +5,6 @@ class ScratchAsset < ApplicationRecord
   has_one_attached :file
 
   validates :filename, presence: true, uniqueness: { scope: %i[project_id uploaded_user_id] }
-  validates :uploaded_user_id, absence: true, if: :global?
-  validates :uploaded_user_id, presence: true, unless: :global?
   validate :belongs_to_scratch_project
 
   scope :global_assets, -> { where(project_id: nil, uploaded_user_id: nil) }

lib/project_importer.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ProjectImporter
+  class ImportError < StandardError; end
+
   attr_reader :name, :identifier, :images, :videos, :audio, :media, :components, :type, :locale
 
   def initialize(**kwargs)
@@ -20,6 +22,7 @@ def import!
       setup_project
       delete_components
       create_components
+      create_scratch_component
       delete_removed_media
       attach_media_if_needed
 
@@ -39,16 +42,50 @@ def setup_project
   end
 
   def delete_components
+    return if project.scratch_project?
+
     project.components.each(&:destroy)
   end
 
   def create_components
+    return if project.scratch_project?
+
     components.each do |component|
+      # .sb3 files are only ever imported as a ScratchComponent (see
+      # create_scratch_component); they carry an :io/:file_path key that is not a
+      # Component attribute, so skip them here to avoid building invalid rows.
+      next if component[:extension]&.casecmp?('sb3')
+
       project_component = Component.new(**component)
       project.components << project_component
     end
   end
 
+  def create_scratch_component
+    return unless project.scratch_project?
+
+    component = components[0]
+    return unless component&.fetch(:extension, nil)&.casecmp?('sb3')
+
+    parsed_content = Sb3Parser.new(component: component).parse
+    project_content = parsed_content.dig(:scratch_component, :content)
+    assets = parsed_content[:assets] || []
+
+    raise ImportError, 'Scratch project content could not be parsed' if project_content.blank?
+
+    project.scratch_component = ScratchComponent.new(content: project_content)
+    project.scratch_assets = assets.map { create_scratch_asset(it) }
+  end
+
+  def create_scratch_asset(asset)
+    filename = asset[:filename]
+    io = asset[:io]
+
+    asset = ScratchAsset.new(filename:, uploaded_user_id: nil)
+    asset.file.attach(io:, filename:)
+    asset
+  end
+
   def delete_removed_media
     return if removed_media_names.empty?
 

lib/sb3_parser.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'marcel'
+require 'stringio'
+require 'zip'
+
+class Sb3Parser
+  class MissingProjectJsonError < StandardError; end
+  class MissingAssetError < StandardError; end
+
+  attr_reader :component, :file_path, :io
+
+  def initialize(component: nil, file_path: nil)
+    @component = component
+    @file_path = component&.fetch(:file_path, nil) || file_path
+    @io = component&.fetch(:io, nil)
+  end
+
+  def parse
+    open_zip do |zip_file|
+      project_json = project_json_entry(zip_file)
+      content = JSON.parse(project_json.get_input_stream.read)
+
+      output = {
+        scratch_component: { content: },
+        assets: assets(zip_file, extract_asset_names(content))
+      }
+      output
+    end
+  end
+
+  private
+
+  def open_zip(&)
+    return Zip::File.open(file_path, &) if file_path
+
+    io.rewind if io.respond_to?(:rewind)
+    Zip::File.open_buffer(io.read) do |zip_file|
+      return yield zip_file
+    end
+  end
+
+  def project_json_entry(zip_file)
+    zip_file.find_entry('project.json') || raise(MissingProjectJsonError, 'project.json not found in SB3 archive')
+  end
+
+  def extract_asset_names(value)
+    case value
+    when Hash
+      names = []
+      names << value['md5ext'] if value['md5ext'].is_a?(String)
+      value.each_value { |item| names.concat(extract_asset_names(item)) }
+      names.uniq
+    when Array
+      value.flat_map { |item| extract_asset_names(item) }.uniq
+    else
+      []
+    end
+  end
+
+  def assets(zip_file, asset_names)
+    asset_names.map do |asset_name|
+      entry = zip_file.find_entry(asset_name) || raise(MissingAssetError, "asset #{asset_name} not found in SB3 archive")
+      asset(entry)
+    end
+  end
+
+  def asset(entry)
+    io = StringIO.new(entry.get_input_stream.read)
+    content_type = Marcel::MimeType.for(io, name: entry.name)
+    io.rewind
+
+    { filename: entry.name, io:, content_type: }
+  end
+end

lib/scratch_asset_importer.rb

@@ -45,6 +45,8 @@ def asset
     end
   end
 
+  private
+
   def create_scratch_asset
     return if ScratchAsset.global_assets.exists?(filename: asset_name)
 

lib/tasks/project_components/scratch-integration-test-starter/project_config.yml

@@ -0,0 +1,3 @@
+NAME: "scratch integration test"
+IDENTIFIER: "editor-scratch-testing-starter"
+TYPE: "code_editor_scratch"

spec/jobs/upload_job_spec.rb

@@ -269,6 +269,116 @@
     end
   end
 
+  context 'when a scratch project is uploaded' do
+    let(:scratch_payload) do
+      {
+        repository: { name: 'my-amazing-repo', owner: { name: 'me' } },
+        commits: [{ added: ['ja-JP/code/scratch-integration-test-starter/main.sb3'], modified: [], removed: [] }]
+      }
+    end
+    let(:scratch_project_json) do
+      {
+        targets: [
+          {
+            costumes: [{ md5ext: 'test_image_1.png' }],
+            sounds: [{ md5ext: 'test_audio_1.mp3' }]
+          }
+        ]
+      }
+    end
+    let(:scratch_sb3_body) do
+      sb3_archive_string(
+        'project.json' => scratch_project_json.to_json,
+        'test_image_1.png' => sb3_fixture_content('test_image_1.png'),
+        'test_audio_1.mp3' => sb3_fixture_content('test_audio_1.mp3')
+      )
+    end
+    let(:raw_response) do
+      {
+        data: {
+          repository: {
+            object: {
+              __typename: 'Tree',
+              entries: [
+                {
+                  name: 'scratch-integration-test-starter',
+                  object: {
+                    __typename: 'Tree',
+                    entries: [
+                      {
+                        name: 'main.sb3',
+                        extension: '.sb3',
+                        object: {
+                          __typename: 'Blob',
+                          text: nil,
+                          isBinary: true
+                        }
+                      },
+                      {
+                        name: 'project_config.yml',
+                        extension: '.yml',
+                        object: {
+                          __typename: 'Blob',
+                          text: "name: \"Scratch Integration Test\"\nidentifier: \"scratch-integration-test-starter\"\ntype: \"code_editor_scratch\"\n",
+                          isBinary: false
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            }
+          }
+        }
+      }.deep_stringify_keys
+    end
+
+    before do
+      allow(GithubApi::Client).to receive(:query).and_return(graphql_response)
+      allow(ProjectImporter).to receive(:new).and_call_original
+
+      stub_request(:get, 'https://github.com/me/my-amazing-repo/raw/branches/whatever/ja-JP/code/scratch-integration-test-starter/main.sb3')
+        .to_return(status: 200, body: scratch_sb3_body, headers: {})
+    end
+
+    it 'imports the Scratch project with the sb3 component as io' do
+      described_class.perform_now(scratch_payload)
+
+      expect(ProjectImporter).to have_received(:new).with(
+        hash_including(
+          name: 'Scratch Integration Test',
+          identifier: 'scratch-integration-test-starter',
+          type: Project::Types::CODE_EDITOR_SCRATCH,
+          locale: 'ja-JP',
+          images: [],
+          videos: [],
+          audio: [],
+          components: [
+            hash_including(
+              name: 'main',
+              extension: 'sb3',
+              io: an_object_responding_to(:read)
+            )
+          ]
+        )
+      )
+    end
+
+    it 'requests the sb3 file from the correct URL' do
+      described_class.perform_now(scratch_payload)
+
+      expect(WebMock).to have_requested(:get, 'https://github.com/me/my-amazing-repo/raw/branches/whatever/ja-JP/code/scratch-integration-test-starter/main.sb3').once
+    end
+
+    it 'saves the Scratch project to the database' do
+      expect { described_class.perform_now(scratch_payload) }.to change(Project, :count).by(1)
+
+      project = Project.find_by(identifier: 'scratch-integration-test-starter', locale: 'ja-JP')
+      expect(project.project_type).to eq(Project::Types::CODE_EDITOR_SCRATCH)
+      expect(project.scratch_component.content).to eq(JSON.parse(scratch_project_json.to_json))
+    end
+  end
+
   context 'when locale is unsupported' do
     let(:raw_response) { { data: { repository: nil } } }
     let(:bad_payload) do