From e2b104b85f3df239b4b41499aec72a5fc74920e3 Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Tue, 23 Jan 2024 20:10:15 +0000 Subject: [PATCH 01/12] Download all listed dependency packages --- lib/importmap/commands.rb | 21 ++--- lib/importmap/package.rb | 117 ++++++++++++++++++++++++++++ lib/importmap/packager.rb | 107 ++++++++----------------- test/packager_integration_test.rb | 104 ++++++++++++++++++++----- test/packager_single_quotes_test.rb | 2 +- test/packager_test.rb | 48 ++++++++---- 6 files changed, 274 insertions(+), 125 deletions(-) create mode 100644 lib/importmap/package.rb diff --git a/lib/importmap/commands.rb b/lib/importmap/commands.rb index 350cc42..11f6947 100644 --- a/lib/importmap/commands.rb +++ b/lib/importmap/commands.rb @@ -14,16 +14,9 @@ def self.exit_on_failure? option :from, type: :string, aliases: :f, default: "jspm" def pin(*packages) if imports = packager.import(*packages, env: options[:env], from: options[:from]) - imports.each do |package, url| - puts %(Pinning "#{package}" to #{packager.vendor_path}/#{package}.js via download from #{url}) - packager.download(package, url) - pin = packager.vendored_pin_for(package, url) - - if packager.packaged?(package) - gsub_file("config/importmap.rb", /^pin "#{package}".*$/, pin, verbose: false) - else - append_to_file("config/importmap.rb", "#{pin}\n", verbose: false) - end + imports.each do |package| + puts %(Pinning "#{package.package_name}" to #{packager.vendor_path}/#{package.vendored_package_folder} via download from #{package.base_url}) + package.download end else puts "Couldn't find any packages in #{packages.inspect} on #{options[:from]}" @@ -35,10 +28,10 @@ def pin(*packages) option :from, type: :string, aliases: :f, default: "jspm" def unpin(*packages) if imports = packager.import(*packages, env: options[:env], from: options[:from]) - imports.each do |package, url| - if packager.packaged?(package) - puts %(Unpinning and removing "#{package}") - packager.remove(package) + imports.each do |package| + if packager.packaged?(package.package_name) + puts %(Unpinning and removing "#{package.package_name}") + package.remove end end else diff --git a/lib/importmap/package.rb b/lib/importmap/package.rb new file mode 100644 index 0000000..970667f --- /dev/null +++ b/lib/importmap/package.rb @@ -0,0 +1,117 @@ +class Importmap::Package + attr_reader :base_url, :main_url, :package_name + + def initialize( + unfiltered_dependencies:, + package_name:, + main_url:, + packager: + ) + @unfiltered_dependencies = unfiltered_dependencies + @package_name = package_name + @main_url = main_url + @packager = packager + + @base_url = extract_base_url_from(main_url) + + dependencies = unfiltered_dependencies.select { _1.start_with?(base_url) } + @dependency_files = dependencies.map { _1[(base_url.size + 1)..] } # @main_file is included in this list + + @main_file = main_url[(base_url.size + 1)..] + end + + def download + @packager.ensure_vendor_directory_exists + remove_existing_package_files + + @dependency_files.each do |file| + download_file(file) + end + + @packager.pin_package_in_importmap(@package_name, vendored_pin) + end + + def remove + remove_existing_package_files + @packager.remove_package_from_importmap(@package_name) + end + + def vendored_pin + filename = vendored_package_path_for_file(@main_file) + version = extract_package_version_from(@main_url) + + %(pin "#{package_name}", to: "#{filename}" # #{version}) + end + + def vendored_package_folder + @packager.vendor_path.join(folder_name) + end + + private + def download_file(file) + response = Net::HTTP.get_response(URI("#{base_url}/#{file}")) + + if response.code == "200" + save_vendored_file(file, response.body) + else + handle_failure_response(response) + end + end + + def save_vendored_file(file, source) + url = "#{base_url}/#{file}" + file_name = vendored_package_path_for_file(file) + ensure_parent_directories_exist_for(file_name) + File.open(file_name, "w+") do |vendored_file| + vendored_file.write "// #{package_name}#{extract_package_version_from(url)}/#{file} downloaded from #{url}\n\n" + + vendored_file.write remove_sourcemap_comment_from(source).force_encoding("UTF-8") + end + end + + def ensure_parent_directories_exist_for(file) + dir_name = File.dirname(file) + + unless File.directory?(dir_name) + FileUtils.mkdir_p(dir_name) + end + end + + def remove_sourcemap_comment_from(source) + source.gsub(/^\/\/# sourceMappingURL=.*/, "") + end + + def vendored_package_path_for_file(file) + vendored_package_folder.join(file) + end + + def handle_failure_response(response) + if error_message = parse_service_error(response) + raise ServiceError, error_message + else + raise HTTPError, "Unexpected response code (#{response.code})" + end + end + + def parse_service_error(response) + JSON.parse(response.body.to_s)["error"] + rescue JSON::ParserError + nil + end + + def remove_existing_package_files + FileUtils.rm_rf vendored_package_folder + end + + def folder_name + @package_name.gsub("/", "--") + end + + def extract_base_url_from(url) + url.match(/^.+@\d+\.\d+\.\d+/)&.to_a&.first + end + + def extract_package_version_from(url) + url.match(/@\d+\.\d+\.\d+/)&.to_a&.first + end +end diff --git a/lib/importmap/packager.rb b/lib/importmap/packager.rb index 76f0661..9ec1fee 100644 --- a/lib/importmap/packager.rb +++ b/lib/importmap/packager.rb @@ -1,6 +1,7 @@ require "net/http" require "uri" require "json" +require "importmap/package" class Importmap::Packager Error = Class.new(StandardError) @@ -10,7 +11,7 @@ class Importmap::Packager singleton_class.attr_accessor :endpoint self.endpoint = URI("https://api.jspm.io/generate") - attr_reader :vendor_path + attr_reader :vendor_path, :importmap_path def initialize(importmap_path = "config/importmap.rb", vendor_path: "vendor/javascript") @importmap_path = Pathname.new(importmap_path) @@ -32,34 +33,29 @@ def import(*packages, env: "production", from: "jspm") end end - def pin_for(package, url) - %(pin "#{package}", to: "#{url}") + def packaged?(package_name) + importmap.match(/^pin ["']#{package_name}["'].*$/) end - def vendored_pin_for(package, url) - filename = package_filename(package) - version = extract_package_version_from(url) + def remove_package_from_importmap(package_name) + all_lines = File.readlines(@importmap_path) + with_lines_removed = all_lines.grep_v(/pin ["']#{package_name}["']/) - if "#{package}.js" == filename - %(pin "#{package}" # #{version}) - else - %(pin "#{package}", to: "#{filename}" # #{version}) + File.open(@importmap_path, "w") do |file| + with_lines_removed.each { |line| file.write(line) } end end - def packaged?(package) - importmap.match(/^pin ["']#{package}["'].*$/) - end - - def download(package, url) - ensure_vendor_directory_exists - remove_existing_package_file(package) - download_package_file(package, url) + def pin_package_in_importmap(package_name, pin) + if packaged?(package_name) + gsub_file(@importmap_path, /^pin "#{package_name}".*$/, pin, verbose: false) + else + File.write(@importmap_path, "#{pin}\n", mode: 'a+') + end end - def remove(package) - remove_existing_package_file(package) - remove_package_from_importmap(package) + def ensure_vendor_directory_exists + FileUtils.mkdir_p @vendor_path end private @@ -74,7 +70,22 @@ def normalize_provider(name) end def extract_parsed_imports(response) - JSON.parse(response.body).dig("map", "imports") + parsed_response = JSON.parse(response.body) + + imports = parsed_response.dig("map", "imports") + static_dependencies = parsed_response["staticDeps"] || [] + dynamic_dependencies = parsed_response["dynamicDeps"] || [] + + dependencies = static_dependencies + dynamic_dependencies + + imports.map do |package, url| + Importmap::Package.new( + unfiltered_dependencies: dependencies, + package_name: package, + main_url: url, + packager: self + ) + end end def handle_failure_response(response) @@ -94,56 +105,4 @@ def parse_service_error(response) def importmap @importmap ||= File.read(@importmap_path) end - - - def ensure_vendor_directory_exists - FileUtils.mkdir_p @vendor_path - end - - def remove_existing_package_file(package) - FileUtils.rm_rf vendored_package_path(package) - end - - def remove_package_from_importmap(package) - all_lines = File.readlines(@importmap_path) - with_lines_removed = all_lines.grep_v(/pin ["']#{package}["']/) - - File.open(@importmap_path, "w") do |file| - with_lines_removed.each { |line| file.write(line) } - end - end - - def download_package_file(package, url) - response = Net::HTTP.get_response(URI(url)) - - if response.code == "200" - save_vendored_package(package, url, response.body) - else - handle_failure_response(response) - end - end - - def save_vendored_package(package, url, source) - File.open(vendored_package_path(package), "w+") do |vendored_package| - vendored_package.write "// #{package}#{extract_package_version_from(url)} downloaded from #{url}\n\n" - - vendored_package.write remove_sourcemap_comment_from(source).force_encoding("UTF-8") - end - end - - def remove_sourcemap_comment_from(source) - source.gsub(/^\/\/# sourceMappingURL=.*/, "") - end - - def vendored_package_path(package) - @vendor_path.join(package_filename(package)) - end - - def package_filename(package) - package.gsub("/", "--") + ".js" - end - - def extract_package_version_from(url) - url.match(/@\d+\.\d+\.\d+/)&.to_a&.first - end end diff --git a/test/packager_integration_test.rb b/test/packager_integration_test.rb index 3600065..640e8f8 100644 --- a/test/packager_integration_test.rb +++ b/test/packager_integration_test.rb @@ -5,7 +5,13 @@ class Importmap::PackagerIntegrationTest < ActiveSupport::TestCase setup { @packager = Importmap::Packager.new(Rails.root.join("config/importmap.rb")) } test "successful import against live service" do - assert_equal "https://ga.jspm.io/npm:react@17.0.2/index.js", @packager.import("react@17.0.2")["react"] + results = @packager.import("react@17.0.2") + + react_result = results.find { _1.package_name == "react" } + object_assign_result = results.find { _1.package_name == "object-assign" } + + assert_equal("https://ga.jspm.io/npm:react@17.0.2/index.js", react_result.main_url) + assert_equal("https://ga.jspm.io/npm:object-assign@4.1.1/index.js", object_assign_result.main_url) end test "missing import against live service" do @@ -25,24 +31,84 @@ class Importmap::PackagerIntegrationTest < ActiveSupport::TestCase test "successful downloads from live service" do Dir.mktmpdir do |vendor_dir| - @packager = Importmap::Packager.new \ - Rails.root.join("config/importmap.rb"), - vendor_path: Pathname.new(vendor_dir) - - package_url = "https://ga.jspm.io/npm:@github/webauthn-json@0.5.7/dist/main/webauthn-json.js" - @packager.download("@github/webauthn-json", package_url) - vendored_package_file = Pathname.new(vendor_dir).join("@github--webauthn-json.js") - assert File.exist?(vendored_package_file) - assert_equal "// @github/webauthn-json@0.5.7 downloaded from #{package_url}", File.readlines(vendored_package_file).first.strip - - package_url = "https://ga.jspm.io/npm:react@17.0.2/index.js" - vendored_package_file = Pathname.new(vendor_dir).join("react.js") - @packager.download("react", package_url) - assert File.exist?(vendored_package_file) - assert_equal "// react@17.0.2 downloaded from #{package_url}", File.readlines(vendored_package_file).first.strip - - @packager.remove("react") - assert_not File.exist?(Pathname.new(vendor_dir).join("react.js")) + importmap_path = Pathname.new(vendor_dir).join("importmap.rb") + + File.new(importmap_path, "w").close + + @packager = Importmap::Packager.new( + importmap_path, + vendor_path: Pathname.new(vendor_dir), + ) + + packages = @packager.import("react@17.0.2") + packages.each(&:download) + + vendored_file = Pathname.new(vendor_dir).join("react/cjs/react.production.min.js") + assert_equal "// react@17.0.2/cjs/react.production.min.js downloaded from https://ga.jspm.io/npm:react@17.0.2/cjs/react.production.min.js", + File.readlines(vendored_file).first.strip + vendored_file = Pathname.new(vendor_dir).join("react/index.js") + assert_equal "// react@17.0.2/index.js downloaded from https://ga.jspm.io/npm:react@17.0.2/index.js", + File.readlines(vendored_file).first.strip + vendored_file = Pathname.new(vendor_dir).join("object-assign/index.js") + assert_equal "// object-assign@4.1.1/index.js downloaded from https://ga.jspm.io/npm:object-assign@4.1.1/index.js", + File.readlines(vendored_file).first.strip + + packages.each(&:remove) + + assert_not File.exist?(Pathname.new(vendor_dir).join("react/cjs/react.production.min.js")) + assert_not File.exist?(Pathname.new(vendor_dir).join("react/index.js")) + assert_not File.exist?(Pathname.new(vendor_dir).join("object-assign/index.js")) + + packages = @packager.import("@github/webauthn-json@0.5.7") + packages.each(&:download) + + vendored_file = Pathname.new(vendor_dir).join("@github--webauthn-json/dist/main/webauthn-json.js") + assert_equal "// @github/webauthn-json@0.5.7/dist/main/webauthn-json.js downloaded from https://ga.jspm.io/npm:@github/webauthn-json@0.5.7/dist/main/webauthn-json.js", + File.readlines(vendored_file).first.strip + + packages.each(&:remove) + + assert_not File.exist?(Pathname.new(vendor_dir).join("webauthn-json/dist/main/webauthn-json.js")) + + packages = @packager.import("tippy.js@6.3.7") + packages.each(&:download) + + assert File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/dom-utils/getWindow.js")) + assert File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/index.js")) + assert File.exist?(Pathname.new(vendor_dir).join("tippy.js/dist/tippy.esm.js")) + + packages.each(&:remove) + + assert_not File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/dom-utils/getWindow.js")) + assert_not File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/index.js")) + assert_not File.exist?(Pathname.new(vendor_dir).join("tippy.js/dist/tippy.esm.js")) + end + end + + test "successful importmap.rb updates from live service" do + Dir.mktmpdir do |vendor_dir| + importmap_path = Pathname.new(vendor_dir).join("importmap.rb") + + File.new(importmap_path, "w").close + + @packager = Importmap::Packager.new( + importmap_path, + vendor_path: Pathname.new(vendor_dir), + ) + + packages = @packager.import("react@17.0.2") + packages.each(&:download) + + importmap = <<~RB + pin "react", to: "#{vendor_dir}/react/index.js" # @17.0.2 + pin "object-assign", to: "#{vendor_dir}/object-assign/index.js" # @4.1.1 + RB + + assert_equal importmap, importmap_path.read + + packages.each(&:remove) + + assert_equal "", importmap_path.read end end end diff --git a/test/packager_single_quotes_test.rb b/test/packager_single_quotes_test.rb index 519d9e7..ce30b42 100644 --- a/test/packager_single_quotes_test.rb +++ b/test/packager_single_quotes_test.rb @@ -16,7 +16,7 @@ class Importmap::PackagerSingleQuotesTest < ActiveSupport::TestCase end test "remove package with single quotes" do - assert @packager.remove("md5") + assert @packager.remove_package_from_importmap("md5") assert_not @packager.packaged?("md5") end end diff --git a/test/packager_test.rb b/test/packager_test.rb index 29ce7f6..072d221 100644 --- a/test/packager_test.rb +++ b/test/packager_test.rb @@ -8,21 +8,44 @@ class Importmap::PackagerTest < ActiveSupport::TestCase test "successful import with mock" do response = Class.new do def body - { "map" => { "imports" => imports } }.to_json - end - - def imports { - "react" => "https://ga.jspm.io/npm:react@17.0.2/index.js", - "object-assign" => "https://ga.jspm.io/npm:object-assign@4.1.1/index.js" - } + "staticDeps" => [ + "https://ga.jspm.io/npm:react@17.0.2/index.js", + "https://ga.jspm.io/npm:object-assign@4.1.1/index.js" + ], + "dynamicDeps" => [], + "map" => { + "imports" => { + "react" => "https://ga.jspm.io/npm:react@17.0.2/index.js", + "object-assign" => "https://ga.jspm.io/npm:object-assign@4.1.1/index.js" + } + } + }.to_json end def code() "200" end end.new @packager.stub(:post_json, response) do - assert_equal(response.imports, @packager.import("react@17.0.2")) + expected_return = [ + { + package_name: "react", + main_url: "https://ga.jspm.io/npm:react@17.0.2/index.js", + dependency_urls: ["https://ga.jspm.io/npm:react@17.0.2/index.js"] + }, + { + package_name: "object-assign", + main_url: "https://ga.jspm.io/npm:object-assign@4.1.1/index.js", + dependency_urls: ["https://ga.jspm.io/npm:object-assign@4.1.1/index.js"] + } + ] + results = @packager.import("react@17.0.2") + + react_result = results.find { _1.package_name == "react" } + object_assign_result = results.find { _1.package_name == "object-assign" } + + assert_equal("https://ga.jspm.io/npm:react@17.0.2/index.js", react_result.main_url) + assert_equal("https://ga.jspm.io/npm:object-assign@4.1.1/index.js", object_assign_result.main_url) end end @@ -46,13 +69,4 @@ def code() "200" end assert @packager.packaged?("md5") assert_not @packager.packaged?("md5-extension") end - - test "pin_for" do - assert_equal %(pin "react", to: "https://cdn/react"), @packager.pin_for("react", "https://cdn/react") - end - - test "vendored_pin_for" do - assert_equal %(pin "react" # @17.0.2), @packager.vendored_pin_for("react", "https://cdn/react@17.0.2") - assert_equal %(pin "javascript/react", to: "javascript--react.js" # @17.0.2), @packager.vendored_pin_for("javascript/react", "https://cdn/react@17.0.2") - end end From 1d9e296c52be24b815461b94c36cda7b66af3e70 Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Wed, 24 Jan 2024 09:22:31 +0000 Subject: [PATCH 02/12] Vendor without specifying the vendor folder --- lib/importmap/package.rb | 14 +++++++------- test/packager_integration_test.rb | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/importmap/package.rb b/lib/importmap/package.rb index 970667f..f9a01bd 100644 --- a/lib/importmap/package.rb +++ b/lib/importmap/package.rb @@ -36,18 +36,18 @@ def remove @packager.remove_package_from_importmap(@package_name) end - def vendored_pin - filename = vendored_package_path_for_file(@main_file) - version = extract_package_version_from(@main_url) - - %(pin "#{package_name}", to: "#{filename}" # #{version}) - end - def vendored_package_folder @packager.vendor_path.join(folder_name) end private + def vendored_pin + filename = "#{package_name}/#{@main_file}" + version = extract_package_version_from(@main_url) + + %(pin "#{package_name}", to: "#{filename}" # #{version}) + end + def download_file(file) response = Net::HTTP.get_response(URI("#{base_url}/#{file}")) diff --git a/test/packager_integration_test.rb b/test/packager_integration_test.rb index 640e8f8..05de623 100644 --- a/test/packager_integration_test.rb +++ b/test/packager_integration_test.rb @@ -100,8 +100,8 @@ class Importmap::PackagerIntegrationTest < ActiveSupport::TestCase packages.each(&:download) importmap = <<~RB - pin "react", to: "#{vendor_dir}/react/index.js" # @17.0.2 - pin "object-assign", to: "#{vendor_dir}/object-assign/index.js" # @4.1.1 + pin "react", to: "react/index.js" # @17.0.2 + pin "object-assign", to: "object-assign/index.js" # @4.1.1 RB assert_equal importmap, importmap_path.read From 35a29623da68be89848bc9e7b8472ed3a5e6c1d9 Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Wed, 24 Jan 2024 09:41:48 +0000 Subject: [PATCH 03/12] Added in error classes --- lib/importmap/package.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/importmap/package.rb b/lib/importmap/package.rb index f9a01bd..7240b63 100644 --- a/lib/importmap/package.rb +++ b/lib/importmap/package.rb @@ -1,4 +1,8 @@ class Importmap::Package + Error = Class.new(StandardError) + HTTPError = Class.new(Error) + ServiceError = Error.new(Error) + attr_reader :base_url, :main_url, :package_name def initialize( From 6c55a6147bc640db9c67e6f8117db6a001187e76 Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Wed, 24 Jan 2024 17:38:44 +0000 Subject: [PATCH 04/12] Update pinning code to correctly reference folder --- lib/importmap/package.rb | 2 +- test/packager_integration_test.rb | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/importmap/package.rb b/lib/importmap/package.rb index 7240b63..45674a5 100644 --- a/lib/importmap/package.rb +++ b/lib/importmap/package.rb @@ -46,7 +46,7 @@ def vendored_package_folder private def vendored_pin - filename = "#{package_name}/#{@main_file}" + filename = "#{folder_name}/#{@main_file}" version = extract_package_version_from(@main_url) %(pin "#{package_name}", to: "#{filename}" # #{version}) diff --git a/test/packager_integration_test.rb b/test/packager_integration_test.rb index 05de623..c35c4e3 100644 --- a/test/packager_integration_test.rb +++ b/test/packager_integration_test.rb @@ -109,6 +109,20 @@ class Importmap::PackagerIntegrationTest < ActiveSupport::TestCase packages.each(&:remove) assert_equal "", importmap_path.read + + packages = @packager.import("tippy.js@6.3.7") + packages.each(&:download) + + importmap = <<~RB + pin "tippy.js", to: "tippy.js/dist/tippy.esm.js" # @6.3.7 + pin "@popperjs/core", to: "@popperjs--core/lib/index.js" # @2.11.8 + RB + + assert_equal importmap, importmap_path.read + + packages.each(&:remove) + + assert_equal "", importmap_path.read end end end From 84673d8d13e0a9eeaeb3ab57a76516f3e05e6c9e Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Fri, 26 Jan 2024 17:03:53 +0000 Subject: [PATCH 05/12] Move over to using the download API --- Gemfile.lock | 8 +-- lib/importmap/jspmApi.rb | 84 +++++++++++++++++++++++++++++++ lib/importmap/package.rb | 41 ++++++--------- lib/importmap/packager.rb | 70 ++++++-------------------- test/packager_integration_test.rb | 44 ++++++++-------- test/packager_test.rb | 18 ++----- 6 files changed, 144 insertions(+), 121 deletions(-) create mode 100644 lib/importmap/jspmApi.rb diff --git a/Gemfile.lock b/Gemfile.lock index cf2b0bb..4056010 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -104,13 +104,13 @@ GEM mini_mime (1.1.2) minitest (5.16.2) nio4r (2.5.8) - nokogiri (1.14.0-aarch64-linux) + nokogiri (1.16.0-aarch64-linux) racc (~> 1.4) - nokogiri (1.14.0-arm64-darwin) + nokogiri (1.16.0-arm64-darwin) racc (~> 1.4) - nokogiri (1.14.0-x86_64-darwin) + nokogiri (1.16.0-x86_64-darwin) racc (~> 1.4) - nokogiri (1.14.0-x86_64-linux) + nokogiri (1.16.0-x86_64-linux) racc (~> 1.4) public_suffix (4.0.7) racc (1.6.2) diff --git a/lib/importmap/jspmApi.rb b/lib/importmap/jspmApi.rb new file mode 100644 index 0000000..9eda427 --- /dev/null +++ b/lib/importmap/jspmApi.rb @@ -0,0 +1,84 @@ +class Importmap::JspmApi + Error = Class.new(StandardError) + HTTPError = Class.new(Error) + ServiceError = Error.new(Error) + + singleton_class.attr_accessor :generate_endpoint, :download_endpoint + self.generate_endpoint = "https://api.jspm.io/generate" + self.download_endpoint = "https://api.jspm.io/download" + + def generate(install:, flatten_scope:, env:, provider:) + response = post_json(self.class.generate_endpoint, { + install:, + flattenScope: flatten_scope, + env:, + provider: normalize_provider(provider) + }) + + response_json(response) + end + + def download(versioned_package_name:, provider:) + response = post_json("#{self.class.download_endpoint}/#{versioned_package_name}", { + provider: normalize_provider(provider) + }) + + json = response_json(response) + + files = json.dig(versioned_package_name, "files") + package_url = json.dig(versioned_package_name, "pkgUrl") + + output_files = {} + + files.each do |file| + output_files[file] = fetch_file(package_url, file) + end + + output_files + end + + private + def fetch_file(url, file) + response = Net::HTTP.get_response(URI("#{url}#{file}")) + + if response.code == "200" + response.body + else + handle_failure_response(response) + end + rescue => error + raise HTTPError, "Unexpected transport error (#{error.class}: #{error.message})" + end + + def response_json(response) + case response.code + when "200" then JSON.parse(response.body) + when "404", "401" then {} + else handle_failure_response(response) + end + end + + def normalize_provider(name) + name.to_s == "jspm" ? "jspm.io" : name.to_s + end + + def post_json(endpoint, body) + Net::HTTP.post(URI(endpoint), body.to_json, "Content-Type" => "application/json") + rescue => error + raise HTTPError, "Unexpected transport error (#{error.class}: #{error.message})" + end + + def handle_failure_response(response) + if error_message = parse_service_error(response) + raise ServiceError, error_message + else + raise HTTPError, "Unexpected response code (#{response.code})" + end + end + + def parse_service_error(response) + JSON.parse(response.body.to_s)["error"] + rescue JSON::ParserError + nil + end +end diff --git a/lib/importmap/package.rb b/lib/importmap/package.rb index 45674a5..03159b1 100644 --- a/lib/importmap/package.rb +++ b/lib/importmap/package.rb @@ -1,25 +1,21 @@ -class Importmap::Package - Error = Class.new(StandardError) - HTTPError = Class.new(Error) - ServiceError = Error.new(Error) +require "importmap/jspmApi" +class Importmap::Package attr_reader :base_url, :main_url, :package_name def initialize( - unfiltered_dependencies:, package_name:, main_url:, - packager: + packager:, + provider: ) - @unfiltered_dependencies = unfiltered_dependencies @package_name = package_name @main_url = main_url @packager = packager + @provider = provider @base_url = extract_base_url_from(main_url) - - dependencies = unfiltered_dependencies.select { _1.start_with?(base_url) } - @dependency_files = dependencies.map { _1[(base_url.size + 1)..] } # @main_file is included in this list + @version = extract_package_version_from(@main_url) @main_file = main_url[(base_url.size + 1)..] end @@ -28,8 +24,12 @@ def download @packager.ensure_vendor_directory_exists remove_existing_package_files - @dependency_files.each do |file| - download_file(file) + jspm_api = Importmap::JspmApi.new + + files = jspm_api.download(versioned_package_name: "#{@package_name}#{@version}", provider: @provider) + + files.each do |file, downloaded_file| + save_vendored_file(file, downloaded_file) end @packager.pin_package_in_importmap(@package_name, vendored_pin) @@ -47,27 +47,16 @@ def vendored_package_folder private def vendored_pin filename = "#{folder_name}/#{@main_file}" - version = extract_package_version_from(@main_url) - %(pin "#{package_name}", to: "#{filename}" # #{version}) - end - - def download_file(file) - response = Net::HTTP.get_response(URI("#{base_url}/#{file}")) - - if response.code == "200" - save_vendored_file(file, response.body) - else - handle_failure_response(response) - end + %(pin "#{package_name}", to: "#{filename}" # #{@version}) end def save_vendored_file(file, source) - url = "#{base_url}/#{file}" file_name = vendored_package_path_for_file(file) ensure_parent_directories_exist_for(file_name) + File.open(file_name, "w+") do |vendored_file| - vendored_file.write "// #{package_name}#{extract_package_version_from(url)}/#{file} downloaded from #{url}\n\n" + vendored_file.write "// #{@package_name}#{@version}/#{file} downloaded from #{base_url}/#{file}\n\n" vendored_file.write remove_sourcemap_comment_from(source).force_encoding("UTF-8") end diff --git a/lib/importmap/packager.rb b/lib/importmap/packager.rb index 9ec1fee..404422f 100644 --- a/lib/importmap/packager.rb +++ b/lib/importmap/packager.rb @@ -2,15 +2,9 @@ require "uri" require "json" require "importmap/package" +require "importmap/jspmApi" class Importmap::Packager - Error = Class.new(StandardError) - HTTPError = Class.new(Error) - ServiceError = Error.new(Error) - - singleton_class.attr_accessor :endpoint - self.endpoint = URI("https://api.jspm.io/generate") - attr_reader :vendor_path, :importmap_path def initialize(importmap_path = "config/importmap.rb", vendor_path: "vendor/javascript") @@ -19,18 +13,16 @@ def initialize(importmap_path = "config/importmap.rb", vendor_path: "vendor/java end def import(*packages, env: "production", from: "jspm") - response = post_json({ - "install" => Array(packages), - "flattenScope" => true, - "env" => [ "browser", "module", env ], - "provider" => normalize_provider(from) - }) + jspm_api = Importmap::JspmApi.new - case response.code - when "200" then extract_parsed_imports(response) - when "404", "401" then nil - else handle_failure_response(response) - end + response = jspm_api.generate( + install: Array(packages), + flatten_scope: true, + env: [ "browser", "module", env ], + provider: from + ) + + extract_parsed_imports(response, from) end def packaged?(package_name) @@ -50,7 +42,7 @@ def pin_package_in_importmap(package_name, pin) if packaged?(package_name) gsub_file(@importmap_path, /^pin "#{package_name}".*$/, pin, verbose: false) else - File.write(@importmap_path, "#{pin}\n", mode: 'a+') + File.write(@importmap_path, "#{pin}\n", mode: "a+") end end @@ -59,49 +51,19 @@ def ensure_vendor_directory_exists end private - def post_json(body) - Net::HTTP.post(self.class.endpoint, body.to_json, "Content-Type" => "application/json") - rescue => error - raise HTTPError, "Unexpected transport error (#{error.class}: #{error.message})" - end - - def normalize_provider(name) - name.to_s == "jspm" ? "jspm.io" : name.to_s - end - - def extract_parsed_imports(response) - parsed_response = JSON.parse(response.body) + def extract_parsed_imports(response, provider) + imports = response.dig("map", "imports") - imports = parsed_response.dig("map", "imports") - static_dependencies = parsed_response["staticDeps"] || [] - dynamic_dependencies = parsed_response["dynamicDeps"] || [] - - dependencies = static_dependencies + dynamic_dependencies - - imports.map do |package, url| + imports&.map do |package, url| Importmap::Package.new( - unfiltered_dependencies: dependencies, package_name: package, main_url: url, - packager: self + packager: self, + provider: ) end end - def handle_failure_response(response) - if error_message = parse_service_error(response) - raise ServiceError, error_message - else - raise HTTPError, "Unexpected response code (#{response.code})" - end - end - - def parse_service_error(response) - JSON.parse(response.body.to_s)["error"] - rescue JSON::ParserError - nil - end - def importmap @importmap ||= File.read(@importmap_path) end diff --git a/test/packager_integration_test.rb b/test/packager_integration_test.rb index c35c4e3..c27fb53 100644 --- a/test/packager_integration_test.rb +++ b/test/packager_integration_test.rb @@ -19,14 +19,14 @@ class Importmap::PackagerIntegrationTest < ActiveSupport::TestCase end test "failed request against live bad domain" do - original_endpoint = Importmap::Packager.endpoint - Importmap::Packager.endpoint = URI("https://invalid./error") + original_endpoint = Importmap::JspmApi.generate_endpoint + Importmap::JspmApi.generate_endpoint = URI("https://invalid./error") - assert_raises(Importmap::Packager::HTTPError) do + assert_raises(Importmap::JspmApi::HTTPError) do @packager.import("missing-package-that-doesnt-exist@17.0.2") end ensure - Importmap::Packager.endpoint = original_endpoint + Importmap::JspmApi.generate_endpoint = original_endpoint end test "successful downloads from live service" do @@ -70,18 +70,18 @@ class Importmap::PackagerIntegrationTest < ActiveSupport::TestCase assert_not File.exist?(Pathname.new(vendor_dir).join("webauthn-json/dist/main/webauthn-json.js")) - packages = @packager.import("tippy.js@6.3.7") - packages.each(&:download) + #packages = @packager.import("tippy.js@6.3.7") + #packages.each(&:download) - assert File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/dom-utils/getWindow.js")) - assert File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/index.js")) - assert File.exist?(Pathname.new(vendor_dir).join("tippy.js/dist/tippy.esm.js")) + #assert File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/dom-utils/getWindow.js")) + #assert File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/index.js")) + #assert File.exist?(Pathname.new(vendor_dir).join("tippy.js/dist/tippy.esm.js")) - packages.each(&:remove) + #packages.each(&:remove) - assert_not File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/dom-utils/getWindow.js")) - assert_not File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/index.js")) - assert_not File.exist?(Pathname.new(vendor_dir).join("tippy.js/dist/tippy.esm.js")) + #assert_not File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/dom-utils/getWindow.js")) + #assert_not File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/index.js")) + #assert_not File.exist?(Pathname.new(vendor_dir).join("tippy.js/dist/tippy.esm.js")) end end @@ -110,19 +110,19 @@ class Importmap::PackagerIntegrationTest < ActiveSupport::TestCase assert_equal "", importmap_path.read - packages = @packager.import("tippy.js@6.3.7") - packages.each(&:download) + #packages = @packager.import("tippy.js@6.3.7") + #packages.each(&:download) - importmap = <<~RB - pin "tippy.js", to: "tippy.js/dist/tippy.esm.js" # @6.3.7 - pin "@popperjs/core", to: "@popperjs--core/lib/index.js" # @2.11.8 - RB + #importmap = <<~RB + # pin "tippy.js", to: "tippy.js/dist/tippy.esm.js" # @6.3.7 + # pin "@popperjs/core", to: "@popperjs--core/lib/index.js" # @2.11.8 + #RB - assert_equal importmap, importmap_path.read + #assert_equal importmap, importmap_path.read - packages.each(&:remove) + #packages.each(&:remove) - assert_equal "", importmap_path.read + #assert_equal "", importmap_path.read end end end diff --git a/test/packager_test.rb b/test/packager_test.rb index 072d221..dbfda5e 100644 --- a/test/packager_test.rb +++ b/test/packager_test.rb @@ -26,19 +26,7 @@ def body def code() "200" end end.new - @packager.stub(:post_json, response) do - expected_return = [ - { - package_name: "react", - main_url: "https://ga.jspm.io/npm:react@17.0.2/index.js", - dependency_urls: ["https://ga.jspm.io/npm:react@17.0.2/index.js"] - }, - { - package_name: "object-assign", - main_url: "https://ga.jspm.io/npm:object-assign@4.1.1/index.js", - dependency_urls: ["https://ga.jspm.io/npm:object-assign@4.1.1/index.js"] - } - ] + Net::HTTP.stub(:post, response) do results = @packager.import("react@17.0.2") react_result = results.find { _1.package_name == "react" } @@ -52,14 +40,14 @@ def code() "200" end test "missing import with mock" do response = Class.new { def code() "404" end }.new - @packager.stub(:post_json, response) do + Net::HTTP.stub(:post, response) do assert_nil @packager.import("missing-package-that-doesnt-exist@17.0.2") end end test "failed request with mock" do Net::HTTP.stub(:post, proc { raise "Unexpected Error" }) do - assert_raises(Importmap::Packager::HTTPError) do + assert_raises(Importmap::JspmApi::HTTPError) do @packager.import("missing-package-that-doesnt-exist@17.0.2") end end From 96143397d48ccf30ffc749576eacc790227b154c Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Fri, 26 Jan 2024 17:04:28 +0000 Subject: [PATCH 06/12] Remove a slow test case This is mostly covered by other cases so I'm happy removing it. Ideally we'd use one HTTP2 session to speed up the downloads --- test/packager_integration_test.rb | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/test/packager_integration_test.rb b/test/packager_integration_test.rb index c27fb53..2273af0 100644 --- a/test/packager_integration_test.rb +++ b/test/packager_integration_test.rb @@ -69,19 +69,6 @@ class Importmap::PackagerIntegrationTest < ActiveSupport::TestCase packages.each(&:remove) assert_not File.exist?(Pathname.new(vendor_dir).join("webauthn-json/dist/main/webauthn-json.js")) - - #packages = @packager.import("tippy.js@6.3.7") - #packages.each(&:download) - - #assert File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/dom-utils/getWindow.js")) - #assert File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/index.js")) - #assert File.exist?(Pathname.new(vendor_dir).join("tippy.js/dist/tippy.esm.js")) - - #packages.each(&:remove) - - #assert_not File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/dom-utils/getWindow.js")) - #assert_not File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/index.js")) - #assert_not File.exist?(Pathname.new(vendor_dir).join("tippy.js/dist/tippy.esm.js")) end end @@ -109,20 +96,6 @@ class Importmap::PackagerIntegrationTest < ActiveSupport::TestCase packages.each(&:remove) assert_equal "", importmap_path.read - - #packages = @packager.import("tippy.js@6.3.7") - #packages.each(&:download) - - #importmap = <<~RB - # pin "tippy.js", to: "tippy.js/dist/tippy.esm.js" # @6.3.7 - # pin "@popperjs/core", to: "@popperjs--core/lib/index.js" # @2.11.8 - #RB - - #assert_equal importmap, importmap_path.read - - #packages.each(&:remove) - - #assert_equal "", importmap_path.read end end end From b7457cab4f1f01b4768d513608b3746b99855a19 Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Fri, 26 Jan 2024 18:51:10 +0000 Subject: [PATCH 07/12] =?UTF-8?q?Rename=20jspmApi=20=E2=86=92=20jspm=5Fapi?= =?UTF-8?q?,=20using=20Net::HTTP.start=20over=20individual=20calls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/importmap/{jspmApi.rb => jspm_api.rb} | 14 +++++--- lib/importmap/package.rb | 2 +- lib/importmap/packager.rb | 8 ++--- test/jspm_api_integration_test.rb | 43 +++++++++++++++++++++++ test/packager_integration_test.rb | 13 +++++++ 5 files changed, 70 insertions(+), 10 deletions(-) rename lib/importmap/{jspmApi.rb => jspm_api.rb} (85%) create mode 100644 test/jspm_api_integration_test.rb diff --git a/lib/importmap/jspmApi.rb b/lib/importmap/jspm_api.rb similarity index 85% rename from lib/importmap/jspmApi.rb rename to lib/importmap/jspm_api.rb index 9eda427..f60f301 100644 --- a/lib/importmap/jspmApi.rb +++ b/lib/importmap/jspm_api.rb @@ -25,21 +25,25 @@ def download(versioned_package_name:, provider:) json = response_json(response) + return {} if json.blank? + files = json.dig(versioned_package_name, "files") - package_url = json.dig(versioned_package_name, "pkgUrl") + package_uri = URI(json.dig(versioned_package_name, "pkgUrl")) output_files = {} - files.each do |file| - output_files[file] = fetch_file(package_url, file) + Net::HTTP.start(package_uri.hostname, { use_ssl: true }) do |http| + files.each do |file| + output_files[file] = fetch_file(http, "#{package_uri.path}/#{file}") + end end output_files end private - def fetch_file(url, file) - response = Net::HTTP.get_response(URI("#{url}#{file}")) + def fetch_file(http, path) + response = http.get(path) if response.code == "200" response.body diff --git a/lib/importmap/package.rb b/lib/importmap/package.rb index 03159b1..0203dec 100644 --- a/lib/importmap/package.rb +++ b/lib/importmap/package.rb @@ -1,4 +1,4 @@ -require "importmap/jspmApi" +require "importmap/jspm_api" class Importmap::Package attr_reader :base_url, :main_url, :package_name diff --git a/lib/importmap/packager.rb b/lib/importmap/packager.rb index 404422f..ced46ce 100644 --- a/lib/importmap/packager.rb +++ b/lib/importmap/packager.rb @@ -2,7 +2,7 @@ require "uri" require "json" require "importmap/package" -require "importmap/jspmApi" +require "importmap/jspm_api" class Importmap::Packager attr_reader :vendor_path, :importmap_path @@ -16,10 +16,10 @@ def import(*packages, env: "production", from: "jspm") jspm_api = Importmap::JspmApi.new response = jspm_api.generate( - install: Array(packages), + install: Array(packages), flatten_scope: true, - env: [ "browser", "module", env ], - provider: from + env: [ "browser", "module", env ], + provider: from ) extract_parsed_imports(response, from) diff --git a/test/jspm_api_integration_test.rb b/test/jspm_api_integration_test.rb new file mode 100644 index 0000000..cbf8364 --- /dev/null +++ b/test/jspm_api_integration_test.rb @@ -0,0 +1,43 @@ +require "test_helper" +require "importmap/jspm_api" + +class Importmap::JspmApiIntegrationTest < ActiveSupport::TestCase + setup do + @jspm_api = Importmap::JspmApi.new + end + + test "#download when given a valid input" do + result = @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm.io") + + assert result.keys.include?("lib/dom-utils/getWindow.js") + assert result.keys.include?("lib/index.js") + + result = @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm") + + assert result.keys.include?("lib/dom-utils/getWindow.js") + assert result.keys.include?("lib/index.js") + end + + test "#download when given a bad package" do + result = @jspm_api.download(versioned_package_name: "@popperjs/corenoversion", provider: "jspm.io") + + assert_equal result, {} + end + + test "#download when given a bad provider" do + result = @jspm_api.download(versioned_package_name: "@popperjs/corenoversion", provider: "jspmfoobarbaz") + + assert_equal result, {} + end + + test "#download when endpoint is incorrect" do + original_endpoint = Importmap::JspmApi.download_endpoint + Importmap::JspmApi.download_endpoint = URI("https://invalid./error") + + assert_raises(Importmap::JspmApi::HTTPError) do + @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm.io") + end + ensure + Importmap::JspmApi.download_endpoint = original_endpoint + end +end diff --git a/test/packager_integration_test.rb b/test/packager_integration_test.rb index 2273af0..fdf008c 100644 --- a/test/packager_integration_test.rb +++ b/test/packager_integration_test.rb @@ -69,6 +69,19 @@ class Importmap::PackagerIntegrationTest < ActiveSupport::TestCase packages.each(&:remove) assert_not File.exist?(Pathname.new(vendor_dir).join("webauthn-json/dist/main/webauthn-json.js")) + + packages = @packager.import("tippy.js@6.3.7") + packages.each(&:download) + + assert File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/dom-utils/getWindow.js")) + assert File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/index.js")) + assert File.exist?(Pathname.new(vendor_dir).join("tippy.js/dist/tippy.esm.js")) + + packages.each(&:remove) + + assert_not File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/dom-utils/getWindow.js")) + assert_not File.exist?(Pathname.new(vendor_dir).join("@popperjs--core/lib/index.js")) + assert_not File.exist?(Pathname.new(vendor_dir).join("tippy.js/dist/tippy.esm.js")) end end From 0647eaeac5ae8a99f2f9667fa04f3e0307fc865b Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Fri, 26 Jan 2024 21:29:43 +0000 Subject: [PATCH 08/12] Added more tests for jspm_api --- test/jspm_api_integration_test.rb | 50 +++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/test/jspm_api_integration_test.rb b/test/jspm_api_integration_test.rb index cbf8364..16bcafc 100644 --- a/test/jspm_api_integration_test.rb +++ b/test/jspm_api_integration_test.rb @@ -40,4 +40,54 @@ class Importmap::JspmApiIntegrationTest < ActiveSupport::TestCase ensure Importmap::JspmApi.download_endpoint = original_endpoint end + + test "#generate when given valid input" do + response = @jspm_api.generate( + install: "tippy.js", + flatten_scope: true, + env: nil, + provider: "jspm.io" + ) + + expected_response = { + "staticDeps" => [ + "https://ga.jspm.io/npm:@popperjs/core@2.11.8/dist/cjs/popper.js", + "https://ga.jspm.io/npm:tippy.js@6.3.7/dist/tippy.cjs.js" + ], + "dynamicDeps" => [], + "map" => { "imports" => { + "tippy.js" => "https://ga.jspm.io/npm:tippy.js@6.3.7/dist/tippy.cjs.js", + "@popperjs/core" => "https://ga.jspm.io/npm:@popperjs/core@2.11.8/dist/cjs/popper.js" + }} + } + + assert_equal response, expected_response + end + + test "#generate when given non existent package" do + response = @jspm_api.generate( + install: "tippy.jsbutnotreallyalibrary", + flatten_scope: true, + env: nil, + provider: "jspm.io" + ) + + assert_equal response, {} + end + + test "#generate when endpoint is incorrect" do + original_endpoint = Importmap::JspmApi.generate_endpoint + Importmap::JspmApi.generate_endpoint = URI("https://invalid./error") + + assert_raises(Importmap::JspmApi::HTTPError) do + @jspm_api.generate( + install: "tippy.jsbutnotreallyalibrary", + flatten_scope: true, + env: nil, + provider: "jspm.io" + ) + end + ensure + Importmap::JspmApi.generate_endpoint = original_endpoint + end end From 1ebbcd3713b4245565f21574ce342bb529f2d0b9 Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Fri, 26 Jan 2024 21:39:44 +0000 Subject: [PATCH 09/12] Change testing to ESM build --- test/jspm_api_integration_test.rb | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/test/jspm_api_integration_test.rb b/test/jspm_api_integration_test.rb index 16bcafc..2e45ed4 100644 --- a/test/jspm_api_integration_test.rb +++ b/test/jspm_api_integration_test.rb @@ -45,23 +45,16 @@ class Importmap::JspmApiIntegrationTest < ActiveSupport::TestCase response = @jspm_api.generate( install: "tippy.js", flatten_scope: true, - env: nil, + env: [ "browser", "module", nil ], provider: "jspm.io" ) - expected_response = { - "staticDeps" => [ - "https://ga.jspm.io/npm:@popperjs/core@2.11.8/dist/cjs/popper.js", - "https://ga.jspm.io/npm:tippy.js@6.3.7/dist/tippy.cjs.js" - ], - "dynamicDeps" => [], - "map" => { "imports" => { - "tippy.js" => "https://ga.jspm.io/npm:tippy.js@6.3.7/dist/tippy.cjs.js", - "@popperjs/core" => "https://ga.jspm.io/npm:@popperjs/core@2.11.8/dist/cjs/popper.js" - }} + expected_imports = { + "tippy.js" => "https://ga.jspm.io/npm:tippy.js@6.3.7/dist/tippy.esm.js", + "@popperjs/core" => "https://ga.jspm.io/npm:@popperjs/core@2.11.8/lib/index.js" } - assert_equal response, expected_response + assert_equal expected_imports, response.dig("map", "imports") end test "#generate when given non existent package" do From 0c63b8bf810c891d1af1297727924992948cab33 Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Sat, 27 Jan 2024 17:14:57 +0000 Subject: [PATCH 10/12] Fix calling Thor methods in non Thor context --- lib/importmap/packager.rb | 9 ++++---- test/packager_test.rb | 47 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/lib/importmap/packager.rb b/lib/importmap/packager.rb index ced46ce..fc4c402 100644 --- a/lib/importmap/packager.rb +++ b/lib/importmap/packager.rb @@ -26,6 +26,7 @@ def import(*packages, env: "production", from: "jspm") end def packaged?(package_name) + importmap = File.read(@importmap_path) importmap.match(/^pin ["']#{package_name}["'].*$/) end @@ -40,7 +41,9 @@ def remove_package_from_importmap(package_name) def pin_package_in_importmap(package_name, pin) if packaged?(package_name) - gsub_file(@importmap_path, /^pin "#{package_name}".*$/, pin, verbose: false) + importmap = File.read(@importmap_path) + modified_importmap = importmap.gsub(/^pin "#{package_name}".*$/, pin) + File.open(@importmap_path, "w") { _1.puts modified_importmap } else File.write(@importmap_path, "#{pin}\n", mode: "a+") end @@ -63,8 +66,4 @@ def extract_parsed_imports(response, provider) ) end end - - def importmap - @importmap ||= File.read(@importmap_path) - end end diff --git a/test/packager_test.rb b/test/packager_test.rb index dbfda5e..ac29946 100644 --- a/test/packager_test.rb +++ b/test/packager_test.rb @@ -57,4 +57,51 @@ def code() "200" end assert @packager.packaged?("md5") assert_not @packager.packaged?("md5-extension") end + + test "pin_package_in_importmap" do + Dir.mktmpdir do |vendor_dir| + importmap_path = Pathname.new(vendor_dir).join("importmap.rb") + + File.new(importmap_path, "w").close + + packager = Importmap::Packager.new( + importmap_path, + vendor_path: Pathname.new(vendor_dir), + ) + + assert_not packager.packaged?("md5") + + packager.pin_package_in_importmap("md5", %(pin "md5", to: "md5/md5.js" # @2.3.0)) + + assert_equal %(pin "md5", to: "md5/md5.js" # @2.3.0), importmap_path.readlines(chomp: true).first + + packager.pin_package_in_importmap("md5", %(pin "md5", to: "md5/md5.js" # @2.3.5)) + + assert_equal %(pin "md5", to: "md5/md5.js" # @2.3.5), importmap_path.readlines(chomp: true).first + end + end + + + test "remove_package_from_importmap" do + Dir.mktmpdir do |vendor_dir| + importmap_path = Pathname.new(vendor_dir).join("importmap.rb") + + File.new(importmap_path, "w").close + + packager = Importmap::Packager.new( + importmap_path, + vendor_path: Pathname.new(vendor_dir), + ) + + assert_not packager.packaged?("md5") + + packager.pin_package_in_importmap("md5", %(pin "md5", to: "md5/md5.js" # @2.3.0)) + + assert_equal %(pin "md5", to: "md5/md5.js" # @2.3.0), importmap_path.readlines(chomp: true).first + + packager.remove_package_from_importmap("md5") + + assert_nil importmap_path.readlines(chomp: true).first + end + end end From 62f7f93ec63ea5ed57b4b7de5bb444db75702162 Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Sun, 28 Jan 2024 11:59:06 +0000 Subject: [PATCH 11/12] Update usage of download API to exclude unneeded files --- lib/importmap/jspm_api.rb | 5 +++-- lib/importmap/package.rb | 6 +++++- test/jspm_api_integration_test.rb | 10 +++++----- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/importmap/jspm_api.rb b/lib/importmap/jspm_api.rb index f60f301..c0d40e4 100644 --- a/lib/importmap/jspm_api.rb +++ b/lib/importmap/jspm_api.rb @@ -18,9 +18,10 @@ def generate(install:, flatten_scope:, env:, provider:) response_json(response) end - def download(versioned_package_name:, provider:) + def download(versioned_package_name:, provider:, exclude:) response = post_json("#{self.class.download_endpoint}/#{versioned_package_name}", { - provider: normalize_provider(provider) + provider: normalize_provider(provider), + exclude: }) json = response_json(response) diff --git a/lib/importmap/package.rb b/lib/importmap/package.rb index 0203dec..93b209b 100644 --- a/lib/importmap/package.rb +++ b/lib/importmap/package.rb @@ -26,7 +26,11 @@ def download jspm_api = Importmap::JspmApi.new - files = jspm_api.download(versioned_package_name: "#{@package_name}#{@version}", provider: @provider) + files = jspm_api.download( + versioned_package_name: "#{@package_name}#{@version}", + provider: @provider, + exclude: ["unused", "readme", "types"] + ) files.each do |file, downloaded_file| save_vendored_file(file, downloaded_file) diff --git a/test/jspm_api_integration_test.rb b/test/jspm_api_integration_test.rb index 2e45ed4..c28363e 100644 --- a/test/jspm_api_integration_test.rb +++ b/test/jspm_api_integration_test.rb @@ -7,25 +7,25 @@ class Importmap::JspmApiIntegrationTest < ActiveSupport::TestCase end test "#download when given a valid input" do - result = @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm.io") + result = @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm.io", exclude: []) assert result.keys.include?("lib/dom-utils/getWindow.js") assert result.keys.include?("lib/index.js") - result = @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm") + result = @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm", exclude: []) assert result.keys.include?("lib/dom-utils/getWindow.js") assert result.keys.include?("lib/index.js") end test "#download when given a bad package" do - result = @jspm_api.download(versioned_package_name: "@popperjs/corenoversion", provider: "jspm.io") + result = @jspm_api.download(versioned_package_name: "@popperjs/corenoversion", provider: "jspm.io", exclude: []) assert_equal result, {} end test "#download when given a bad provider" do - result = @jspm_api.download(versioned_package_name: "@popperjs/corenoversion", provider: "jspmfoobarbaz") + result = @jspm_api.download(versioned_package_name: "@popperjs/corenoversion", provider: "jspmfoobarbaz", exclude: []) assert_equal result, {} end @@ -35,7 +35,7 @@ class Importmap::JspmApiIntegrationTest < ActiveSupport::TestCase Importmap::JspmApi.download_endpoint = URI("https://invalid./error") assert_raises(Importmap::JspmApi::HTTPError) do - @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm.io") + @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm.io", exclude: []) end ensure Importmap::JspmApi.download_endpoint = original_endpoint From ebddf48364c46498df3109de4ab5a0b1e0cff660 Mon Sep 17 00:00:00 2001 From: Caleb Owens Date: Sun, 28 Jan 2024 22:23:14 +0000 Subject: [PATCH 12/12] Move over to passing packages as array in POST body --- lib/importmap/jspm_api.rb | 26 ++++++++++++++------------ lib/importmap/package.rb | 6 ++++-- test/jspm_api_integration_test.rb | 18 +++++++++--------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/lib/importmap/jspm_api.rb b/lib/importmap/jspm_api.rb index c0d40e4..041114f 100644 --- a/lib/importmap/jspm_api.rb +++ b/lib/importmap/jspm_api.rb @@ -18,8 +18,9 @@ def generate(install:, flatten_scope:, env:, provider:) response_json(response) end - def download(versioned_package_name:, provider:, exclude:) - response = post_json("#{self.class.download_endpoint}/#{versioned_package_name}", { + def download(versioned_package_names:, provider:, exclude:) + response = post_json("#{self.class.download_endpoint}", { + packages: versioned_package_names, provider: normalize_provider(provider), exclude: }) @@ -28,18 +29,19 @@ def download(versioned_package_name:, provider:, exclude:) return {} if json.blank? - files = json.dig(versioned_package_name, "files") - package_uri = URI(json.dig(versioned_package_name, "pkgUrl")) - - output_files = {} - - Net::HTTP.start(package_uri.hostname, { use_ssl: true }) do |http| - files.each do |file| - output_files[file] = fetch_file(http, "#{package_uri.path}/#{file}") + json.transform_values do |package_download_details| + files = package_download_details["files"] + package_uri = URI(package_download_details["pkgUrl"]) + + Net::HTTP.start(package_uri.hostname, { use_ssl: true }) do |http| + files.map do |file| + [ + file, + fetch_file(http, "#{package_uri.path}/#{file}") + ] + end.to_h end end - - output_files end private diff --git a/lib/importmap/package.rb b/lib/importmap/package.rb index 93b209b..7e8134c 100644 --- a/lib/importmap/package.rb +++ b/lib/importmap/package.rb @@ -26,11 +26,13 @@ def download jspm_api = Importmap::JspmApi.new + versioned_package_name = "#{@package_name}#{@version}" + files = jspm_api.download( - versioned_package_name: "#{@package_name}#{@version}", + versioned_package_names: [versioned_package_name], provider: @provider, exclude: ["unused", "readme", "types"] - ) + )[versioned_package_name] files.each do |file, downloaded_file| save_vendored_file(file, downloaded_file) diff --git a/test/jspm_api_integration_test.rb b/test/jspm_api_integration_test.rb index c28363e..c965446 100644 --- a/test/jspm_api_integration_test.rb +++ b/test/jspm_api_integration_test.rb @@ -7,25 +7,25 @@ class Importmap::JspmApiIntegrationTest < ActiveSupport::TestCase end test "#download when given a valid input" do - result = @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm.io", exclude: []) + result = @jspm_api.download(versioned_package_names: ["@popperjs/core@2.11.8"], provider: "jspm.io", exclude: []) - assert result.keys.include?("lib/dom-utils/getWindow.js") - assert result.keys.include?("lib/index.js") + assert result["@popperjs/core@2.11.8"].keys.include?("lib/dom-utils/getWindow.js") + assert result["@popperjs/core@2.11.8"].keys.include?("lib/index.js") - result = @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm", exclude: []) + result = @jspm_api.download(versioned_package_names: ["@popperjs/core@2.11.8"], provider: "jspm", exclude: []) - assert result.keys.include?("lib/dom-utils/getWindow.js") - assert result.keys.include?("lib/index.js") + assert result["@popperjs/core@2.11.8"].keys.include?("lib/dom-utils/getWindow.js") + assert result["@popperjs/core@2.11.8"].keys.include?("lib/index.js") end test "#download when given a bad package" do - result = @jspm_api.download(versioned_package_name: "@popperjs/corenoversion", provider: "jspm.io", exclude: []) + result = @jspm_api.download(versioned_package_names: ["@popperjs/corenoversion"], provider: "jspm.io", exclude: []) assert_equal result, {} end test "#download when given a bad provider" do - result = @jspm_api.download(versioned_package_name: "@popperjs/corenoversion", provider: "jspmfoobarbaz", exclude: []) + result = @jspm_api.download(versioned_package_names: ["@popperjs/corenoversion"], provider: "jspmfoobarbaz", exclude: []) assert_equal result, {} end @@ -35,7 +35,7 @@ class Importmap::JspmApiIntegrationTest < ActiveSupport::TestCase Importmap::JspmApi.download_endpoint = URI("https://invalid./error") assert_raises(Importmap::JspmApi::HTTPError) do - @jspm_api.download(versioned_package_name: "@popperjs/core@2.11.8", provider: "jspm.io", exclude: []) + @jspm_api.download(versioned_package_names: ["@popperjs/core@2.11.8"], provider: "jspm.io", exclude: []) end ensure Importmap::JspmApi.download_endpoint = original_endpoint