summaryrefslogtreecommitdiff
path: root/jni/ruby/lib/rubygems/indexer.rb
diff options
context:
space:
mode:
authorJari Vetoniemi <jari.vetoniemi@indooratlas.com>2020-03-16 18:49:26 +0900
committerJari Vetoniemi <jari.vetoniemi@indooratlas.com>2020-03-30 00:39:06 +0900
commitfcbf63e62c627deae76c1b8cb8c0876c536ed811 (patch)
tree64cb17de3f41a2b6fef2368028fbd00349946994 /jni/ruby/lib/rubygems/indexer.rb
Fresh start
Diffstat (limited to 'jni/ruby/lib/rubygems/indexer.rb')
-rw-r--r--jni/ruby/lib/rubygems/indexer.rb498
1 files changed, 498 insertions, 0 deletions
diff --git a/jni/ruby/lib/rubygems/indexer.rb b/jni/ruby/lib/rubygems/indexer.rb
new file mode 100644
index 0000000..f94fb1b
--- /dev/null
+++ b/jni/ruby/lib/rubygems/indexer.rb
@@ -0,0 +1,498 @@
+require 'rubygems'
+require 'rubygems/package'
+require 'time'
+
+begin
+ gem 'builder'
+ require 'builder/xchar'
+rescue LoadError
+end
+
+##
+# Top level class for building the gem repository index.
+
+class Gem::Indexer
+
+ include Gem::UserInteraction
+
+ ##
+ # Build indexes for RubyGems 1.2.0 and newer when true
+
+ attr_accessor :build_modern
+
+ ##
+ # Index install location
+
+ attr_reader :dest_directory
+
+ ##
+ # Specs index install location
+
+ attr_reader :dest_specs_index
+
+ ##
+ # Latest specs index install location
+
+ attr_reader :dest_latest_specs_index
+
+ ##
+ # Prerelease specs index install location
+
+ attr_reader :dest_prerelease_specs_index
+
+ ##
+ # Index build directory
+
+ attr_reader :directory
+
+ ##
+ # Create an indexer that will index the gems in +directory+.
+
+ def initialize(directory, options = {})
+ require 'fileutils'
+ require 'tmpdir'
+ require 'zlib'
+
+ unless defined?(Builder::XChar) then
+ raise "Gem::Indexer requires that the XML Builder library be installed:" +
+ "\n\tgem install builder"
+ end
+
+ options = { :build_modern => true }.merge options
+
+ @build_modern = options[:build_modern]
+
+ @dest_directory = directory
+ @directory = File.join(Dir.tmpdir, "gem_generate_index_#{$$}")
+
+ marshal_name = "Marshal.#{Gem.marshal_version}"
+
+ @master_index = File.join @directory, 'yaml'
+ @marshal_index = File.join @directory, marshal_name
+
+ @quick_dir = File.join @directory, 'quick'
+ @quick_marshal_dir = File.join @quick_dir, marshal_name
+ @quick_marshal_dir_base = File.join "quick", marshal_name # FIX: UGH
+
+ @quick_index = File.join @quick_dir, 'index'
+ @latest_index = File.join @quick_dir, 'latest_index'
+
+ @specs_index = File.join @directory, "specs.#{Gem.marshal_version}"
+ @latest_specs_index =
+ File.join(@directory, "latest_specs.#{Gem.marshal_version}")
+ @prerelease_specs_index =
+ File.join(@directory, "prerelease_specs.#{Gem.marshal_version}")
+ @dest_specs_index =
+ File.join(@dest_directory, "specs.#{Gem.marshal_version}")
+ @dest_latest_specs_index =
+ File.join(@dest_directory, "latest_specs.#{Gem.marshal_version}")
+ @dest_prerelease_specs_index =
+ File.join(@dest_directory, "prerelease_specs.#{Gem.marshal_version}")
+
+ @files = []
+ end
+
+ ##
+ # Abbreviate the spec for downloading. Abbreviated specs are only used for
+ # searching, downloading and related activities and do not need deployment
+ # specific information (e.g. list of files). So we abbreviate the spec,
+ # making it much smaller for quicker downloads.
+ #--
+ # TODO move to Gem::Specification
+
+ def abbreviate(spec)
+ spec.files = []
+ spec.test_files = []
+ spec.rdoc_options = []
+ spec.extra_rdoc_files = []
+ spec.cert_chain = []
+ spec
+ end
+
+ ##
+ # Build various indicies
+
+ def build_indicies
+ Gem::Specification.dirs = []
+ Gem::Specification.add_specs(*map_gems_to_specs(gem_file_list))
+
+ build_marshal_gemspecs
+ build_modern_indicies if @build_modern
+
+ compress_indicies
+ end
+
+ ##
+ # Builds Marshal quick index gemspecs.
+
+ def build_marshal_gemspecs
+ count = Gem::Specification.count { |s| not s.default_gem? }
+ progress = ui.progress_reporter count,
+ "Generating Marshal quick index gemspecs for #{count} gems",
+ "Complete"
+
+ files = []
+
+ Gem.time 'Generated Marshal quick index gemspecs' do
+ Gem::Specification.each do |spec|
+ next if spec.default_gem?
+ spec_file_name = "#{spec.original_name}.gemspec.rz"
+ marshal_name = File.join @quick_marshal_dir, spec_file_name
+
+ marshal_zipped = Gem.deflate Marshal.dump(spec)
+ open marshal_name, 'wb' do |io| io.write marshal_zipped end
+
+ files << marshal_name
+
+ progress.updated spec.original_name
+ end
+
+ progress.done
+ end
+
+ @files << @quick_marshal_dir
+
+ files
+ end
+
+ ##
+ # Build a single index for RubyGems 1.2 and newer
+
+ def build_modern_index(index, file, name)
+ say "Generating #{name} index"
+
+ Gem.time "Generated #{name} index" do
+ open(file, 'wb') do |io|
+ specs = index.map do |*spec|
+ # We have to splat here because latest_specs is an array, while the
+ # others are hashes.
+ spec = spec.flatten.last
+ platform = spec.original_platform
+
+ # win32-api-1.0.4-x86-mswin32-60
+ unless String === platform then
+ alert_warning "Skipping invalid platform in gem: #{spec.full_name}"
+ next
+ end
+
+ platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
+ [spec.name, spec.version, platform]
+ end
+
+ specs = compact_specs(specs)
+ Marshal.dump(specs, io)
+ end
+ end
+ end
+
+ ##
+ # Builds indicies for RubyGems 1.2 and newer. Handles full, latest, prerelease
+
+ def build_modern_indicies
+ specs = Gem::Specification.reject { |s| s.default_gem? }
+
+ prerelease, released = specs.partition { |s|
+ s.version.prerelease?
+ }
+ latest_specs =
+ Gem::Specification.latest_specs.reject { |s| s.default_gem? }
+
+ build_modern_index(released.sort, @specs_index, 'specs')
+ build_modern_index(latest_specs.sort, @latest_specs_index, 'latest specs')
+ build_modern_index(prerelease.sort, @prerelease_specs_index,
+ 'prerelease specs')
+
+ @files += [@specs_index,
+ "#{@specs_index}.gz",
+ @latest_specs_index,
+ "#{@latest_specs_index}.gz",
+ @prerelease_specs_index,
+ "#{@prerelease_specs_index}.gz"]
+ end
+
+ def map_gems_to_specs gems
+ gems.map { |gemfile|
+ if File.size(gemfile) == 0 then
+ alert_warning "Skipping zero-length gem: #{gemfile}"
+ next
+ end
+
+ begin
+ spec = Gem::Package.new(gemfile).spec
+ spec.loaded_from = gemfile
+
+ # HACK: fuck this shit - borks all tests that use pl1
+ # if File.basename(gemfile, ".gem") != spec.original_name then
+ # exp = spec.full_name
+ # exp << " (#{spec.original_name})" if
+ # spec.original_name != spec.full_name
+ # msg = "Skipping misnamed gem: #{gemfile} should be named #{exp}"
+ # alert_warning msg
+ # next
+ # end
+
+ abbreviate spec
+ sanitize spec
+
+ spec
+ rescue SignalException
+ alert_error "Received signal, exiting"
+ raise
+ rescue Exception => e
+ msg = ["Unable to process #{gemfile}",
+ "#{e.message} (#{e.class})",
+ "\t#{e.backtrace.join "\n\t"}"].join("\n")
+ alert_error msg
+ end
+ }.compact
+ end
+
+ ##
+ # Compresses indicies on disk
+ #--
+ # All future files should be compressed using gzip, not deflate
+
+ def compress_indicies
+ say "Compressing indicies"
+
+ Gem.time 'Compressed indicies' do
+ if @build_modern then
+ gzip @specs_index
+ gzip @latest_specs_index
+ gzip @prerelease_specs_index
+ end
+ end
+ end
+
+ ##
+ # Compacts Marshal output for the specs index data source by using identical
+ # objects as much as possible.
+
+ def compact_specs(specs)
+ names = {}
+ versions = {}
+ platforms = {}
+
+ specs.map do |(name, version, platform)|
+ names[name] = name unless names.include? name
+ versions[version] = version unless versions.include? version
+ platforms[platform] = platform unless platforms.include? platform
+
+ [names[name], versions[version], platforms[platform]]
+ end
+ end
+
+ ##
+ # Compress +filename+ with +extension+.
+
+ def compress(filename, extension)
+ data = Gem.read_binary filename
+
+ zipped = Gem.deflate data
+
+ open "#{filename}.#{extension}", 'wb' do |io|
+ io.write zipped
+ end
+ end
+
+ ##
+ # List of gem file names to index.
+
+ def gem_file_list
+ Dir[File.join(@dest_directory, "gems", '*.gem')]
+ end
+
+ ##
+ # Builds and installs indicies.
+
+ def generate_index
+ make_temp_directories
+ build_indicies
+ install_indicies
+ rescue SignalException
+ ensure
+ FileUtils.rm_rf @directory
+ end
+
+ ##
+ # Zlib::GzipWriter wrapper that gzips +filename+ on disk.
+
+ def gzip(filename)
+ Zlib::GzipWriter.open "#{filename}.gz" do |io|
+ io.write Gem.read_binary(filename)
+ end
+ end
+
+ ##
+ # Install generated indicies into the destination directory.
+
+ def install_indicies
+ verbose = Gem.configuration.really_verbose
+
+ say "Moving index into production dir #{@dest_directory}" if verbose
+
+ files = @files
+ files.delete @quick_marshal_dir if files.include? @quick_dir
+
+ if files.include? @quick_marshal_dir and not files.include? @quick_dir then
+ files.delete @quick_marshal_dir
+
+ dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
+
+ FileUtils.mkdir_p File.dirname(dst_name), :verbose => verbose
+ FileUtils.rm_rf dst_name, :verbose => verbose
+ FileUtils.mv(@quick_marshal_dir, dst_name,
+ :verbose => verbose, :force => true)
+ end
+
+ files = files.map do |path|
+ path.sub(/^#{Regexp.escape @directory}\/?/, '') # HACK?
+ end
+
+ files.each do |file|
+ src_name = File.join @directory, file
+ dst_name = File.join @dest_directory, file
+
+ FileUtils.rm_rf dst_name, :verbose => verbose
+ FileUtils.mv(src_name, @dest_directory,
+ :verbose => verbose, :force => true)
+ end
+ end
+
+ ##
+ # Make directories for index generation
+
+ def make_temp_directories
+ FileUtils.rm_rf @directory
+ FileUtils.mkdir_p @directory, :mode => 0700
+ FileUtils.mkdir_p @quick_marshal_dir
+ end
+
+ ##
+ # Ensure +path+ and path with +extension+ are identical.
+
+ def paranoid(path, extension)
+ data = Gem.read_binary path
+ compressed_data = Gem.read_binary "#{path}.#{extension}"
+
+ unless data == Gem.inflate(compressed_data) then
+ raise "Compressed file #{compressed_path} does not match uncompressed file #{path}"
+ end
+ end
+
+ ##
+ # Sanitize the descriptive fields in the spec. Sometimes non-ASCII
+ # characters will garble the site index. Non-ASCII characters will
+ # be replaced by their XML entity equivalent.
+
+ def sanitize(spec)
+ spec.summary = sanitize_string(spec.summary)
+ spec.description = sanitize_string(spec.description)
+ spec.post_install_message = sanitize_string(spec.post_install_message)
+ spec.authors = spec.authors.collect { |a| sanitize_string(a) }
+
+ spec
+ end
+
+ ##
+ # Sanitize a single string.
+
+ def sanitize_string(string)
+ return string unless string
+
+ # HACK the #to_s is in here because RSpec has an Array of Arrays of
+ # Strings for authors. Need a way to disallow bad values on gemspec
+ # generation. (Probably won't happen.)
+ string = string.to_s
+
+ begin
+ Builder::XChar.encode string
+ rescue NameError, NoMethodError
+ string.to_xs
+ end
+ end
+
+ ##
+ # Perform an in-place update of the repository from newly added gems.
+
+ def update_index
+ make_temp_directories
+
+ specs_mtime = File.stat(@dest_specs_index).mtime
+ newest_mtime = Time.at 0
+
+ updated_gems = gem_file_list.select do |gem|
+ gem_mtime = File.stat(gem).mtime
+ newest_mtime = gem_mtime if gem_mtime > newest_mtime
+ gem_mtime >= specs_mtime
+ end
+
+ if updated_gems.empty? then
+ say 'No new gems'
+ terminate_interaction 0
+ end
+
+ specs = map_gems_to_specs updated_gems
+ prerelease, released = specs.partition { |s| s.version.prerelease? }
+
+ Gem::Specification.dirs = []
+ Gem::Specification.add_specs(*specs)
+
+ files = build_marshal_gemspecs
+
+ Gem.time 'Updated indexes' do
+ update_specs_index released, @dest_specs_index, @specs_index
+ update_specs_index released, @dest_latest_specs_index, @latest_specs_index
+ update_specs_index(prerelease,
+ @dest_prerelease_specs_index,
+ @prerelease_specs_index)
+ end
+
+ compress_indicies
+
+ verbose = Gem.configuration.really_verbose
+
+ say "Updating production dir #{@dest_directory}" if verbose
+
+ files << @specs_index
+ files << "#{@specs_index}.gz"
+ files << @latest_specs_index
+ files << "#{@latest_specs_index}.gz"
+ files << @prerelease_specs_index
+ files << "#{@prerelease_specs_index}.gz"
+
+ files = files.map do |path|
+ path.sub(/^#{Regexp.escape @directory}\/?/, '') # HACK?
+ end
+
+ files.each do |file|
+ src_name = File.join @directory, file
+ dst_name = File.join @dest_directory, file # REFACTOR: duped above
+
+ FileUtils.mv src_name, dst_name, :verbose => verbose,
+ :force => true
+
+ File.utime newest_mtime, newest_mtime, dst_name
+ end
+ end
+
+ ##
+ # Combines specs in +index+ and +source+ then writes out a new copy to
+ # +dest+. For a latest index, does not ensure the new file is minimal.
+
+ def update_specs_index(index, source, dest)
+ specs_index = Marshal.load Gem.read_binary(source)
+
+ index.each do |spec|
+ platform = spec.original_platform
+ platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
+ specs_index << [spec.name, spec.version, platform]
+ end
+
+ specs_index = compact_specs specs_index.uniq.sort
+
+ open dest, 'wb' do |io|
+ Marshal.dump specs_index, io
+ end
+ end
+end