|
| 1 | +# frozen_string_literal: true |
| 2 | + |
| 3 | +require "active_support/core_ext/string/filters" |
| 4 | +require "compact_index" |
| 5 | +require "gemstash" |
| 6 | +require "stringio" |
| 7 | +require "zlib" |
| 8 | + |
| 9 | +module Gemstash |
| 10 | + # Comment |
| 11 | + class CompactIndexBuilder |
| 12 | + include Gemstash::Env::Helper |
| 13 | + attr_reader :result |
| 14 | + |
| 15 | + def self.serve(app, ...) |
| 16 | + app.content_type "text/plain; charset=utf-8" |
| 17 | + body = new(app.auth, ...).serve |
| 18 | + app.etag Digest::MD5.hexdigest(body) |
| 19 | + sha256 = Digest::SHA256.base64digest(body) |
| 20 | + app.headers "Accept-Ranges" => "bytes", "Digest" => "sha-256=#{sha256}", "Repr-Digest" => "sha-256=:#{sha256}:", |
| 21 | + "Content-Length" => body.bytesize.to_s |
| 22 | + body |
| 23 | + end |
| 24 | + |
| 25 | + def self.invalidate_stored(name) |
| 26 | + storage = Gemstash::Storage.for("private").for("compact_index") |
| 27 | + storage.resource("names").delete(:names) |
| 28 | + storage.resource("versions").delete(:versions) |
| 29 | + storage.resource("info/#{name}").delete(:info) |
| 30 | + end |
| 31 | + |
| 32 | + def initialize(auth) |
| 33 | + @auth = auth |
| 34 | + end |
| 35 | + |
| 36 | + def serve |
| 37 | + check_auth if gemstash_env.config[:protected_fetch] |
| 38 | + fetch_from_storage |
| 39 | + return result if result |
| 40 | + |
| 41 | + build_result |
| 42 | + store_result |
| 43 | + result |
| 44 | + end |
| 45 | + |
| 46 | + private |
| 47 | + |
| 48 | + def storage |
| 49 | + @storage ||= Gemstash::Storage.for("private").for("compact_index") |
| 50 | + end |
| 51 | + |
| 52 | + def fetch_from_storage |
| 53 | + resource = fetch_resource |
| 54 | + return unless resource.exist?(key) |
| 55 | + |
| 56 | + @result = resource.load(key).content(key) |
| 57 | + rescue StandardError |
| 58 | + # On the off-chance of a race condition between specs.exist? and specs.load |
| 59 | + @result = nil |
| 60 | + end |
| 61 | + |
| 62 | + def store_result |
| 63 | + fetch_resource.save(key => @result) |
| 64 | + end |
| 65 | + |
| 66 | + def check_auth |
| 67 | + @auth.check("fetch") |
| 68 | + end |
| 69 | + |
| 70 | + # Comment |
| 71 | + class Versions < CompactIndexBuilder |
| 72 | + def fetch_resource |
| 73 | + storage.resource("versions") |
| 74 | + end |
| 75 | + |
| 76 | + def build_result(force_rebuild: false) |
| 77 | + resource = fetch_resource |
| 78 | + base = !force_rebuild && resource.exist?("versions.list") && resource.content("versions.list") |
| 79 | + Tempfile.create("versions.list") do |file| |
| 80 | + versions_file = CompactIndex::VersionsFile.new(file.path) |
| 81 | + if base |
| 82 | + file.write(base) |
| 83 | + file.close |
| 84 | + @result = versions_file.contents( |
| 85 | + compact_index_versions(versions_file.updated_at.to_time) |
| 86 | + ) |
| 87 | + else |
| 88 | + ts = Time.now.iso8601 |
| 89 | + versions_file.create( |
| 90 | + compact_index_public_versions(ts), ts |
| 91 | + ) |
| 92 | + @result = file.read |
| 93 | + resource.save("versions.list" => @result) |
| 94 | + end |
| 95 | + end |
| 96 | + end |
| 97 | + |
| 98 | + private |
| 99 | + |
| 100 | + def compact_index_versions(date) |
| 101 | + all_versions = Sequel::Model.db[<<~SQL.squish, date, date].to_a |
| 102 | + SELECT r.name as name, v.created_at as date, v.info_checksum as info_checksum, v.number as number, v.platform as platform |
| 103 | + FROM rubygems AS r, versions AS v |
| 104 | + WHERE v.rubygem_id = r.id AND |
| 105 | + v.created_at > ? |
| 106 | +
|
| 107 | + UNION ALL |
| 108 | +
|
| 109 | + SELECT r.name as name, v.yanked_at as date, v.yanked_info_checksum as info_checksum, '-'||v.number as number, v.platform as platform |
| 110 | + FROM rubygems AS r, versions AS v |
| 111 | + WHERE v.rubygem_id = r.id AND |
| 112 | + v.indexed is false AND |
| 113 | + v.yanked_at > ? |
| 114 | +
|
| 115 | + ORDER BY date, number, platform, name |
| 116 | + SQL |
| 117 | + |
| 118 | + # not ordered correctly in sqlite for some reason |
| 119 | + all_versions.sort_by! {|v| [v[:date], v[:number], v[:platform], v[:name]] } |
| 120 | + map_gem_versions(all_versions.map {|v| [v[:name], [v]] }) |
| 121 | + end |
| 122 | + |
| 123 | + def compact_index_public_versions(date) |
| 124 | + all_versions = Sequel::Model.db[<<~SQL.squish, date, date].to_a |
| 125 | + SELECT r.name, v.indexed, COALESCE(v.yanked_at, v.created_at) as stamp, |
| 126 | + COALESCE(v.yanked_info_checksum, v.info_checksum) as info_checksum, v.number, v.platform |
| 127 | + FROM rubygems AS r, versions AS v |
| 128 | + WHERE v.rubygem_id = r.id AND |
| 129 | + (v.created_at <= ? OR v.yanked_at <= ?) |
| 130 | + ORDER BY name, COALESCE(v.yanked_at, v.created_at), number, platform |
| 131 | + SQL |
| 132 | + |
| 133 | + versions_by_gem = all_versions.group_by {|row| row[:name] } |
| 134 | + versions_by_gem.each_value do |versions| |
| 135 | + info_checksum = versions.last[:info_checksum] |
| 136 | + versions.select! {|v| v[:indexed] == true } |
| 137 | + # Set all versions' info_checksum to work around https://github.com/bundler/compact_index/pull/20 |
| 138 | + versions.each {|v| v[:info_checksum] = info_checksum } |
| 139 | + end |
| 140 | + |
| 141 | + map_gem_versions(versions_by_gem) |
| 142 | + end |
| 143 | + |
| 144 | + def map_gem_versions(versions_by_gem) |
| 145 | + versions_by_gem.map do |name, versions| |
| 146 | + CompactIndex::Gem.new( |
| 147 | + name, |
| 148 | + versions.map do |row| |
| 149 | + CompactIndex::GemVersion.new( |
| 150 | + row[:number], |
| 151 | + row[:platform], |
| 152 | + nil, # sha256 |
| 153 | + row[:info_checksum], |
| 154 | + nil, # dependencies |
| 155 | + nil, # version.required_ruby_version, |
| 156 | + nil, # version.required_rubygems_version |
| 157 | + ) |
| 158 | + end |
| 159 | + ) |
| 160 | + end |
| 161 | + end |
| 162 | + |
| 163 | + def key |
| 164 | + :versions |
| 165 | + end |
| 166 | + end |
| 167 | + |
| 168 | + # Comment |
| 169 | + class Info < CompactIndexBuilder |
| 170 | + def initialize(auth, name) |
| 171 | + super(auth) |
| 172 | + @name = name |
| 173 | + end |
| 174 | + |
| 175 | + def fetch_resource |
| 176 | + storage.resource("info/#{@name}") |
| 177 | + end |
| 178 | + |
| 179 | + def build_result |
| 180 | + @result = CompactIndex.info(requirements_and_dependencies) |
| 181 | + end |
| 182 | + |
| 183 | + private |
| 184 | + |
| 185 | + def requirements_and_dependencies |
| 186 | + group_by_columns = "number, platform, sha256, info_checksum, required_ruby_version, required_rubygems_version, versions.created_at" |
| 187 | + |
| 188 | + dep_req_agg = "string_agg(dependencies.requirements, '@' ORDER BY dependencies.rubygem_name, dependencies.id) as dep_req_agg" |
| 189 | + |
| 190 | + dep_name_agg = "string_agg(dependencies.rubygem_name, ',' ORDER BY dependencies.rubygem_name) AS dep_name_agg" |
| 191 | + |
| 192 | + DB::Rubygem.db[<<~SQL.squish, @name]. |
| 193 | + SELECT #{group_by_columns}, #{dep_req_agg}, #{dep_name_agg} |
| 194 | + FROM rubygems |
| 195 | + LEFT JOIN versions ON versions.rubygem_id = rubygems.id |
| 196 | + LEFT JOIN dependencies ON dependencies.version_id = versions.id |
| 197 | + WHERE rubygems.name = ? AND versions.indexed = true |
| 198 | + GROUP BY #{group_by_columns} |
| 199 | + ORDER BY versions.created_at, number, platform, dep_name_agg |
| 200 | + SQL |
| 201 | + map do |row| |
| 202 | + reqs = row[:dep_req_agg]&.split("@") |
| 203 | + dep_names = row[:dep_name_agg]&.split(",") |
| 204 | + |
| 205 | + raise "Dependencies and requirements are not the same size:\n reqs: #{reqs.inspect}\n dep_names: #{dep_names.inspect}\n row: #{row.inspect}" if dep_names&.size != reqs&.size |
| 206 | + |
| 207 | + deps = [] |
| 208 | + if reqs |
| 209 | + dep_names.zip(reqs).each do |name, req| |
| 210 | + deps << CompactIndex::Dependency.new(name, req) |
| 211 | + end |
| 212 | + end |
| 213 | + |
| 214 | + CompactIndex::GemVersion.new( |
| 215 | + row[:number], |
| 216 | + row[:platform], |
| 217 | + row[:sha256], |
| 218 | + nil, # info_checksum |
| 219 | + deps, |
| 220 | + row[:required_ruby_version], |
| 221 | + row[:required_rubygems_version] |
| 222 | + ) |
| 223 | + end |
| 224 | + end |
| 225 | + |
| 226 | + def key |
| 227 | + :info |
| 228 | + end |
| 229 | + end |
| 230 | + |
| 231 | + # Comment |
| 232 | + class Names < CompactIndexBuilder |
| 233 | + def fetch_resource |
| 234 | + storage.resource("names") |
| 235 | + end |
| 236 | + |
| 237 | + def build_result |
| 238 | + names = DB::Rubygem.db[<<~SQL.squish].map {|row| row[:name] } |
| 239 | + SELECT name |
| 240 | + FROM rubygems |
| 241 | + LEFT JOIN versions ON versions.rubygem_id = rubygems.id |
| 242 | + WHERE versions.indexed = true |
| 243 | + GROUP BY name |
| 244 | + HAVING COUNT(versions.id) > 0 |
| 245 | + ORDER BY name |
| 246 | + SQL |
| 247 | + @result = CompactIndex.names(names).encode("UTF-8") |
| 248 | + end |
| 249 | + |
| 250 | + private |
| 251 | + |
| 252 | + def key |
| 253 | + :names |
| 254 | + end |
| 255 | + end |
| 256 | + end |
| 257 | +end |
0 commit comments