| Class | Gem::RemoteFetcher |
| In: |
lib/rubygems/remote_fetcher.rb
|
| Parent: | Object |
RemoteFetcher handles the details of fetching gems and gem information from a remote source.
| BuiltinSSLCerts | = | "/etc/ssl/certs/ca-certificates.crt" |
Cached RemoteFetcher instance.
# File lib/rubygems/remote_fetcher.rb, line 42
42: def self.fetcher
43: @fetcher ||= self.new Gem.configuration[:http_proxy]
44: end
Initialize a remote fetcher using the source URI and possible proxy information.
proxy
variable setting
HTTP_PROXY_PASS)
# File lib/rubygems/remote_fetcher.rb, line 57
57: def initialize(proxy = nil)
58: require 'net/http'
59: require 'stringio'
60: require 'time'
61: require 'uri'
62:
63: Socket.do_not_reverse_lookup = true
64:
65: @connections = {}
66: @requests = Hash.new 0
67: @proxy_uri =
68: case proxy
69: when :no_proxy then nil
70: when nil then get_proxy_from_env
71: when URI::HTTP then proxy
72: else URI.parse(proxy)
73: end
74: @user_agent = user_agent
75: end
# File lib/rubygems/remote_fetcher.rb, line 356
356: def add_rubygems_trusted_certs(store)
357: if File.file? BuiltinSSLCerts
358: store.add_file BuiltinSSLCerts
359: end
360: end
# File lib/rubygems/remote_fetcher.rb, line 333
333: def configure_connection_for_https(connection)
334: require 'net/https'
335:
336: connection.use_ssl = true
337: connection.verify_mode =
338: Gem.configuration.ssl_verify_mode || OpenSSL::SSL::VERIFY_PEER
339:
340: store = OpenSSL::X509::Store.new
341:
342: if Gem.configuration.ssl_ca_cert
343: if File.directory? Gem.configuration.ssl_ca_cert
344: store.add_path Gem.configuration.ssl_ca_cert
345: else
346: store.add_file Gem.configuration.ssl_ca_cert
347: end
348: else
349: store.set_default_paths
350: add_rubygems_trusted_certs(store)
351: end
352:
353: connection.cert_store = store
354: end
Creates or an HTTP connection based on uri, or retrieves an existing connection, using a proxy if needed.
# File lib/rubygems/remote_fetcher.rb, line 306
306: def connection_for(uri)
307: net_http_args = [uri.host, uri.port]
308:
309: if @proxy_uri then
310: net_http_args += [
311: @proxy_uri.host,
312: @proxy_uri.port,
313: @proxy_uri.user,
314: @proxy_uri.password
315: ]
316: end
317:
318: connection_id = [Thread.current.object_id, *net_http_args].join ':'
319: @connections[connection_id] ||= Net::HTTP.new(*net_http_args)
320: connection = @connections[connection_id]
321:
322: if https?(uri) and !connection.started? then
323: configure_connection_for_https(connection)
324: end
325:
326: connection.start unless connection.started?
327:
328: connection
329: rescue OpenSSL::SSL::SSLError, Errno::EHOSTDOWN => e
330: raise FetchError.new(e.message, uri)
331: end
# File lib/rubygems/remote_fetcher.rb, line 362
362: def correct_for_windows_path(path)
363: if path[0].chr == '/' && path[1].chr =~ /[a-z]/i && path[2].chr == ':'
364: path = path[1..-1]
365: else
366: path
367: end
368: end
Moves the gem spec from source_uri to the cache dir unless it is already there. If the source_uri is local the gem cache dir copy is always replaced.
# File lib/rubygems/remote_fetcher.rb, line 100
100: def download(spec, source_uri, install_dir = Gem.dir)
101: Gem.ensure_gem_subdirectories(install_dir) rescue nil
102:
103: if File.writable?(install_dir)
104: cache_dir = File.join install_dir, "cache"
105: else
106: cache_dir = File.join Gem.user_dir, "cache"
107: end
108:
109: gem_file_name = File.basename spec.cache_file
110: local_gem_path = File.join cache_dir, gem_file_name
111:
112: FileUtils.mkdir_p cache_dir rescue nil unless File.exist? cache_dir
113:
114: # Always escape URI's to deal with potential spaces and such
115: unless URI::Generic === source_uri
116: source_uri = URI.parse(URI.const_defined?(:DEFAULT_PARSER) ?
117: URI::DEFAULT_PARSER.escape(source_uri.to_s) :
118: URI.escape(source_uri.to_s))
119: end
120:
121: scheme = source_uri.scheme
122:
123: # URI.parse gets confused by MS Windows paths with forward slashes.
124: scheme = nil if scheme =~ /^[a-z]$/i
125:
126: case scheme
127: when 'http', 'https' then
128: unless File.exist? local_gem_path then
129: begin
130: say "Downloading gem #{gem_file_name}" if
131: Gem.configuration.really_verbose
132:
133: remote_gem_path = source_uri + "gems/#{gem_file_name}"
134:
135: gem = self.fetch_path remote_gem_path
136: rescue Gem::RemoteFetcher::FetchError
137: raise if spec.original_platform == spec.platform
138:
139: alternate_name = "#{spec.original_name}.gem"
140:
141: say "Failed, downloading gem #{alternate_name}" if
142: Gem.configuration.really_verbose
143:
144: remote_gem_path = source_uri + "gems/#{alternate_name}"
145:
146: gem = self.fetch_path remote_gem_path
147: end
148:
149: File.open local_gem_path, 'wb' do |fp|
150: fp.write gem
151: end
152: end
153: when 'file' then
154: begin
155: path = source_uri.path
156: path = File.dirname(path) if File.extname(path) == '.gem'
157:
158: remote_gem_path = correct_for_windows_path(File.join(path, 'gems', gem_file_name))
159:
160: FileUtils.cp(remote_gem_path, local_gem_path)
161: rescue Errno::EACCES
162: local_gem_path = source_uri.to_s
163: end
164:
165: say "Using local gem #{local_gem_path}" if
166: Gem.configuration.really_verbose
167: when nil then # TODO test for local overriding cache
168: source_path = if Gem.win_platform? && source_uri.scheme &&
169: !source_uri.path.include?(':') then
170: "#{source_uri.scheme}:#{source_uri.path}"
171: else
172: source_uri.path
173: end
174:
175: source_path = unescape source_path
176:
177: begin
178: FileUtils.cp source_path, local_gem_path unless
179: File.expand_path(source_path) == File.expand_path(local_gem_path)
180: rescue Errno::EACCES
181: local_gem_path = source_uri.to_s
182: end
183:
184: say "Using local gem #{local_gem_path}" if
185: Gem.configuration.really_verbose
186: else
187: raise Gem::InstallError, "unsupported URI scheme #{source_uri.scheme}"
188: end
189:
190: local_gem_path
191: end
Given a name and requirement, downloads this gem into cache and returns the filename. Returns nil if the gem cannot be located.
# File lib/rubygems/remote_fetcher.rb, line 84
84: def download_to_cache dependency
85: found = Gem::SpecFetcher.fetcher.fetch dependency, true, true,
86: dependency.prerelease?
87:
88: return if found.empty?
89:
90: spec, source_uri = found.sort_by { |(s,_)| s.version }.last
91:
92: download spec, source_uri
93: end
# File lib/rubygems/remote_fetcher.rb, line 258
258: def escape(str)
259: return unless str
260: @uri_parser ||= uri_escaper
261: @uri_parser.escape str
262: end
File Fetcher. Dispatched by fetch_path. Use it instead.
# File lib/rubygems/remote_fetcher.rb, line 196
196: def fetch_file uri, *_
197: Gem.read_binary correct_for_windows_path uri.path
198: end
HTTP Fetcher. Dispatched by fetch_path. Use it instead.
# File lib/rubygems/remote_fetcher.rb, line 203
203: def fetch_http uri, last_modified = nil, head = false, depth = 0
204: fetch_type = head ? Net::HTTP::Head : Net::HTTP::Get
205: response = request uri, fetch_type, last_modified
206:
207: case response
208: when Net::HTTPOK, Net::HTTPNotModified then
209: head ? response : response.body
210: when Net::HTTPMovedPermanently, Net::HTTPFound, Net::HTTPSeeOther,
211: Net::HTTPTemporaryRedirect then
212: raise FetchError.new('too many redirects', uri) if depth > 10
213:
214: location = URI.parse response['Location']
215:
216: if https?(uri) && !https?(location)
217: raise FetchError.new("redirecting to non-https resource: #{location}", uri)
218: end
219:
220: fetch_http(location, last_modified, head, depth + 1)
221: else
222: raise FetchError.new("bad response #{response.message} #{response.code}", uri)
223: end
224: end
Downloads uri and returns it as a String.
# File lib/rubygems/remote_fetcher.rb, line 231
231: def fetch_path(uri, mtime = nil, head = false)
232: uri = URI.parse uri unless URI::Generic === uri
233:
234: raise ArgumentError, "bad uri: #{uri}" unless uri
235: raise ArgumentError, "uri scheme is invalid: #{uri.scheme.inspect}" unless
236: uri.scheme
237:
238: data = send "fetch_#{uri.scheme}", uri, mtime, head
239: data = Gem.gunzip data if data and not head and uri.to_s =~ /gz$/
240: data
241: rescue FetchError
242: raise
243: rescue Timeout::Error
244: raise FetchError.new('timed out', uri.to_s)
245: rescue IOError, SocketError, SystemCallError => e
246: raise FetchError.new("#{e.class}: #{e}", uri.to_s)
247: end
Returns the size of uri in bytes.
# File lib/rubygems/remote_fetcher.rb, line 252
252: def fetch_size(uri) # TODO: phase this out
253: response = fetch_path(uri, nil, true)
254:
255: response['content-length'].to_i
256: end
Returns an HTTP proxy URI if one is set in the environment variables.
# File lib/rubygems/remote_fetcher.rb, line 279
279: def get_proxy_from_env
280: env_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
281:
282: return nil if env_proxy.nil? or env_proxy.empty?
283:
284: uri = URI.parse(normalize_uri(env_proxy))
285:
286: if uri and uri.user.nil? and uri.password.nil? then
287: # Probably we have http_proxy_* variables?
288: uri.user = escape(ENV['http_proxy_user'] || ENV['HTTP_PROXY_USER'])
289: uri.password = escape(ENV['http_proxy_pass'] || ENV['HTTP_PROXY_PASS'])
290: end
291:
292: uri
293: end
# File lib/rubygems/remote_fetcher.rb, line 502
502: def https?(uri)
503: uri.scheme.downcase == 'https'
504: end
Normalize the URI by adding "http://" if it is missing.
# File lib/rubygems/remote_fetcher.rb, line 298
298: def normalize_uri(uri)
299: (uri =~ /^(https?|ftp|file):/) ? uri : "http://#{uri}"
300: end
Read the data from the (source based) URI, but if it is a file:// URI, read from the filesystem instead.
# File lib/rubygems/remote_fetcher.rb, line 374
374: def open_uri_or_path(uri, last_modified = nil, head = false, depth = 0)
375: raise "NO: Use fetch_path instead"
376: # TODO: deprecate for fetch_path
377: end
Performs a Net::HTTP request of type request_class on uri returning a Net::HTTP response object. request maintains a table of persistent connections to reduce connect overhead.
# File lib/rubygems/remote_fetcher.rb, line 384
384: def request(uri, request_class, last_modified = nil)
385: request = request_class.new uri.request_uri
386:
387: unless uri.nil? || uri.user.nil? || uri.user.empty? then
388: request.basic_auth uri.user, uri.password
389: end
390:
391: request.add_field 'User-Agent', @user_agent
392: request.add_field 'Connection', 'keep-alive'
393: request.add_field 'Keep-Alive', '30'
394:
395: if last_modified then
396: last_modified = last_modified.utc
397: request.add_field 'If-Modified-Since', last_modified.rfc2822
398: end
399:
400: yield request if block_given?
401:
402: connection = connection_for uri
403:
404: retried = false
405: bad_response = false
406:
407: begin
408: @requests[connection.object_id] += 1
409:
410: say "#{request.method} #{uri}" if
411: Gem.configuration.really_verbose
412:
413: file_name = File.basename(uri.path)
414: # perform download progress reporter only for gems
415: if request.response_body_permitted? && file_name =~ /\.gem$/
416: reporter = ui.download_reporter
417: response = connection.request(request) do |incomplete_response|
418: if Net::HTTPOK === incomplete_response
419: reporter.fetch(file_name, incomplete_response.content_length)
420: downloaded = 0
421: data = ''
422:
423: incomplete_response.read_body do |segment|
424: data << segment
425: downloaded += segment.length
426: reporter.update(downloaded)
427: end
428: reporter.done
429: if incomplete_response.respond_to? :body=
430: incomplete_response.body = data
431: else
432: incomplete_response.instance_variable_set(:@body, data)
433: end
434: end
435: end
436: else
437: response = connection.request request
438: end
439:
440: say "#{response.code} #{response.message}" if
441: Gem.configuration.really_verbose
442:
443: rescue Net::HTTPBadResponse
444: say "bad response" if Gem.configuration.really_verbose
445:
446: reset connection
447:
448: raise FetchError.new('too many bad responses', uri) if bad_response
449:
450: bad_response = true
451: retry
452: # HACK work around EOFError bug in Net::HTTP
453: # NOTE Errno::ECONNABORTED raised a lot on Windows, and make impossible
454: # to install gems.
455: rescue EOFError, Timeout::Error,
456: Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE
457:
458: requests = @requests[connection.object_id]
459: say "connection reset after #{requests} requests, retrying" if
460: Gem.configuration.really_verbose
461:
462: raise FetchError.new('too many connection resets', uri) if retried
463:
464: reset connection
465:
466: retried = true
467: retry
468: end
469:
470: response
471: end
Resets HTTP connection connection.
# File lib/rubygems/remote_fetcher.rb, line 476
476: def reset(connection)
477: @requests.delete connection.object_id
478:
479: connection.finish
480: connection.start
481: end
# File lib/rubygems/remote_fetcher.rb, line 264
264: def unescape(str)
265: return unless str
266: @uri_parser ||= uri_escaper
267: @uri_parser.unescape str
268: end
# File lib/rubygems/remote_fetcher.rb, line 270
270: def uri_escaper
271: URI::Parser.new
272: rescue NameError
273: URI
274: end
# File lib/rubygems/remote_fetcher.rb, line 483
483: def user_agent
484: ua = "RubyGems/#{Gem::VERSION} #{Gem::Platform.local}"
485:
486: ruby_version = RUBY_VERSION
487: ruby_version += 'dev' if RUBY_PATCHLEVEL == -1
488:
489: ua << " Ruby/#{ruby_version} (#{RUBY_RELEASE_DATE}"
490: if RUBY_PATCHLEVEL >= 0 then
491: ua << " patchlevel #{RUBY_PATCHLEVEL}"
492: elsif defined?(RUBY_REVISION) then
493: ua << " revision #{RUBY_REVISION}"
494: end
495: ua << ")"
496:
497: ua << " #{RUBY_ENGINE}" if defined?(RUBY_ENGINE) and RUBY_ENGINE != 'ruby'
498:
499: ua
500: end