ruby - 使用 progressbar/ruby-progressbar gem 时出现 Zlib::BufError

Question

我使用以下 Ruby 片段下载 a 8.9MB file .

require 'open-uri'
require 'net/http'
require 'uri'

def http_download_no_progress_bar(uri, filename)
  uri.open(read_timeout: 500) do |file|
    open filename, 'w' do |io|
      file.each_line do |line|
        io.write line
      end
    end
  end
end

我想添加 progressbar gem 可视化下载过程:

require 'open-uri'
require 'progressbar'
require 'net/http'
require 'uri'

def http_download_with_progressbar(uri, filename)
  progressbar = nil
  uri.open(
    read_timeout: 500,
    content_length_proc: lambda { |total|
    if total && 0 < total.to_i
      progressbar = ProgressBar.new("...", total)
      progressbar.file_transfer_mode
    end
    },
    progress_proc: lambda { |step|
       progressbar.set step if progressbar
    }
  ) do |file|
    open filename, 'w' do |io|
      file.each_line do |line|
        io.write line
      end
    end
  end
end

但是，它现在失败并出现以下错误:

/home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:357:in `finish': 
buffer error (Zlib::BufError)oooooo  |   8.0MB   8.6MB/s ETA:   0:00:00

    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:357:in `finish'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:262:in `ensure in inflater'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:262:in `inflater'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:274:in `read_body_0'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:201:in `read_body'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:328:in `block (2 levels) in open_http'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:1415:in `block (2 levels) in transport_request'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http/response.rb:162:in `reading_body'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:1414:in `block in transport_request'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:1405:in `catch'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:1405:in `transport_request'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:1378:in `request'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:319:in `block in open_http'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/net/http.rb:853:in `start'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:313:in `open_http'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:724:in `buffer_open'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:210:in `block in open_loop'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:208:in `catch'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:208:in `open_loop'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:149:in `open_uri'
    from /home/user/.rvm/rubies/ruby-2.1.1/lib/ruby/2.1.0/open-uri.rb:704:in `open'

同时我也尝试了 ruby-progressbar gem :

require 'open-uri'
require 'ruby-progressbar'
require 'net/http'
require 'uri'

def http_download_with_ruby_progressbar(uri, filename)
  progressbar = nil
  uri.open(
    read_timeout: 500,
    content_length_proc: lambda { |total|
      if total && 0 < total.to_i
        progressbar = ProgressBar.create(title: filename, total: total)
      end
      },
      progress_proc: lambda { |step|
        progressbar.progress = step if progressbar
      }
  ) do |file|
    open filename, 'w' do |io|
      file.each_line do |line|
        io.write line
      end
    end
  end
end

它因相同的错误 而失败。这是 associated issue对于这个问题。

Answer 1

问题是您尝试下载的文件，因为每种方法都适用于 this file : https://androidnetworktester.googlecode.com/files/1mb.txt。

问题是您的文件比它说的要大。 content_length_proc 表示它是 8549968 字节(8.15MB)，而它是 101187668 字节(96.5MB)(下载文件后用 ls 检查)。现在我有一个不会崩溃并给你一个进度条的替代方案:

def http_download_with_words(uri, filename)
  bytes_total = nil
  uri.open(
           read_timeout: 500,
           :content_length_proc => lambda{|content_length|
             bytes_total = content_length},
           :progress_proc => lambda{|bytes_transferred|
             if bytes_total
               # Print progress
               print("\r#{bytes_transferred}/#{bytes_total}")
             else
               # We don’t know how much we get, so just print number
               # of transferred bytes
               print("\r#{bytes_transferred} (total size unknown)")
             end
             }
           ) do |file|
    open filename, 'w' do |io|
      file.each_line do |line|
        io.write line
      end
    end
  end
end

http_download_with_words(URI( 'http://data.wien.gv.at/daten/geo?service=WFS&request=GetFeature&version=1.1.0&typeName=ogdwien%3aBAUMOGD&srsName=EPSG:4326' ), 'temp.txt')

这是不言自明的，(参见 here。)

现在我无法弄清楚的部分是进度条 gem 究竟是如何干扰 ZLib 的。大多数事情在过程中似乎都运行良好(例如让它们打印随机的东西)所以我假设这两个进度条在完成时做了一些奇怪的事情，以某种方式扰乱了传输。如果有人能弄清楚这是为什么，我会非常感兴趣？