在Crystal中线程化HTTP请求

Question

我有需要以“并行”运行的代码（不是真的，我知道Crystal不支持并行性）。

require "http/client"

thread_count = 4
resps = [] of HTTP::Client::Response
mutex = Thread::Mutex.new

urls = [] of String
(1..10).each { |i| urls << "http://httpbin.org/delay#{i}"}

threads = Array.new(thread_count) {
  Thread.new do
    while url = mutex.synchronize { urls.pop? }
      resp = HTTP::Client.get(url)
      mutex.synchronize { resps << resp }
    end
  end
}

threads.map(&.join)

Thread类的源文件说不要使用它。无论如何，这段代码不适用于HTTP::Client。

Answer 1

使用spawn：

require "http/client"
require "json"

WORKER_COUNT = 4

to_fetch = Channel(String?).new(WORKER_COUNT)
responses = Channel(HTTP::Client::Response?).new(WORKER_COUNT)

WORKER_COUNT.times do
  spawn do
    loop do
      url = to_fetch.receive
      responses.send url ? HTTP::Client.get(url) : nil
      break unless url
    end
  end
end

spawn do
  10.times do |i|
    to_fetch.send "http://httpbin.org/delay/#{i}"
  end
  WORKER_COUNT.times do
    to_fetch.send nil
  end
end

start = Time.now
worker_done_count = 0
loop do
  response = responses.receive
  if response
    puts "#{Time.now - start}: fetched #{JSON.parse(response.body)["url"]}"
  else
    worker_done_count += 1
    break if worker_done_count == WORKER_COUNT
  end
end