javascript - 在 Nodejs 中将数千个并发 https 请求分成更小的 block

Question

我正在开发一个需要进行数千个并发 https 调用的项目。 我的第一个问题是:并发限制在哪里？可以同时调用多少个电话？它取决于什么？ 其次，我在ubuntu系统上工作，在大多数情况下，系统由于请求太多而挂起。我如何将这数千个并发请求分成一次 50 或 100 个请求的 block ，并在完成这些请求后调用下一组请求。

我的代码如下:

items.forEach(function(item) {
        request().then(function() {
            if(counter == items.length) {
                deferred.resolve("Success");
                return deferred.promise;
            }
        })
        .catch(function(err) {
            deferred.reject(err);
        });
        counter++;
    });

Items 数组大多超过 10k。

Answer 1

您可以使用Promise.map bluebird 中的并发选项

var Promise = require("bluebird");
var join = Promise.join;
var fs = Promise.promisifyAll(require("fs"));
var concurrency = parseFloat(process.argv[2] || "Infinity");
console.time("reading files");
fs.readdirAsync(".").map(function(fileName) {
    var stat = fs.statAsync(fileName);
    var contents = fs.readFileAsync(fileName).catch(function ignore() {});
    return join(stat, contents, function(stat, contents) {
        return {
            stat: stat,
            fileName: fileName,
            contents: contents
        }
    });
// The return value of .map is a promise that is fulfilled with an array of the mapped values
// That means we only get here after all the files have been statted and their contents read
// into memory. If you need to do more operations per file, they should be chained in the map
// callback for concurrency.
}, {concurrency: concurrency}).call("sort", function(a, b) {
    return a.fileName.localeCompare(b.fileName);
}).then(function() {
    console.timeEnd("reading files");
});