node.js - 使用 Node 限制对第 3 部分 Apis 的请求

Question

所以我正在构建一个空间映射并将关系转储到一个 mongodb 中。在这个场景中，我有区域、星座和太阳系。其中 1 个区域可以有多个星座，每个星座可以有多个太阳系。我有一个允许抓取其中一些内容的 api，但它需要对每个项目进行 api 调用。提出我进行 ~6000 次 api 调用的问题。

这是第 3 方的 api 标准:

• 一般速率限制:每秒 150 个请求
• 突发大小:400
• 并发连接数:20

这是我的数据库模型:

区域模型:

var mongoose = require('mongoose');
var Schema = mongoose.Schema;

//Schema Definition
var regionSchema = new Schema({
    _id: Number,
    name: String,
    description: String,
    href: String,
    constellations: [{
        type: Schema.ObjectId,
        ref: 'Constellation'
    }]//Reference
});

//Expose (export) the model
module.exports = mongoose.model('Region', regionSchema);

星座模型:

var mongoose = require('mongoose');
var Schema = mongoose.Schema;

//Schema Definition
var constellationSchema = new Schema({
    _id: Number,
    name: String,
    href: String,
    solarSystems: [{
        type: Schema.ObjectId,
        ref: 'SolarSystem'
    }]
});

//Expose (export) the model
module.exports = mongoose.model('Constellation', constellationSchema);

太阳系模型:

var mongoose = require('mongoose');
var Schema = mongoose.Schema;

//Schema Definition
var solarSystemSchema = new Schema({
    _id: Number,
    name: String,
    imgUrl: String
});

//Expose (export) the model
module.exports = mongoose.model('SolarSystem', solarSystemSchema);

我也在尝试以正确的顺序保存它们，以便为关系填充引用。

这是我的代码:

function getAllRegions(req, res){
   getAllRegionsHrefs().then(function (hrefs){
       var newRegions = [];
       for(var href in hrefs){
           var options = {
               uri: hrefs[href],
               json: true
           };
           RequestPromise(options).then(function (responseItem){
               var constellationObjects = [];
               for(var item in responseItem.constellations){
                   var newConstellation = constellationModel({
                       _id: responseItem.constellations[item].id,
                       href: 'https://getspaceInfoHere.com/constellations/'+responseItem.constellations[item].id+'/'
                   });
                   newConstellation.save();
                   constellationObjects.push(newConstellation);
               }
               var newRegion = regionModel({
                   _id: responseItem.id,
                   name: responseItem.name,
                   description: responseItem.description,
                   href: 'https://getspaceInfoHere.com/regions/'+responseItem.id+'/',
                   constellations: constellationObjects
               });
               newRegion.save();
               newRegions.push(newRegion);
               console.log(newRegion);
           });
       }
   });
}

function getAllRegionsHrefs(){
    var options = {
        uri: universeInfoEndpoint,
        json: true
    };
   return RequestPromise(options).then(function (responseItems){
        var regionHrefs = [];
        for(var item in responseItems.items){
                regionHrefs.push(responseItems.items[item].href);
        }
        return regionHrefs;
    });
}

现在我什至没有尝试获取提供系统信息的详细星座信息(然后提供一个 href 来获取详细的系统信息)并且我遇到了我的最大值。限制这种情况的最佳方法是什么，以便我可以保持在参数范围内？

更新

function getAllRegions(req, res){
    getAllRegionsHrefs().then(function (hrefs){
        var chunks = _.chunk(hrefs, 25);
        return Promise.map(chunks, function(chunk) {
            return Promise.map(chunk, getRegion).then(function (getRegionResults){
                for(var item in getRegionResults) {
                    Promise.map(getRegionResults[item].constellations, getConstellationInfo).then(function (constellationInfo) {
                        var chunks = _.chunk(constellationInfo, 150);
                        return Promise.map(chunks, function (chunk) {
                            return Promise.map(chunk, getSystem).delay(20000);
                        })
                    }).delay(20000);
                }
            }).delay(200000);
        });
    });
}

function getSystem(systems){
    for(var updateSystem in systems){
        var options = {
            uri: systems[updateSystem].href,
            json: true
        };
         RequestPromise(options).then(function (responseItem){
             //Grab the system in the db and update it with its info
            systemModel.findOne({ _id: systems[updateSystem]._id }, function (err, doc){
                doc.name = responseItem.name;
                doc.save();
            });

        });
    }
}

function getConstellationInfo(constellation) {
    var options = {
        uri: constellation.href,
        json: true
    };
    return RequestPromise(options).then(function (responseItem){
        var arrayOfSystems = [];
        for(var system in responseItem.systems){
            var newSystem = new systemModel({
                _id: responseItem.systems[system].id,
                href: responseItem.systems[system].href
            });
            newSystem.save();
            arrayOfSystems.push(newSystem);
        }
        //find the constellation and update it with its info
        constellationModel.findOne({ _id: constellation._id }, function (err, doc){
            doc.name = responseItem.name;
            doc.solarSystems = arrayOfSystems;
            doc.save();
        });
        return arrayOfSystems;
    });
}


function getRegion(href) {
    var options = {
        uri: href,
        json: true
    };
    return RequestPromise(options).then(function (responseItem){
        var constellationObjects = [];
        for(var item in responseItem.constellations){
            var newConstellation = constellationModel({
                _id: responseItem.constellations[item].id,
                href: eveConstellationCrestEndpoint + responseItem.constellations[item].id+'/'
            });
            newConstellation.save();
            constellationObjects.push(newConstellation);
        }
        var newRegion = regionModel({
            _id: responseItem.id,
            name: responseItem.name,
            description: responseItem.description,
            href: universeEndpoint + responseItem.id+'/',
            constellations: constellationObjects
        });
        newRegion.save();
        return newRegion;
    });
}

function getAllRegionsHrefs(){
    var options = {
        uri: universeEndpoint,
        json: true
    };
   return RequestPromise(options).then(function (responseItems){
        var regionHrefs = [];
        for(var item in responseItems.items){
                regionHrefs.push(responseItems.items[item].href);
        }
        return regionHrefs;
    });
}

现在这适用于整个链(获取区域、星座和系统信息)，但超时有效并开始拒绝系统级别的连接。有什么建议吗？

Answer 1

您可以通过将 href 分成 20 个一组并在每个 block 之后设置延迟来实现此目的，您可能想要使用这些参数:

使用 lodash 的 _.chunk和 bluebird 的 Promise.delay与 Promise.map :

function getAllRegions(req, res){
   getAllRegionsHrefs().then(function (hrefs){
       var chunks = _.chunk(hrefs, 20);
       return Promise.map(chunks, function(chunk) {
         // tune the delay to what you need it to be
         // it will wait the delay (in ms) before starting the next chunk of requests
         return Promise.map(chunk, getRegion).delay(150000);
       });
   });
}

function getRegion(href) {
    var options = {
        uri: hrefs[href],
        json: true
    };
    return RequestPromise(options).then(function (responseItem){
        var constellationObjects = [];
        for(var item in responseItem.constellations){
            var newConstellation = constellationModel({
                _id: responseItem.constellations[item].id,
                href: 'https://getspaceInfoHere.com/constellations/'+responseItem.constellations[item].id+'/'
            });
            newConstellation.save();
            constellationObjects.push(newConstellation);
        }
        var newRegion = regionModel({
            _id: responseItem.id,
            name: responseItem.name,
            description: responseItem.description,
            href: 'https://getspaceInfoHere.com/regions/'+responseItem.id+'/',
            constellations: constellationObjects
        });
        newRegion.save();
        console.log(newRegion);
        return newRegion;
    });
}