javascript - 在 mongodb 中一次插入 2 个集合中的大数据

Question

我正在尝试在两个集合中插入大约 20,000 个大数据。我有一个数组中的路径列表，它为我提供了一个 XML 数据，我在其中过滤它并将其推送到数据库。 "results"是路径数组。

这是代码

const https = require('https');
var xpath = require('xpath')
var dom = require('xmldom').DOMParser

    results.forEach(ress => {
        const options = {
          hostname: 'abcd.com',
          port: 443,
          path: '/local/content' + ress,
          method: 'GET',
          rejectUnauthorized: false,
          requestCert: true,
          agent: false
        };
        https.request(options, (res) => {
          res.setEncoding('utf-8');
          res.on('data', (d) => {  //here i am getting XML data
            var doc = new dom().parseFromString(d);
            var nodes = xpath.select("//content", doc);
              nodes.forEach((n, i) => {
                pathvalue.push({
                   ........
                })
                projectdep.push({
                 .........
                })
              });
              dbo.collection("comp").insertMany(pathvalue, (err, post) => {
              });
              dbo.collection("pro").insertMany(projectdep, (err, post) => {
              });
          });
        }).on('error', (e) => {
          console.error(e);
        }).end();
      });

请帮我插入一个循环内循环的大数据。

Answer 1

您可能希望将请求方法 promise 为

const https = require('https');

function httpRequest(params, postData) {
    return new Promise(function(resolve, reject) {
        var req = https.request(params, function(res) {
            // reject on bad status
            if (res.statusCode < 200 || res.statusCode >= 300) {
                return reject(new Error('statusCode=' + res.statusCode));
            }
            // cumulate data
            var body = [];
            res.on('data', function(chunk) {
                body.push(chunk);
            });
            // resolve on end
            res.on('end', function() {
                try {
                    body = JSON.parse(Buffer.concat(body).toString());
                } catch(e) {
                    reject(e);
                }
                resolve(body);
            });
        });
        // reject on request error
        req.on('error', function(err) {
            reject(err);
        });

        if (postData) {
            req.write(postData);
        }
        // IMPORTANT
        req.end();
    });
}

然后创建一个可以在异步/等待中解决的 promise 列表

const xpath = require('xpath')
const DOMParser = require('xmldom').DOMParser;

const docsPromises = results.map(res => {
    const options = {
        hostname: 'abcd.com',
        port: 443,
        path: '/local/content' + res,
        method: 'GET',
        rejectUnauthorized: false,
        requestCert: true,
        agent: false
    };

    return httpRequest(options, null);
});

(async () => {
    try {
        const docs = await Promise.all(docsPromises);
        const pathvalue = []
        const projectdep = []

        docs.forEach(d => {
            const doc = new DOMParser().parseFromString(d);
            const nodes = xpath.select("//content", doc);
            nodes.forEach((node, i) => {
                pathvalue.push({ ...node })
                projectdep.push({ ...node })
            });
        })

        await dbo.collection("comp").insertMany(pathvalue);
        await dbo.collection("pro").insertMany(projectdep);
    } catch (err) {
        console.error(err)
    }
})();