node.js - 如何在API导入请求期间避免 "Allocation failed - JavaScript heap out of memory"

Question

我有一个nodejs脚本，它读取excel文件中的数据并通过“请求”将其发送到nodejs API。该脚本适用于少量数据。但是当我用 60.000 行的 Excel 进行测试时，它会因错误而中断:
fatal error :CALL_AND_RETRY_LAST 分配失败 - JavaScript 堆内存不足

function fillWithFile(file, path, header, fileConfig) { 
    let workbook = XLSX.readFile(path + '/' + file); // read file
    for (var sheetIterator = 0; sheetIterator < workbook.SheetNames.length; sheetIterator++) { // for each tab in excel file
        let worksheetJson = XLSX.utils.sheet_to_json(workbook.Sheets[workbook.SheetNames[sheetIterator]]); // transforme one tab of excel to JSON
        let config = fileConfig[workbook.SheetNames[sheetIterator]]; // get the config for the current tab

        let datasPromises = getPromises(worksheetJson, config, header);

        Promise.all(datasPromises).then(function (data) {
            var dataString = '{"data":' + JSON.stringify(data) + ',"options":{"purgeBefore":false}}';
            let options = {
                url: API_URL + '/' + config.service + 's/import',
                method: 'POST',
                headers: header,
                body: dataString
            }
            request.post(options, function (err, res, body) {
                if (err) throw err;
                console.log('Server responded with:', body);
            });
        });
    }
}

function getPromises(worksheetJson, config, header) {
        let datasPromises = [];
        let promises = [];
        for (let lineIterator = 0; lineIterator < worksheetJson.length; lineIterator++) { // for each line
            datasPromises.push(new Promise(function (resolve, reject) {
                for (let key in config.columns) { // for each column in config
                    if (config.columns[key].service !== undefined) { // if service exist we need to get id of the object in this service.
                        promises.push(new Promise(function (resolve, reject) {
                            findChildren(worksheetJson[lineIterator], config.columns[key], header)
                                .then(function (res) {
                                    resolve({ key: key, data: res.id });
                                });
                        }));
                    }
                    else {
                        promises.push(new Promise(function (resolve, reject) {
                            resolve({ key: key, data: worksheetJson[lineIterator][config.columns[key]] });
                        }));
                    }
                }
                let tmpObj = {};
                Promise.all(promises).then(function (values) {
                    for (var i = 0; i < values.length; i++) {
                        tmpObj[values[i].key] = values[i].data;
                    }
                    resolve(tmpObj);
                });
            }));
        }
        return datasPromises;
}

function findChildren(sheetData, config, header) { // get children with get request
    let dataObj = {};
    let searchParams;
    for (let key in config.columns) {
        dataObj[key] = sheetData[config.columns[key]];
        if (searchParams === undefined) searchParams = key + '=' + sheetData[config.columns[key]];
        else searchParams += '&' + key + '=' + sheetData[config.columns[key]];
    }

    var headers = {
        'Authorization': header.Authorization,
        'Accept': 'application/json, text/plain, */*',
    };
    var options = {
        url: API_URL + '/' + config.service + 's?' + searchParams,
        headers: headers
    };

    return new Promise(function (resolve, reject) {
        request(options, function (error, response, body) {
            if (!error && response.statusCode == 200) {
                try {
                        resolve(JSON.parse(body));
                } catch (e) {
                    reject(e);
                }
            }
            else {
                return reject(error);
            }
        });
    });
}

该脚本使用了大量内存，最终崩溃了...... 有人知道我该如何解决这个问题吗？ 我想，我必须找到一种方法来强制立即解决 promise ，以避免内存溢出。 感谢您的帮助。

Answer 1

我认为问题在于您无法控制并行性，并且当所有内容“同时”运行时 - 它会导致内存问题。

我尝试使用 async/await 重写代码 - 实际上所有操作现在都是串行的:

async function fillWithFile(file, path, header, fileConfig) {
  let workbook = XLSX.readFile(path + '/' + file); // read file
  for (var sheetIterator = 0; sheetIterator < workbook.SheetNames.length; sheetIterator++) { // for each tab in excel file
    const worksheetJson = XLSX.utils.sheet_to_json(workbook.Sheets[workbook.SheetNames[sheetIterator]]); // transforme one tab of excel to JSON
    const config = fileConfig[workbook.SheetNames[sheetIterator]]; // get the config for the current tab

    const data = await getData(worksheetJson, config, header);

    const dataString = '{"data":' + JSON.stringify(data) + ',"options":{"purgeBefore":false}}';
    const options = {
      url: API_URL + '/' + config.service + 's/import',
      method: 'POST',
      headers: header,
      body: dataString
    };

    await new Promise((resolve, reject) => {
      request.post(options, function (err, res, body) {
        if (err) {
          reject(err);
          return;
        }
        console.log('Server responded with:', body);
        resolve(body);
      })
    });
  }
}

async function getData(worksheetJson, config, header) {
  const data = [];
  for (let lineIterator = 0; lineIterator < worksheetJson.length; lineIterator++) { // for each line
    const values = {};
    for (let key in config.columns) { // for each column in config
      if (config.columns[key].service !== undefined) { // if service exist we need to get id of the object in this service.
        const res = await findChildren(worksheetJson[lineIterator], config.columns[key], header);
        values[key] = res.id;
      }
      else {
        values[key] = worksheetJson[lineIterator][config.columns[key]];
      }
    }
    data.push(values);    
  }
  return data;
}

function findChildren(sheetData, config, header) { // get children with get request
  let dataObj = {};
  let searchParams;
  for (let key in config.columns) {
    dataObj[key] = sheetData[config.columns[key]];
    if (searchParams === undefined) searchParams = key + '=' + sheetData[config.columns[key]];
    else searchParams += '&' + key + '=' + sheetData[config.columns[key]];
  }

  var headers = {
    'Authorization': header.Authorization,
    'Accept': 'application/json, text/plain, */*',
  };
  var options = {
    url: API_URL + '/' + config.service + 's?' + searchParams,
    headers: headers
  };

  return new Promise(function (resolve, reject) {
    request(options, function (error, response, body) {
      if (!error && response.statusCode === 200) {
        try {
          resolve(JSON.parse(body));
        } catch (e) {
          reject(e);
        }
      }
      else {
        return reject(error);
      }
    });
  });
}

您现在像这样运行导入:

fillWithFile(..args).then(() => console.log('done'))