node.js - NodeJS : Breaking a large amount of synchronous tasks into async tasks

Question

我正在处理大量作业，然后将它们写入数据库。工作流程是这样的:

1. 将大约 100 MB 数据读取到缓冲区
2. 循环数据，然后处理(同步工作)并写入磁盘(异步工作)

我遇到的问题是，它将完成所有 100 MB 数据的循环，同时将事件循环后面的所有磁盘写入排队。因此，它将首先迭代所有数据，然后运行异步作业。

我想中断遍历数组的同步任务，以便每次迭代都在事件循环后面排队。

var lotsOfWorkToBeDone = ['tens of thousands of job', 'tens of thousands of job', 'tens of thousands of job', 'tens of thousands of job', 'tens of thousands of job', 'tens of thousands of job', 'tens of thousands of job']

while (true) {
  var job = lotsOfWorkToBeDone.pop()
  if (!job) {
    break
  }
  var syncResult = syncWork(job)
  asyncWork(syncResult)
}

function syncWork(job) {
  console.log('sync work:', job)
  return 'Sync Result of ' + job
};

function asyncWork(syncResult) {
  setTimeout(function() {
    console.log('async work: ', syncResult)
  }, 0)
}


// Desire Outcome
// sync work: tens of thousands of job
// async work: Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work: Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work: Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work: Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work: Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work: Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work: Sync Result of tens of thousands of job

// Actual Outcome
// sync work: tens of thousands of job
// sync work: tens of thousands of job
// sync work: tens of thousands of job
// sync work: tens of thousands of job
// sync work: tens of thousands of job
// sync work: tens of thousands of job
// sync work: tens of thousands of job
// async work: Sync Result of tens of thousands of job
// async work: Sync Result of tens of thousands of job
// async work: Sync Result of tens of thousands of job
// async work: Sync Result of tens of thousands of job
// async work: Sync Result of tens of thousands of job
// async work: Sync Result of tens of thousands of job
// async work: Sync Result of tens of thousands of job

注意:该示例是现实的简化版本。我没有可以迭代的数组。我有一个很大的缓冲区，我会处理直到 EOF(因此，while 循环)

Answer 1

使用 async.whilst 似乎达到了预期的结果。

我暂时不会接受自己的答案，因为我对人们对此解决方案的评论感兴趣。可能有更好的解决方案

var async = require('async')
var lotsOfWorkToBeDone = ['tens of thousands of job', 'tens of thousands of job', 'tens of thousands of job', 'tens of thousands of job', 'tens of thousands of job', 'tens of thousands of job', 'tens of thousands of job']

var job;
async.whilst(function() {
  job = lotsOfWorkToBeDone.pop()
  return job
}, function(callback) {
  var syncResult = syncWork(job)
  asyncWork(syncResult, callback)
}, function(err) {
  console.log('error: ', err)
})

function syncWork(job) {
  console.log('sync work:', job)
  return 'Sync Result of ' + job
};

function asyncWork(syncResult, callback) {
  setTimeout(function() {
    console.log('async work: ', syncResult)
    callback()
  }, 0)
}

// sync work: tens of thousands of job
// async work:  Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work:  Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work:  Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work:  Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work:  Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work:  Sync Result of tens of thousands of job
// sync work: tens of thousands of job
// async work:  Sync Result of tens of thousands of job