node.js - Nodejs > Gulp > through2 > 限制为 16 个文件？

Question

更新 好的，这似乎与 through2 的“highWaterMark”属性相关联。基本上，它的意思是“不要缓冲超过 x 个文件，等待有人使用它，然后才接受另一批文件”。由于它在设计上就是这样工作的，因此正在审查这个问题中的片段。必须有更好的方法来处理许多文件。

快速修复，允许 8000 个文件:

  through.obj({ highWaterMark: 8000 }, (file, enc, next) => { ... })

原始问题

我正在使用 gulp 任务来创建翻译文件。它扫描 src *.i18n.json 的文件夹文件并保存一个.json它在源文件中找到的每种语言。

它工作正常 - 直到找到超过 16 个文件。它正在使用 through2用于每个文件的处理。请参阅下面的源代码。方法processAll18nFiles()是一个自定义管道，它接收匹配的输入文件，读取每个文件的内容，动态构建结果字典，然后最终将其交给 on('finish)编写字典的处理程序。

在 Windows 和 Mac 上测试。我的方法似乎有一个限制，因为它对于 16 个或更少的文件运行良好。

仍在寻找，欢迎提供线索:-)

源文件示例:signs.i18n.json

{
  "path": "profile.signs",
  "data": {
    "title": {
      "fr": "mes signes précurseurs",
      "en": "my warning signs"
    },
    "add": {
      "fr": "ajouter un nouveau signe",
      "en": "add a new warning sign"
    }
  }
}

输出文件示例:en.json

{"profile":{"signs":{"title":"my warning signs","add":"add a new warning sign"}}}

gulpfile.js

const fs = require('fs');
const path = require('path');
const gulp = require('gulp');
const watch = require('gulp-watch');
const through = require('through2');

const searchPatternFolder = 'src/app/**/*.i18n.json';
const outputFolder = path.join('src', 'assets', 'i18n');

gulp.task('default', () => {
  console.log('Ionosphere Gulp tasks');
  console.log(' > gulp i18n         builds the i18n file.');
  console.log(' > gulp i18n:watch   watches i18n file and trigger build.');
});

gulp.task('i18n:watch', () => watch(searchPatternFolder, { ignoreInitial: false }, () => gulp.start('i18n')));
gulp.task('i18n', done => processAll18nFiles(done));

function processAll18nFiles(done) {
  const dictionary = {};
  console.log('[i18n] Rebuilding...');
  gulp
    .src(searchPatternFolder)
    .pipe(
      through.obj((file, enc, next) => {
        console.log('doing ', file.path);
        const i18n = JSON.parse(file.contents.toString('utf8'));
        composeDictionary(dictionary, i18n.data, i18n.path.split('.'));
        next(null, file);
      })
    )
    .on('finish', () => {
      const writes = [];
      Object.keys(dictionary).forEach(langKey => {
        console.log('lang key ', langKey);
        writes.push(writeDictionary(langKey, dictionary[langKey]));
      });
      Promise.all(writes)
        .then(data => done())
        .catch(err => console.log('ERROR ', err));
    });
}

function composeDictionary(dictionary, data, path) {
  Object.keys(data)
    .map(key => ({ key, data: data[key] }))
    .forEach(({ key, data }) => {
      if (isString(data)) {
        setDictionaryEntry(dictionary, key, path, data);
      } else {
        composeDictionary(dictionary, data, [...path, key]);
      }
    });
}

function isString(x) {
  return Object.prototype.toString.call(x) === '[object String]';
}

function initDictionaryEntry(key, dictionary) {
  if (!dictionary[key]) {
    dictionary[key] = {};
  }
  return dictionary[key];
}

function setDictionaryEntry(dictionary, langKey, path, data) {
  initDictionaryEntry(langKey, dictionary);
  let subDict = dictionary[langKey];
  path.forEach(subKey => {
    isLastToken = path[path.length - 1] === subKey;
    if (isLastToken) {
      subDict[subKey] = data;
    } else {
      subDict = initDictionaryEntry(subKey, subDict);
    }
  });
}

function writeDictionary(lang, data) {
  return new Promise((resolve, reject) => {
    fs.writeFile(
      path.join(outputFolder, lang + '.json'),
      JSON.stringify(data),
      'utf8',
      err => (err ? reject(err) : resolve())
    );
  });
}

Answer 1

好的，正如所解释的 here ，必须消耗管道。这是通过添加“数据”事件的处理程序来完成的，例如:

  gulp
    .src(searchPatternFolder)
    .pipe(
      through.obj({ highWaterMark: 4, objectMode: true }, (file, enc, next) => {
        const { data, path } = JSON.parse(file.contents.toString('utf8'));
        next(null, { data, path });
      })
    )
    // The next line handles the "consumption" of upstream pipings
    .on('data', ({ data, path }) => ++count && composeDictionary(dictionary, data, path.split('.')))
    .on('end', () =>
      Promise.all(Object.keys(dictionary).map(langKey => writeDictionary(langKey, dictionary[langKey])))
        .then(() => {
          console.log(`[i18n] Done, ${count} files processed, language count: ${Object.keys(dictionary).length}`);
          done();
        })
        .catch(err => console.log('ERROR ', err))
    );