我需要一些帮助来正确构建代码以使用 S3 存储桶和 Lambda 函数处理一些文本文件。
我想使用由在 S3 存储桶中创建新对象触发的 Lambda 函数来读取文件并提取一些数据并将其写入放置在另一个 S3 存储桶中的文件。
到目前为止,我的函数可以很好地将文件从一个 S3 存储桶复制到另一个存储桶,但我不太清楚如何添加一个函数来处理文件并将结果写出到最终的 S3 目的地。
这些文件是简单的文本文件,我需要从文件中的每一行中提取数据。
如果我当前使用的 Node.js 代码添加了一个附加函数来处理文件,请参见下面的注释 - 请参阅带有 ?? 的注释我在哪里寻求帮助。
// dependencies
var async = require('async');
var AWS = require('aws-sdk');
var util = require('util');
// get reference to S3 client
var s3 = new AWS.S3();
exports.handler = function(event, context) {
// Read options from the event.
console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
var srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
var srcKey =
decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
var dstBucket = "inputBucket";
var dstKey = srcKey + ".txt";
// Sanity check: validate that source and destination are different buckets.
if (srcBucket == dstBucket) {
console.error("Destination bucket must not match source bucket.");
return;
}
// Infer the file type.
var typeMatch = srcKey.match(/\.([^.]*)$/);
if (!typeMatch) {
console.error('unable to infer file type for key ' + srcKey);
return;
}
var imageType = typeMatch[1];
if (imageType != "txt") {
console.log('skipping non-image ' + srcKey);
return;
}
// Download the image from S3, transform, and upload to a different S3 bucket.
async.waterfall([
function download(next) {
// Download the file from S3 into a buffer.
s3.getObject({
Bucket: srcBucket,
Key: srcKey
},
next);
},
function transform(response, next) {
// Read the file we have just downloaded
// ? response.Body ?
var rl = require('readline').createInterface({
input: require('fs').createReadStream('file.in')
});
// Process each line here writing the result to an output buffer?
rl.on('line', function (line) {
console.log('Line from file:', line);
//Do something with the line...
//Create some output string 'outputline'
//Write 'outputline' to an output buffer 'outbuff'
// ??
});
// Now pass the output buffer to the next function
// so it can be uploaded to another S3 bucket
// ??
next;
}
function upload(response, next) {
// Stream the file to a different S3 bucket.
s3.putObject({
Bucket: dstBucket,
Key: dstKey,
Body: response.Body,
ContentType: response.contentType
},
next);
}
], function (err) {
if (err) {
console.error(
'Unable to process ' + srcBucket + '/' + srcKey +
' and upload to ' + dstBucket + '/' + dstKey +
' due to an error: ' + err
);
} else {
console.log(
'Successfully processed ' + srcBucket + '/' + srcKey +
' and uploaded to ' + dstBucket + '/' + dstKey
);
}
context.done();
}
);
};
最佳答案
s3.getObject的回调内部
s3.getObject(params,function(err,data){})
如果您的文件是文本,那么您可以将文本提取为字符串
data.Body.toString("utf-8")
关于node.js - 如何从 Lambda 函数解析 AWS S3 文件,我们在Stack Overflow上找到一个类似的问题: https://stackoverflow.com/questions/34132526/