node.js - NodeJS、MSSQL 批量从 API 获取并保存天气观测结果

Question

我正在尝试编写一个 Nodejs 文件来循环访问气象局 JSON 文件中的 700 个天气报告。每个文件包含过去 24 小时针对特定天气站点的大量天气观测数据:空气温度、阵风公里小时等。对于每个文件，我想找到每个观测值的最小/最大/平均平均值(air_temp_min、air_temp_max、air_temp_avg) )。然后我想将其提交到 MSSQL 数据库。我计划每天执行一次，因此每个位置观察都将作为单行输入到具有多列的数据库中。

我对异步操作很陌生，并且在将一个操作/函数/ promise 链接到下一个操作/函数/ promise 或超过最大监听器时总是遇到麻烦。

所以我需要关于如何优雅而有效地做到这一点的建议或指导。我不介意使用哪些包，我尝试过 request、axios、mssql，很乏味。我可以让他们在一条线上单独工作，但不能以和谐的方式一起工作。

起点是迭代每个站点的 URL。这是一个示例。

http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95767.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95772.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94650.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95774.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95754.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95747.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95779.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94774.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94781.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95770.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95758.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.99738.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94775.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94776.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94773.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95773.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94588.json

提前致谢。

Answer 1

因此，您可以使用模块 request (+request-promise-native)、mssql，然后使用 lodash(用于聚合，例如最大值、最小值、平均值)和 moment 进行日期解析。这里有一些示例代码，可以满足您的需求。

您需要运行下面的命令行来安装依赖项:

npm install mssql request request-promise-native lodash moment --save  

Now as regards scheduling the calls, I'd suggest maybe just setting up a scheduled task to call the script once a day (depending on your OS.)

Some ToDos.. maybe look at ensuring we're closing connections nicely etc and some exception handling in case something chokes!

After running this, I see the rows below in the db:

Weather_data

timestamp               |location_name                  |air_temp_max |air_temp_min |air_temp_avg
-----------------------------------------------------------------------------------------------------
2019-03-11 00:00:00.000 |Lake Macquarie - Cooranbong    |32.4         |17.8         |23.1027777777778
2019-03-11 00:00:00.000 |Maitland Airport               |33.9         |19.9         |24.4478260869565
2019-03-11 00:00:00.000 |Mangrove Mountain              |30           |18.1         |22.3784722222222
2019-03-11 00:00:00.000 |Merriwa                        |34.8         |18.6         |25.4435897435898
2019-03-11 00:00:00.000 |Murrurundi Gap                 |31.4         |17.4         |23.3627906976744
2019-03-11 00:00:00.000 |Nelson Bay                     |26.4         |22.5         |24.7
2019-03-11 00:00:00.000 |Newcastle Nobbys               |27.7         |21.1         |23.1243055555556
2019-03-11 00:00:00.000 |Newcastle University           |26.8         |21.6         |24.2
2019-03-11 00:00:00.000 |Norah Head                     |29.5         |20.6         |23.3048611111111
2019-03-11 00:00:00.000 |Scone Airport                  |35.8         |20.4         |26.0236363636363
2019-03-11 00:00:00.000 |Singleton (Defence)            |34.5         |20           |25.1141891891892
2019-03-11 00:00:00.000 |Tocal                          |33.6         |20           |24.7993055555556
2019-03-11 00:00:00.000 |Williamtown                    |31.5         |20.7         |23.6551724137931
2019-03-11 00:00:00.000 |Armidale                       |20.7         |19.8         |20.3
2019-03-11 00:00:00.000 |Armidale Airport               |29.7         |13.9         |22.0801282051282
2019-03-11 00:00:00.000 |Glen Innes Airport             |29.7         |15.1         |21.8228187919463

天气架构 SQL

CREATE TABLE weather_data (
    timestamp datetime,
    location_name nvarchar(200),
    air_temp_max float,
    air_temp_min float,
    air_temp_avg float,
);

Node 代码:

index.js

const rp = require('request-promise-native');
const _ = require('lodash');
const sql = require('mssql');
const moment = require('moment');

const config = {
    user: 'db_user',  // Change as appropriate
    password: 'pass',  // Change as appropriate
    server: 'localhost', // Change as appropriate
    database: 'Weather',
    port: 1433 // Default port, change as appropriate
}

let sqlConnPool = null;

const urlList = [
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95767.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95772.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94650.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95774.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95754.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95747.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95779.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94774.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94781.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95770.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95758.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.99738.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94775.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94776.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94773.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95773.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94588.json'
];

async function downloadStationData() {
    for(url of urlList)  {
        let res = await downloadAndSaveWeatherData(url);
        console.log("Res: ", res);
    }
}

sql.connect(config).then(pool => {
    console.log("Connected to database...");
    sqlConnPool = pool;
    downloadStationData()
});

async function downloadAndSaveWeatherData(url) {
    console.log("downloadAndSaveWeatherData: Downloading from: ", url);
    let weatherData = await downloadWeatherData(url);
    if (weatherData.observations && weatherData.observations.data && weatherData.observations.data.length > 0) {
        console.log("downloadAndSaveWeatherData: Observations (count): ", weatherData.observations.data.length);
        let processedData = processWeatherData(weatherData.observations.data);
        console.log("downloadAndSaveWeatherData: processed data: ", processedData);
        let writeResult = await writeWeatherDataToDatabase(processedData);
        return { status: "OK"};
    }
}

// Get max, min, average etc.
function processWeatherData(data) {
    const result = { 
        timestamp: moment(data[0].aifstime_utc.substring(0,8)).toDate(),
        location_name: data[0].name 
    };

    const fields = ["air_temp"];

    fields.forEach(field => {
        result[`${field}_max`] = _.max(data.map(row => row[field])),
        result[`${field}_min`] = _.min(data.map(row => row[field])),
        result[`${field}_avg`] = _.mean(data.map(row => row[field]))
    });

    return result;
}

function downloadWeatherData(url) {
    const options = { url: url, json: true, resolveWithFullResponse: true }
    return rp(options).then((response) => response.body);
}

function writeWeatherDataToDatabase(processedData) {
    const request = sqlConnPool.request()
    return new Promise((resolve, reject) => {
        request.input('timestamp', sql.DateTime, processedData.timestamp);
        request.input('location_name', sql.NVarChar, processedData.location_name);
        request.input('air_temp_max', sql.Float, processedData.air_temp_max);
        request.input('air_temp_min', sql.Float, processedData.air_temp_min);
        request.input('air_temp_avg', sql.Float, processedData.air_temp_avg);
        request.query('insert into weather_data (timestamp, location_name, air_temp_max, air_temp_min, air_temp_avg) values (@timestamp, @location_name, @air_temp_max, @air_temp_min, @air_temp_avg)',
        (err, result) => {
            if (err) {
                console.error("writeWeatherDataToDatabase: Error occurred: ", err);
                reject(err);
            } else {
                resolve(result);
            }
        });
    });
}