javascript - 如何自动化获取纳斯达克历史股票价格的过程？

Question

我正在尝试通过 https://www.nasdaq.com/symbol/aapl/historical 获取 200 多个不同股票的 10 年历史每日价格的原始数据。我现在要做的是搜索每个股票代码，从“时间范围”下拉菜单中选择“10 年”，等待响应，然后单击底 Angular 的“以 Excel 格式下载此文件”。

显然，这是非常乏味且耗时的，所以我想看看是否有任何方法可以自动执行此操作。

我注意到“以 Excel 格式下载此文件”按钮调用了 JS 函数:

function getQuotes(download) {
        if (!download)
            showLoadingSpinner();

        var data = $("[id$='ddlTimeFrame']").val();
        var submitString = data + '|' + download + "|" + quoteBoxSelectedSymbol; if (!download) {
            $.ajax({
                type: "POST",
                url: baseUrl,
                data: submitString,
                contentType: "application/json",
                success: function (response) {
                    $("[id$='historicalContainer']").html(response);
                    $(".genTable tbody tr:odd").addClass("genTablealt");
                    hideLoadingSpinner();
                }
            });
        }
        else {
            $("[id$='submitString']").val(submitString);
            $("#getFile").submit();
        }
    }

下载变量似乎是一个 bool 值，当按下按钮时它会作为 true 传递。有没有办法将函数生成的任何端点注入(inject)到像 Postman 这样的东西中？我对 JavaScript 不太熟悉。

Answer 1

您至少需要 2 个请求

    1. GET request:    to get the required cookies
    2. POST request:   to change timespan/
                       -> parse response / downlaod excel

1 - initial get request [set_cookie]

    GET /symbol/aapl/historical HTTP/1.1
    Host: www.nasdaq.com
    cache-control: no-cache

Tests

<pre><code>pm.test("valid response", function () {
    pm.response.to.have.status(200);
    pm.response.to.be.withBody;
    pm.expect(pm.response.text()).to.include("Results for: ");
});
</code></pre>

<p></p>

只有下载excel时才需要以下内容

<pre><code>// ... continue test code ...
var input = pm.response.text();
var regex = /"__VIEWSTATE" value="(.*?)"/;
var match = regex.exec(input);
if (match) pm.globals.set("__VIEWSTATE", match[1]);  
var regex = /"__VIEWSTATEGENERATOR" value="(.*?)"/;
var match = regex.exec(input);
if (match) pm.globals.set("__VIEWSTATEGENERATOR", match[1]);  
var regex = /"__VIEWSTATEENCRYPTED" value="(.*?)"/;
var match = regex.exec(input);
if (match) pm.globals.set("__VIEWSTATEENCRYPTED", match[1]);  
var regex = /"__EVENTVALIDATION" value="(.*?)"/;
var match = regex.exec(input);
if (match) pm.globals.set("__EVENTVALIDATION", match[1]);
//-----
var regex = /<option value="(\d+[dmy])">10 Years<\/option>/;
var match = regex.exec(input);
if (match) var timeframe = match[1];
pm.globals.set("timeframe", timeframe);
//-----
var regex = /var quoteBoxSelectedSymbol="(.*?)";/;
var match = regex.exec(input);
if (match) var symbol = match[1];
pm.globals.set("symbol", symbol);
//-----
var submitNoDownload = timeframe + "|false|" + symbol;
pm.globals.set("submitNoDownload", submitNoDownload);
var submitToDownload = timeframe + "|true|" + symbol;
pm.globals.set("submitToDownload", submitToDownload);
</code></pre>

<p></p>

2 - 直接从页面获取数据；无需下载

    POST /symbol/aapl/historical HTTP/1.1
    Host: www.nasdaq.com
    x-requested-with: XMLHttpRequest
    content-type: application/json
    accept: */*
    cache-control: no-cache

    {{submitNoDownload}}

Tests

<pre><code>pm.test("valid response", function () {
    pm.response.to.have.status(200);
    pm.response.to.be.withBody;
    pm.expect(pm.response.text()).to.include("Results for: ");
});

var input = pm.response.text();
var regex = /<tr>\s+(?:<td>\s+(\S+)\s+<\/td>\s+)(?:<td>\s+(\S+)\s+<\/td>\s+)(?:<td>\s+(\S+)\s+<\/td>\s+)(?:<td>\s+(\S+)\s+<\/td>\s+)(?:<td>\s+(\S+)\s+<\/td>\s+)(?:<td>\s+(\S+)\s+<\/td>\s+)<\/tr>/mg;
var match = regex.exec(input);
while (match != null) {
    // Iterate thru all rows/data
    pm.globals.set("date", match[1]);  
    pm.globals.set("open", match[2]);  
    pm.globals.set("high", match[3]);  
    pm.globals.set("low", match[4]);   
    pm.globals.set("close-last", match[5]);   
    pm.globals.set("volume", match[6]);  
    match = regex.exec(input);
}
</code></pre>

<p></p>

3 - 下载 Excel

要事第一!我们这里有什么..

<pre><code>var data = $("[id$='ddlTimeFrame']").val();
var submitString = data + '|' + download + "|" + quoteBoxSelectedSymbol; 
</code></pre>

<p></p>

1. $(...) = (jquery) 当前文档
2. [id$='ddlTimeFrame'] = 选择/返回 id = "ddlTimeFrame"的 html 元素(见下文)
3. .val() = 获取所选 html 元素的值

上面的代码将选择下面的html代码

<pre><code><select id="ddlTimeFrame" name="ddlTimeFrame" onchange="getQuotes(false)">
    <option value="5d">5 Days</option>
    <option value="1m">1 Month</option>
    <option value="3m" selected="selected">3 Months</option>
    <option value="6m">6 Months</option>
    <option value="1y">1 Year</option>
    <option value="18m">18 Months</option>
    <option value="2y">2 Years</option>
    <option value="3y">3 Years</option>
    <option value="4y">4 Years</option>
    <option value="5y">5 Years</option>
    <option value="6y">6 Years</option>
    <option value="7y">7 Years</option>
    <option value="8y">8 Years</option>
    <option value="9y">9 Years</option>
    <option value="10y">10 Years</option>
</select>
</code></pre>

<p></p>

html 代码中的某处

<pre><code>var quoteBoxSelectedSymbol="AAPL";
</code></pre>

<p></p>

1. 数据 = 10y
2. 下载 = true

3. quoteBoxSelectedSymbol = AAPL

4. submitString = 10y|true|AAPL

    POST /symbol/aapl/historical HTTP/1.1
    Host: www.nasdaq.com
    x-requested-with: XMLHttpRequest
    Content-Type: application/x-www-form-urlencoded
    accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3
    cache-control: no-cache

    __VIEWSTATE:{{__VIEWSTATE}}
    __VIEWSTATEGENERATOR:{{__VIEWSTATEGENERATOR}}
    __VIEWSTATEENCRYPTED:{{__VIEWSTATEENCRYPTED}}
    __EVENTVALIDATION:{{__EVENTVALIDATION}}
    ctl00$quotes_content_left$submitString:{{submitToDownload}}