javascript - Node js 请求获取 ETIMEDOUT 'ip address'

Question

这就是我在代码中所做的

我正在阅读一个包含大约 3500 个链接的文本文件，然后我正在阅读每个链接，过滤我想要的链接，并发出请求以获取状态代码、链接和页面标题(使用 Cheerio)。在循环第 100 个或第 200 个链接后，我得到“connect ETIMEDOUT 40...:443”。链接看起来不错。这里发生了什么？网络服务器是否认为它是 DDOS 将我踢出局？我正在为我工​​作的公司这样做，但这显然不是本意。如果你们中的任何人想测试大量链接，我使用 https://hackertarget.com/extract-links/获取链接，然后将其放入文本文件中。

这是我的代码

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var axios = require('axios');
const fs = require('fs');
const readline = require('readline');

var main = [];
var linkdata = [];

const rl = readline.createInterface({
  input: fs.createReadStream('C:/Users/Jay/Documents/Javascript/crawl/links.txt'),
  crlfDelay: Infinity
});

rl.on('line', (link) => {
  if (link.startsWith('https://www.example.com')) {
    var encodeLink = encodeURI(link)
    request(encodeURI(encodeLink), function (error, response, body) {
      console.log("Link: ",encodeLink)
      if (error) {
        console.log("Error:Request " + error);
      }
      // Check status code (200 is HTTP OK)

      if (response.statusCode === 200) {
        // Parse the document body
        var $ = cheerio.load(body);
        var Status_200 = {
          "status Code": response.statusCode,
          "Page title:": $('title').text(),
          "Original Link": encodeLink,
        }
        main.push(Status_200)
      }

      if (response.statusCode === 302 || response.statusCode === 404 || response.statusCode === 500) {
        // Parse the document body
        var Status_Errors = {
          "status Code": response.statusCode,
          "Page title:": $('title').text(),
          "Original Link": encodeLink,
        }
        main.push(Status_Errors)

      }
      //console.log(JSON.stringify(main))
      fs.writeFile("C:/Users/Jay/Documents/Javascript/crawl/output.json", JSON.stringify(main), (err) => {
        if (err) console.log(err);
        console.log("Successfully Written to File.");
      });
    })
  }
});

Answer 1

自从使用异步以来，尝试一下 catch 来看看这是否有助于解决您遇到的内存错误，无论如何，这可能是个好习惯

try {
        const body = response.data;
        if (response.status === 200) {
          //do ur thing
        }
  
        if (response.status === 302 || response.status === 404 || response.status === 500) {
          // Parse the document body
                   //do ur thing

        }
            fs.writeFile("C:/Users/T440/Documents/crawl/output.json", JSON.stringify(main), (err) => {
            if (err) console.log(err);
            console.log("Successfully Written to File.");
          });
      } catch (error) {

                   //catch them erros

        }
        main.push(Status_ErrorsCatch)