python - 网页抓取 - 进入第 2 页

Question

如何进入数据集的第二页？无论我做什么，它只返回第 1 页。

import bs4
from urllib.request import urlopen as uReq
from bs4 import BeautifulSoup as soup

myURL = 'https://jobs.collinsaerospace.com/search-jobs/'

uClient = uReq(myURL)
page_html = uClient.read()
uClient.close()

page_soup = soup(page_html, "html.parser")
container = page_soup.findAll("section", {"id":"search-results"}, {"data-current-page":"4"})


for child in container:
    for heading in child.find_all('h2'):
        print(heading.text)

Answer 1

该站点实际上使用 JSON 返回包含所有条目的 HTML。此 API 允许指定页码以及每页返回的记录数，增加此值将进一步提高速度。

返回的 JSON 包含 3 个键。过滤信息、结果 HTML 和指示是否返回工作的标志。当您到达页面末尾时，最后一个条目可用于发出信号。

您可能想看看非常流行的 Python requests 库，它可以简化为您生成正确 URL 的过程，而且速度也很快。

import bs4
import requests
from bs4 import BeautifulSoup as soup

params = {            
    "CurrentPage" : 1,
    "RecordsPerPage" : 100,
    "SearchResultsModuleName" : "Search Results",
    "SearchFiltersModuleName" : "Search Filters",
    "SearchType" : 5,
}

myURL = 'https://jobs.collinsaerospace.com/search-jobs/results'
page = 1
more_jobs = True

while more_jobs:
    print(f"\nPage {page}")
    params['CurrentPage'] = page
    req = requests.get(myURL, params=params)
    json = req.json()
    page_soup = soup(json['results'], "html.parser")
    container = page_soup.findAll("section", {"id":"search-results"}, {"data-current-page":"4"})

    for child in container:
        for heading in child.find_all('h2'):
            print(heading.text)

    more_jobs = json['hasJobs']  # Did this return any jobs?
    page += 1