python - Scrapy只爬取一页

Question

我做了一个scrapy蜘蛛，我想爬行所有页面，但它只爬到第二页然后停止。 似乎在 if next_page: 循环中，url 仅更改为第二页，然后粘在那里。我认为我误解了 http 响应的工作原理，因为它似乎只抓取起始页面上的下一页链接。

import scrapy

from tutorial.items import TriniCarsItem

class TCS(scrapy.Spider):
    name = "TCS"
    allowed_domains = ["TCS.com"]
    start_urls = [
        "http://www.TCS.com/database/featuredcarsList.php"]

    def parse(self, response):
        for href in response.css("table > tr > td > a::attr('href')"):
            url = response.urljoin(href.extract())
            yield(scrapy.Request(url, callback=self.parse_dir_contents))
        next_page = response.css("body > table > tr > td > font > b > a::attr('href')")
        if next_page:
            url = response.urljoin(next_page[0].extract())
            print("THIS IS THE URL =----------------------------- " + url)
            yield(scrapy.Request(url, self.parse))

    def parse_dir_contents(self, response):
        for sel in response.xpath('//table[@width="543"]/tr/td/table/tr/td[2]/table'):
            item = TCSItem()
            item['id'] = sel.xpath('tr[1]/td[1]//text()').extract()
            item['make'] = sel.xpath('tr[3]/td[2]//text()').extract()
            item['model'] = sel.xpath('tr[4]/td[2]//text()').extract()
            item['year'] = sel.xpath('tr[5]/td[2]//text()').extract()
            item['colour'] = sel.xpath('tr[6]/td[2]//text()').extract()
            item['engine_size'] = sel.xpath('tr[7]/td[2]//text()').extract()
            item['mileage'] = sel.xpath('tr[8]/td[2]//text()').extract()
            item['transmission'] = sel.xpath('tr[9]/td[2]//text()').extract()
            item['features'] = sel.xpath('tr[11]/td[2]//text()').extract()
            item['additional_info'] = sel.xpath('tr[12]/td[2]//text()').extract()
            item['contact_name'] = sel.xpath('tr[14]/td[2]//text()').extract()
            item['contact_phone'] = sel.xpath('tr[15]/td[2]//text()').extract()
            item['contact_email'] = sel.xpath('tr[16]/td[2]//text()').extract()
            item['asking_price'] = sel.xpath('tr[17]/td[2]//text()').extract()
            item['date_added'] = sel.xpath('tr[19]/td[2]//text()').extract()
            item['page_views'] = sel.xpath('tr[20]/td[2]//text()').extract()
            #print(make, model, year, colour, engine_size, mileage, transmission, features, 
            #additional_info, contact_name, contact_phone, contact_email, asking_price, date_added, 
            #page_views)
            yield(item)

Answer 1

在第二页上，第一个链接(您选择的链接)是指向上一页的链接。只需按顺序发送任何链接，然后让重复数据删除器消除任何重复项即可:

    if next_page:
        for i in next_page
            url = response.urljoin(i.extract())
            print("THIS IS THE URL =----------------------------- " + url)
            yield(scrapy.Request(url, self.parse))

附注在您的情况下，还请考虑更容易且更大规模并行的方式:

start_urls = [
    "http://www.trinicarsforsale.com/database/featuredcarsList.php?page=%d" % i for i in xrange(1, 460)]

def parse(self, response):
    return self.parse_dir_contents(response):