python - Scrapy Spider 不遵循使用yield 的请求回调

Question

我是 scrapy 的新手，我无法让我的蜘蛛在下面的代码中输入 parse_votes，即使我将其设置为回调。其他解析方法工作正常，我没有收到任何错误并检查了具有正确信息的“链接”变量。帮忙吗？

编辑 - 完整代码

class DeputadosSpider(scrapy.Spider):
    name = "deputies"

    allowed_domains = ["camara.leg.br"]
    start_urls = ["http://www2.camara.leg.br/deputados/pesquisa"]

    def parse(self, response):
        sel = Selector(response)
        sel_options = sel.xpath('//*[@id="deputado"]/option[position()>1]')
        iteration = 1
        # get deputies pages
        for sel_option in sel_options:
            item = DeputiesInfo()           
            item["war_name"] = sel_option.xpath("text()").extract()
            item["link_id"] = sel_option.extract().partition('?')[-1].rpartition('"')[0]
            item["page_link"] = 'http://www.camara.leg.br/internet/Deputado/dep_Detalhe.asp?id=' + item["link_id"]
            item["id"] = iteration
            iteration += 1
            # go scrap their page
            yield scrapy.Request(item["page_link"], callback=self.parse_deputy, meta={'item': item})

    def parse_deputy(self, response):
        item = response.meta['item']
        sel = Selector(response)
        info = sel.xpath('//div[@id="content"]/div/div[1]/ul/li')
        # end to fill the data
        item["full_name"] = info.xpath("text()").extract_first()
        item["party"] = info.xpath("text()").extract()[2].partition('/')[0]
        item["uf"] = info.xpath("text()").extract()[2].partition('/')[-1].rpartition('/')[0]
        item["legislatures"] = info.xpath("text()").extract()[5]
        item["picture"] = sel.xpath('//div[@id="content"]/div/div[1]//img[1]/@src').extract()
        # save data to json file 
        file = open('deputies_info.json', 'a')
        line = json.dumps(dict(item)) + ",\n"
        file.write(line)
        # colect votes info
        get_years = sel.xpath('//*[@id="my-informations"]/div[3]/div/ul/li[1]/a[position()<4]')
        for get_year in get_years:
            vote = VotesInfo()
            vote["deputy_id"] = item["id"]
            vote["year"] = get_year.xpath("text()").extract_first()
            link = get_year.xpath("@href").extract_first()
            print(vote["year"])
            print(link)
            # go to voting pages
            yield scrapy.Request(link, callback=self.parse_votes, meta={'vote': vote})

    def parse_votes(self, response):
        #vote = response.meta['vote']
        print('YYYYYYYYYYYYYUHUL IM IN!!')

Answer 1

您的问题是allowed_domains，因为您尝试在parse_deputy中请求的链接例如是:http://www.camara.gov.br/internet/deputado/RelVotacoes.asp?nuLegislatura=55&nuMatricula=410&dtInicio=01/01/2016&dtFim=30/12/2016 它的域名是 camara.gov.br，因此将其添加到 allowed_domains。

allowed_domains = ["camara.leg.br", "camara.gov.br"]

PS:我运行了您的代码注释allowed_domains，并且parse_votes工作完美。