我正在尝试为每个项目调用 parse_page2 方法。但是每次我运行这个蜘蛛时，我每页只能获取单个项目，所以我如何为每个项目调用 parse_page2 方法。
from scrapy.http import Request
from eScraper.items import EscraperItem
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.spiders import CrawlSpider
#------------------------------------------------------------------------------
class ESpider(CrawlSpider):
name = "atisundarSpider"
allowed_domains = ["atisundar.com"]
URLSList = []
for n in range (1,20):
URLSList.append('http://atisundar.com/collections/sarees?page=' + str(n))
URLSList.append('http://atisundar.com/collections/salwar-suits?page=' + str(n))
start_urls = URLSList
def parse(self, response):
item = EscraperItem()
hxs = HtmlXPathSelector(response)
sites = hxs.select('//div[@class="block product size-medium"]')
items = []
for site in sites:
item = EscraperItem()
item['productDesc'] = ""
item['productSite'] = ["http://atisundar.com/"]
item['productTitle'] = site.select('.//div[@class="main"]/a/@title').extract()
item['productURL'] = ["http://atisundar.com" + site.select('.//div[@class="main"]/a/@href').extract()[0].encode('utf-8')]
item['productPrice'] = site.select('.//p[@class="pricearea"]//span[@class="was-price"]/text()').extract() + site.select('.//p[@class="pricearea"]//span[@class="price"]/text()').extract()
item['productImage'] = [site.select('.//div[@class="main"]/a/img/@src').extract()[0].split('?')[0]] + [site.select('.//div[@class="main"]/a/img/@src').extract()[0].split('?')[0].replace("medium","grande")]
item['image_urls'] = item['productImage']
items.append(item)
secondURL = "http://admin.atisundar.com/store/skuDetails?product_id=" + site.select('.//div[@class="main"]/a/text()').extract()[1].strip().split("#")[-1]
request = Request(secondURL,
callback=self.parse_page2)
request.meta['item'] = item
return request
def parse_page2(self, response):
item = response.meta['item']
#item['other_url'] = response.url
return item
最佳答案
1) 您没有使用 CrawlSpider 功能，我建议您从 BaseSpider 继承您的蜘蛛
2) 在for循环中
for site in sites:
使用 yield 而不是 return ，否则它将在第一次迭代中打破循环。
yield request
3) 在parse_page2中从response.request.meta获取item，而不是从response.meta
item = response.request.meta['item']
现在应该可以工作了。
from scrapy.http import Request
from eScraper.items import EscraperItem
from scrapy.selector import HtmlXPathSelector
#------------------------------------------------------------------------------
from scrapy.spider import BaseSpider
class ESpider(BaseSpider):
name = "atisundarSpider"
allowed_domains = ["atisundar.com"]
URLSList = []
for n in range (1,20):
URLSList.append('http://atisundar.com/collections/sarees?page=' + str(n))
URLSList.append('http://atisundar.com/collections/salwar-suits?page=' + str(n))
start_urls = URLSList
def parse(self, response):
item = EscraperItem()
hxs = HtmlXPathSelector(response)
sites = hxs.select('//div[@class="block product size-medium"]')
for site in sites:
item = EscraperItem()
item['productDesc'] = ""
item['productSite'] = ["http://atisundar.com/"]
item['productTitle'] = site.select('.//div[@class="main"]/a/@title').extract()
item['productURL'] = ["http://atisundar.com" + site.select('.//div[@class="main"]/a/@href').extract()[0].encode('utf-8')]
item['productPrice'] = site.select('.//p[@class="pricearea"]//span[@class="was-price"]/text()').extract() + site.select('.//p[@class="pricearea"]//span[@class="price"]/text()').extract()
item['productImage'] = [site.select('.//div[@class="main"]/a/img/@src').extract()[0].split('?')[0]] + [site.select('.//div[@class="main"]/a/img/@src').extract()[0].split('?')[0].replace("medium","grande")]
item['image_urls'] = item['productImage']
secondURL = "http://admin.atisundar.com/store/skuDetails?product_id=" + site.select('.//div[@class="main"]/a/text()').extract()[1].strip().split("#")[-1]
request = Request(secondURL,
callback=self.parse_page2)
request.meta['item'] = item
yield request
def parse_page2(self, response):
item = response.request.meta['item']
#item['other_url'] = response.url
return item
关于python - 如何为每个项目调用 Parse_page2 方法，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/16791427/