import scrapy
from scrapy.crawler import CrawlerProcess
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.proxy import Proxy, ProxyType
class FooSpider(scrapy.Spider):
name = 'foo'
start_urls = ["https://www.whatismybrowser.com/"]
index=1
def __init__(self, *args, **kwargs):
super(FooSpider, self).__init__(*args, **kwargs)
self.download_delay = 0.25
chrome_options = Options() # Initializing Chrome
#chrome_options.add_argument("--headless")
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--ignore-ssl-errors')
IP = '176.31.69.183' # random free proxy from net
PORT = 8080
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = f'{IP}:{PORT}'
prox.socks_proxy = f'{IP}:{PORT}'
prox.ssl_proxy = f'{IP}:{PORT}'
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
self.browser = webdriver.Chrome(executable_path="/home/timmy/Downloads/chromedriver",options=chrome_options, desired_capabilities=capabilities)
#self.browser.implicitly_wait(60) #
def parse(self,response):
self.browser.get(response.url)
data= self.random_data()
print(data)
process = CrawlerProcess({'LOG_LEVEL':'INFO',})
process.crawl(FooSpider)
spider = next(iter(process.crawlers)).spider
process.start()
这是我遇到的错误
引发异常类(消息、屏幕、堆栈跟踪)
selenium.common.exceptions.InvalidArgumentException:消息:无效参数:无法解析功能:代理
来自无效参数:指定“socksProxy”需要“socksVersion”的整数
我从SSlproxies获得代理我正在尝试使用它,我正在使用这个问题的答案 running-selenium-webdriver-with-a-proxy-in-python但我收到了上面的错误
我该如何解决?
最佳答案
您正在使用旧的 Chrome 代理语法。这是设置无需身份验证的代理服务器的方法。
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument('--proxy-server=http://ipaddress:port')
driver = webdriver.Chrome(chrome_options=chrome_options)
关于python - 将 selenium chrome 与非本地代理一起使用,我们在Stack Overflow上找到一个类似的问题: https://stackoverflow.com/questions/56683265/