|
|
@@ -30,7 +30,6 @@ class CrawlListPageSpider:
|
|
|
self.host = 'https://www.zbytb.com/search'
|
|
|
self.headers = kwargs.get('headers') or headers
|
|
|
self.proxy = Proxy(enable_proxy)
|
|
|
- self.proxies = self.proxy.proxies
|
|
|
self.allow_show_exception = kwargs.get('allow_show_exception', False)
|
|
|
|
|
|
def crawl_request(self, url, **kwargs):
|
|
|
@@ -39,11 +38,10 @@ class CrawlListPageSpider:
|
|
|
success, response = http_request_get(
|
|
|
url,
|
|
|
headers=kwargs.get('headers'),
|
|
|
- proxies=self.proxies,
|
|
|
+ proxies=self.proxy.proxies,
|
|
|
)
|
|
|
if not success and response.status_code == 10000 and retries < 3:
|
|
|
self.proxy.switch()
|
|
|
- self.proxies = self.proxy.proxies
|
|
|
retries += 1
|
|
|
else:
|
|
|
break
|
|
|
@@ -162,3 +160,4 @@ class CrawlListPageSpider:
|
|
|
future.add_done_callback(_callback_err)
|
|
|
futures.append(future)
|
|
|
wait(futures)
|
|
|
+ logger.info("任务结束")
|
