|
@@ -490,13 +490,42 @@ class BusinessBaseDetailSpider(Spider):
|
|
|
"""详情页采集业务基类"""
|
|
|
|
|
|
__business_type__ = "Detail"
|
|
|
- __custom_setting__ = dict(
|
|
|
+ __business_setting__ = dict(
|
|
|
ITEM_FILTER_ENABLE=False
|
|
|
)
|
|
|
|
|
|
err_coll_name = "listdata_err"
|
|
|
_to_db = None
|
|
|
|
|
|
+ def __init__(
|
|
|
+ self,
|
|
|
+ redis_key=None,
|
|
|
+ thread_count=None,
|
|
|
+ begin_callback=None,
|
|
|
+ end_callback=None,
|
|
|
+ delete_keys=(),
|
|
|
+ keep_alive=None,
|
|
|
+ auto_start_requests=None,
|
|
|
+ batch_interval=0,
|
|
|
+ wait_lock=True,
|
|
|
+ **kwargs
|
|
|
+ ):
|
|
|
+ self.__class__.__custom_setting__.update(
|
|
|
+ self.__class__.__business_setting__
|
|
|
+ )
|
|
|
+ super(BusinessBaseDetailSpider, self).__init__(
|
|
|
+ redis_key=redis_key,
|
|
|
+ thread_count=thread_count,
|
|
|
+ begin_callback=begin_callback,
|
|
|
+ end_callback=end_callback,
|
|
|
+ delete_keys=delete_keys,
|
|
|
+ keep_alive=keep_alive,
|
|
|
+ auto_start_requests=auto_start_requests,
|
|
|
+ batch_interval=batch_interval,
|
|
|
+ wait_lock=wait_lock,
|
|
|
+ **kwargs
|
|
|
+ )
|
|
|
+
|
|
|
def get_tasks(self, query, limit=10, is_delete=True, **kwargs):
|
|
|
"""
|
|
|
领取采集任务
|