|
@@ -177,6 +177,7 @@ def by_area_crawl_data(area="", type_=0, **kwargs):
|
|
|
close_spider = True
|
|
|
else:
|
|
|
logger.info(f"{city_dict.get(int(area))}-{channel_dict.get(type_)}-第{page}页数据采集成功")
|
|
|
+ time.sleep(math.log(random.randint(100, 2400), 2))
|
|
|
break
|
|
|
|
|
|
|
|
@@ -189,7 +190,7 @@ def select_types(date: str, area: str, prov: str):
|
|
|
end_time=date,
|
|
|
max_per_page=100
|
|
|
)
|
|
|
- logger.info(f"{date}-第{province_dict.get(int(prov))}地区-{channel_dict.get(type_)}采集结束")
|
|
|
+ logger.info(f"{date}-{province_dict.get(int(prov))}地区-{channel_dict.get(type_)}采集结束")
|
|
|
|
|
|
|
|
|
def select_area(date: str):
|