|
|
@@ -183,7 +183,13 @@ def crawl_spider(area: str, types: int, page: int, **kwargs):
|
|
|
qlm.insert_many(results)
|
|
|
|
|
|
if request_status in ['stop', 'success']:
|
|
|
- logger.info("第{}区-第{}类-第{}页成功上传{}条数据".format(area, types, page, len(results)))
|
|
|
+ logger.info("第{}区-{}-第{}类-第{}页成功上传{}条数据".format(
|
|
|
+ area,
|
|
|
+ begin_time,
|
|
|
+ types,
|
|
|
+ page,
|
|
|
+ len(results))
|
|
|
+ )
|
|
|
return request_status
|
|
|
|
|
|
|
|
|
@@ -227,13 +233,13 @@ def select_types(date: str, area: str):
|
|
|
end_time=date,
|
|
|
max_page=1000
|
|
|
)
|
|
|
- logger.info(f"第{area}区-第{types}类采集任务结束")
|
|
|
+ logger.info(f"第{area}区-{date}-第{types}类采集任务结束")
|
|
|
|
|
|
|
|
|
def select_area(date: str):
|
|
|
for area in range(1, 32):
|
|
|
select_types(date, str(area))
|
|
|
- logger.info("任务结束")
|
|
|
+ logger.info(f"{date} - 任务结束")
|
|
|
|
|
|
|
|
|
def history(date_lst: list):
|
