请求参数分类

zgztb_cookie/zgzbtb_spider.py

@@ -56,8 +56,6 @@ class ListPageSpider(feapder.AirSpider):
         ]
         start_time = '2天'
         page_size = 1000
-        # today = datetime.date.today()
-        # end_day = today - datetime.timedelta(days=-3)
         for menu in task_menus:
             business_type = menu.type
             for page in range(1, 10):
@@ -70,12 +68,18 @@ class ListPageSpider(feapder.AirSpider):
                     'searchTimeStart': '',
                     'searchTimeStop': '',
                     'timeTypeParam': '',
-                    'bulletinIssnTime': start_time,
+                    'bulletinIssnTime': '',
                     'bulletinIssnTimeStart': '',
                     'bulletinIssnTimeStop': '',
                     'pageNo': page,
                     'row': page_size,
                 }
+
+                if business_type in ['开标记录', '评标公示', '中标公告']:
+                    data['timeTypeParam'] = start_time
+                else:
+                    data['bulletinIssnTime'] = start_time
+
                 meta = {
                     'msg': f'{business_type}-第{page}页',
                     'interval': 1,  # 切换代理间隔时长
@@ -110,16 +114,24 @@ class ListPageSpider(feapder.AirSpider):
 
         # 汇总消息
         data = resp_json.get("object")
-        total_page = data.get("page").get("totalPage", 0)
-        total_count = data.get("page").get("totalCount", 0)
-        page_no = data.get("page").get("pageNo", 0)
-        row = data.get("page").get("row", 0)
-        items = data.get("returnlist")
-        if page_no < total_page:
-            info = f'{menu.channel}-共{total_page}页-{total_count}条-第{page_no}页-返回{row}条-实际{len(items)}条'
-        else:
-            info = f'{menu.channel},超出最大采集页码'
-        logger.info(info)
+        return_list = data.get("returnlist")
+
+        page = data.get("page")
+        total_page = page.get("totalPage", 0)
+        total_count = page.get("totalCount", 0)
+        page_no = page.get("pageNo", 0)
+        row = page.get("row", 0)
+
+        msg = f"{menu.channel},超出最大采集页码"
+        if page_no <= total_page:
+            tips = [
+                menu.channel,
+                f'共{total_page}页{total_count}/{len(return_list)}条',
+                f'第{page_no}页{row}条',
+            ]
+            msg = "-".join(tips)
+
+        logger.info(msg)
         return True
 
     def exception_request(self, request, response, e):