旧版接口

qlm/source_qianlima_bak.py

@@ -0,0 +1,100 @@
+# coding: utf-8
+import time
+
+import requests
+
+from utils.databases import mongo_table, redis_client
+from utils.log import logger
+from utils.tools import sha1
+
+qlm = mongo_table('qlm', 'qlm_2021')
+r = redis_client()
+redis_key = "qianlima_2021"
+
+'''
+# areas  地区
+# currentPage 页码
+# numPerPage 每页的条目数
+# types
+全部
+公告 0
+预告 1
+变更 2
+中标 3
+其他 5
+'''
+PROXIES = None
+
+
+def crawl_request(url, headers):
+    """
+    公共方法，get获取url 解析json 数据
+
+    :param url: 访问的url
+    :param headers: 携带参数url
+    :return:
+    """
+    while True:
+        try:
+            get_html = requests.get(url, headers=headers, timeout=5000)
+            # 自动编码，自适应字符编码
+            get_html.encoding = get_html.apparent_encoding
+            logger.info(get_html.status_code)
+            if get_html.status_code in [403, 404, 400, 502, 302]:
+                continue
+            elif get_html.status_code in [200]:
+                return get_html
+        except requests.exceptions.ConnectTimeout:
+            logger.error("Reacquire proxy")
+        except requests.RequestException:
+            time.sleep(3)
+            continue
+
+
+def crawl_spider(area, _type, i):
+    headers = {
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+        "Accept-Encoding": "gzip, deflate",
+        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "DNT": "1",
+        "Host": "search.qianlima.com",
+        "Pragma": "no-cache",
+        "Upgrade-Insecure-Requests": "1",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36 Edg/89.0.774.68",
+        "Cookie": 'UM_distinctid=178af0c6f6f2f3-0e81be36d60604-7166786d-144000-178af0c6f70294; BAIDU_SSP_lcr=https://cn.bing.com/; guest_id=ac5769d7-b906-499d-ab85-47809ee9bc56; gr_user_id=d2cc35f6-ffa2-441b-a9ff-f836345e6f75; Hm_lvt_5dc1b78c0ab996bd6536c3a37f9ceda7=1617844534; seo_refUrl=https%3A//cn.bing.com/; seo_curUrl=www.qianlima.com; qlm_referrer=https://cn.bing.com/; delClose200811=firstGoIn; __jsluid_h=5f702d3c66f33654fc8d1f109062bb23; __jsl_clearance=1617844553.848|0|oACHKEqjLj1O5rc480L59DWlTO4%3D; CNZZDATA1277608403=736687752-1617840159-http%253A%252F%252Fsearch.qianlima.com%252F%7C1617840159; nts_login_tip=1; fromWhereUrl="http://www.qianlima.com/mfzb/"; Hm_lpvt_5dc1b78c0ab996bd6536c3a37f9ceda7=1617844615'
+    }
+    url = "http://search.qianlima.com/api/v1/website/search?filtermode=1&timeType=-1&areas={}&types={}&isfirst=false&searchMode=2&keywords=%20&currentPage={}&numPerPage=1000"
+    list_url = url.format(area, _type, i)
+    print(list_url)
+    req = crawl_request(list_url, headers)
+    info_list = req.json()["data"]["data"]
+    item_list = []
+    for info in info_list:
+        tmid = sha1(str(info["contentid"]))
+        if r.hget(redis_key, tmid) is None:
+            r.hset(redis_key, tmid, str(info["contentid"]))
+            if "popTitle" in info:
+                info["title"] = info["popTitle"]
+            else:
+                info["title"] = info["showTitle"]
+            item_list.append(info)
+    if item_list:
+        qlm.insert_many(item_list)
+    logger.info("{}--{}抓取第{}页数据，共{}条".format(area, _type, i, len(item_list)))
+
+
+def start():
+    # 遍历省份
+    for area in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 11, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]:
+        # 遍历状态
+        # [0,1,2,3,5]
+        for _type in [0, 1, 2, 3, 5]:
+            # 遍历页码
+            # for i in range(1, 11):
+            crawl_spider(area, _type, 1)
+
+
+if __name__ == '__main__':
+    start()