设置编码转换

find_source/crawler/download.py

@@ -29,6 +29,7 @@ class Downloader:
         while retries < max_retries:
             try:
                 response = requests.get(url, **request_params)
+                response.encoding = response.apparent_encoding
                 setattr(response, 'loop_times', 0)  # 正常通信频次默认值0
                 break
             except requests.exceptions.SSLError as e:
@@ -54,6 +55,7 @@ class Downloader:
         return self._requests_by_get(url, **kw)
 
 
-# if __name__ == '__main__':
-#     r = Downloader().get('https://news.lzu.edu.cn/c/202204/88967.html')
-#     print(r, r.loop_times)
+if __name__ == '__main__':
+    # r = Downloader().get('https://news.lzu.edu.cn/c/202204/88967.html')
+    r = Downloader().get('http://www.xxls.gov.cn/')
+    print(r, r.loop_times)