更新：apparent_encoding 到非 'utf-8' 编码强制使用 'utf-8'编码

find_source/crawler/download.py

@@ -3,6 +3,7 @@ import urllib3
 from requests.models import Response
 
 from config.load import headers
+from settings import SPECIAL_ENCODINGS
 
 urllib3.disable_warnings()
 
@@ -41,6 +42,9 @@ class Downloader:
         while retries < self.max_retries:
             try:
                 response = requests.get(url, **request_params)
+                response.encoding = response.apparent_encoding
+                if response.encoding in SPECIAL_ENCODINGS:
+                    response.encoding = 'utf-8'
                 break
             except requests.exceptions.SSLError as e:
                 response.reason = e.__class__.__name__