|
|
@@ -28,12 +28,12 @@ from encode_info import encode_info
|
|
|
MEGABYTES = 1024 * 1024 # 1M
|
|
|
|
|
|
|
|
|
-def splicing(name, response):
|
|
|
- path = 'splicing.js'
|
|
|
+def show_html(business_name, response):
|
|
|
+ path = 'showDetails.js'
|
|
|
with open(path, encoding='utf-8') as rp:
|
|
|
js_script = rp.read()
|
|
|
ctx = execjs.compile(js_script)
|
|
|
- html = ctx.call('splicing', name, response)
|
|
|
+ html = ctx.call('getHtml', business_name, response)
|
|
|
return html
|
|
|
|
|
|
|
|
|
@@ -48,8 +48,7 @@ def get_acw_sc_v2(param):
|
|
|
|
|
|
|
|
|
def pay_proxy():
|
|
|
- tools.delay_time(2)
|
|
|
- url = 'http://tiqu.pyhttp.taolop.com/getflowip?count=1&neek=80160&type=1&sep=1&sb=&ip_si=1&mr=0'
|
|
|
+ url = 'http://tiqu.pyhttp.taolop.com/getflowip?count=1&neek=80160&type=1&sep=1&sb=&ip_si=1&mr=0&username=chukou01&spec=1'
|
|
|
response = requests.get(url)
|
|
|
proxy = response.content.decode().strip()
|
|
|
proxies = {
|
|
|
@@ -152,6 +151,10 @@ class DetailSpider(feapder.AirSpider):
|
|
|
if not upper_limit:
|
|
|
# 情况2.1：结构化数据，直接提取数据
|
|
|
resp_json = response.json
|
|
|
+ data_lst = resp_json['object'][business_keyword]
|
|
|
+ if isinstance(data_lst, list) and len(data_lst) == 0:
|
|
|
+ # 无详情数据不指定分类的名称，防止js拼接找不到分类名称而引发TypeError错误
|
|
|
+ business_keyword = ''
|
|
|
else:
|
|
|
if business_keyword == 'openBidRecord':
|
|
|
return None, DataStreamReadStatus.LOSE
|
|
|
@@ -179,7 +182,7 @@ class DetailSpider(feapder.AirSpider):
|
|
|
"object": {business_keyword: [ret]}
|
|
|
}
|
|
|
|
|
|
- html = splicing(business_keyword, resp_json)
|
|
|
+ html = show_html(business_keyword, resp_json)
|
|
|
return html, DataStreamReadStatus.NORMAL
|
|
|
|
|
|
def start_callback(self):
|
