|
@@ -124,16 +124,16 @@ class Scheduler:
|
|
@property
|
|
@property
|
|
def crawl_task(self):
|
|
def crawl_task(self):
|
|
results = {}
|
|
results = {}
|
|
- url = 'http://cc.spdata.jianyu360.com/crawl/zbytb/task/fetch'
|
|
|
|
|
|
+ url = 'http://cc.spdata.jianyu360.com/task/zbytb/fetch'
|
|
try:
|
|
try:
|
|
response = requests.get(url, timeout=10)
|
|
response = requests.get(url, timeout=10)
|
|
- if response.status_code == 200:
|
|
|
|
- data = response.json()['data']
|
|
|
|
- if len(data) > 0:
|
|
|
|
- results['_id'] = object_id(data['_id'])
|
|
|
|
- for key, val in data.items():
|
|
|
|
- if key != '_id':
|
|
|
|
- results[key] = val
|
|
|
|
|
|
+ response.raise_for_status()
|
|
|
|
+ data = response.json()['data']
|
|
|
|
+ if len(data) > 0:
|
|
|
|
+ results['_id'] = object_id(data['_id'])
|
|
|
|
+ for key, val in data.items():
|
|
|
|
+ if key != '_id':
|
|
|
|
+ results[key] = val
|
|
return results
|
|
return results
|
|
except requests.RequestException:
|
|
except requests.RequestException:
|
|
return results
|
|
return results
|