|
@@ -1850,10 +1850,11 @@ func CreateTaskProcess() {
|
|
|
//task.FrequencyErrTimes = spider.FrequencyErrTimes
|
|
|
//lua、python共有异常publishtime、text
|
|
|
if len(spider.Error) > 0 {
|
|
|
- //1、download:下载异常errtype:5;
|
|
|
- //2、regather:运行异常errtype:4;
|
|
|
- //3、publishtime:时间异常errtype:3;
|
|
|
- //4、text:数据异常errtype:2;
|
|
|
+ //1、annex:附件异常errtype:6
|
|
|
+ //2、download:下载异常errtype:5;
|
|
|
+ //3、regather:运行异常errtype:4;
|
|
|
+ //4、publishtime:时间异常errtype:3;
|
|
|
+ //5、text:数据异常errtype:2;
|
|
|
for stype, info := range LuaErrTypeInfoMap {
|
|
|
if err := spider.Error[stype]; err != nil {
|
|
|
taskStateOk := false
|
|
@@ -1926,7 +1927,7 @@ func CreateTaskProcess() {
|
|
|
task.DescribeMap[TASK_RATEERR] = "采集频率异常:\n 列表页共采集" + fmt.Sprint(spider.ListDownloadAllTimes) + "轮,其中有" + fmt.Sprint(spider.ListOhPercentTimes) + "轮数据全采\n"
|
|
|
}
|
|
|
}
|
|
|
- //5、列表页异常 errtype:7
|
|
|
+ //7、列表页异常 errtype:7
|
|
|
listTagTime, ok := spider.TaskTags[fmt.Sprint(TASK_LISTERR)].(int64)
|
|
|
if !ok || listTagTime < util.GetTime(-7) { //无标签或者上次标记时间不在7天内的建任务
|
|
|
if spider.ListDownloadAllTimes == 0 || (spider.ListNoDataTimes > 0 && spider.ListNoDataTimes == spider.ListDownloadAllTimes) { //spider.ListDownloadAllTimes == 0针对异常不运行爬虫做判断
|