爬虫任务发布时间任务类型更新

src/config.json

@@ -44,16 +44,6 @@
 		"username": "",
 		"password": ""
 	},
-	"msgservers": {
-		"comm": {
-			"addr": "spdata.jianyu360.com:801",
-			"name": "编辑器_队列节点"
-		},
-		"bid": {
-			"addr": "spdata.jianyu360.com:803",
-			"name": "编辑器_并发节点"
-		}
-	},
 	"eventsinfo": {
 		"7000": {
 			"server": "comm",

src/luatask/newtask.go

@@ -21,19 +21,20 @@ var LuaErrTypeInfo = map[string]string{
 	NEWTASK_DOWNLOADERR:  "下载异常",
 	NEWTASK_DATAINFOWARN: "数据异常警告",
 }
-var DataInfoErrMap = map[int]string{
+var DataInfoErrMap = map[int]string{ //需要建数据异常错误的类型
 	1:  "Save Coll Error",
-	2:  "File Size Or Url Error",
 	4:  "Field Value Is Null",
 	9:  "Html Contains Temp Language",
 	10: "Publishtime Is Error",
 	11: "Publishtime Is Zero",
 	12: "Field Type Error",
 }
-var DataInfoWarnMap = map[int]string{
-	5: "Field Value Contains Random Code",
-	6: "Field Value Not Contains Chinese",
-	8: "Detail File Err",
+var DataInfoWarnMap = map[int]string{ //需要建数据异常警告的类型
+	2:  "File Size Or Url Error",
+	5:  "Field Value Contains Random Code",
+	6:  "Field Value Not Contains Chinese",
+	8:  "Detail File Err",
+	10: "Publishtime Is Error", //与数据异常错误中的一致，通过判断异常数据量决定最终属于异常类型还是警告类型
 }
 
 var UpdateLuaconfig [][]map[string]interface{}
@@ -335,6 +336,7 @@ func getSpiderWarnInfo() {
 			infotype := qu.IntAll(tmp["infotype"])
 			level := qu.IntAll(tmp["level"])
 			field := qu.ObjToString(tmp["field"])
+			//参考文档：https://www.kdocs.cn/l/cbZ8u11Akkj4（异常数据统计）
 			if infotype == 3 || infotype == 7 {
 				return
 			}
@@ -832,6 +834,9 @@ func dataInfoErr(sp *NewSpider) {
 		resultDescription := ""
 		for err, _ := range DataInfoErrMap {
 			if wf := sp.WarnInfoMap[err]; wf != nil {
+				if err == 10 && wf.Fields["publishtime"] <= 1 { //发布时间异常，根据异常量划分是错误还是告警
+					continue
+				}
 				tmpDescription := ""
 				for field, href := range wf.Hrefs {
 					tmpDescription += "		字段" + field + "：" + href + "\n"
@@ -1073,6 +1078,9 @@ func dataInfoWarn(sp *NewSpider) {
 			resultDescription := ""
 			for err, _ := range DataInfoWarnMap {
 				if wf := sp.WarnInfoMap[err]; wf != nil {
+					if err == 10 && wf.Fields["publishtime"] > 1 { //发布时间异常，根据异常量划分是错误还是告警
+						continue
+					}
 					tmpDescription := ""
 					for field, href := range wf.Hrefs {
 						tmpDescription += "		字段" + field + "：" + href + "\n"

src/luatask/othertask.go

@@ -60,7 +60,7 @@ func CreateTaskByCodePendstate() {
 			set["s_platform"] = tmp["platform"]
 			set["s_channel"] = tmp["channel"]
 			set["i_event"] = tmp["event"]
-			set["s_type"] = "7"
+			set["s_type"] = "1"
 			set["i_times"] = 1
 			set["i_num"] = 0
 			set["l_comeintime"] = time.Now().Unix()

src/timetask/report.go

@@ -430,7 +430,7 @@ func SpiderWeeklyReport() {
 	bt := mw.Byte.Bytes()
 	now := time.Now()
 	name := qu.FormatDate(&now, qu.Date_Short_Layout) + "爬虫统计周报.xlsx"
-	gm.GSendMail_Bq("jy@jianyu360.cn", "zhangjinkun@topnet.net.cn,maxiaoshan@topnet.net.cn", "", "", "爬虫统计周报", "", name, bt, util.GMail)
+	gm.GSendMail_Bq("jy@jianyu360.cn", "zhangjinkun@topnet.net.cn,baixiaohu@topnet.net.cn", "", "", "爬虫统计周报", "", name, bt, util.GMail)
 }
 
 func GetCodeAndSiteInfo() (int, int) {
@@ -648,7 +648,7 @@ func SpiderWeeklyReportForLua() {
 	msg := fmt.Sprintf(MarkdownModel, text)
 	qu.Debug("msg", msg)
 	resp, err := http.Post(
-		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=d61e5bf5-bd85-4749-996a-f2ff6b7c4177",
+		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=7b082bc8-31d6-41b8-a6d9-1f731505ad0e",
 		"application/json",
 		bytes.NewBuffer([]byte(msg)),
 	)

src/util/config.go

@@ -20,8 +20,6 @@ var (
 	Es                         *es.Elastic     //三个月全量
 	EsIndex                    string          //
 	EsType                     string          //
-	CommServers                map[string]interface{}
-	BidServers                 map[string]interface{}
 	UploadEvents               map[int]string
 	RandomDataPushCron         string
 	QyworkRemindModifyuserCron string
@@ -124,9 +122,6 @@ func InitOther() {
 
 	CloseNum = qu.IntAll(Config["closenum"])
 	DayNum = qu.IntAll(Config["daynum"])
-	MsgServers := Config["msgservers"].(map[string]interface{})
-	CommServers = MsgServers["comm"].(map[string]interface{})
-	BidServers = MsgServers["bid"].(map[string]interface{})
 	//event信息
 	eventsinfo := Config["eventsinfo"].(map[string]interface{})
 	UploadEvents = map[int]string{}