功能修改

src/front/claim.go

@@ -14,6 +14,8 @@ var (
 	ClaimMaxNum        = 10 //已认领爬虫上限
 	ClaimLock          = &sync.Mutex{}
 	ClaimPriorityLimit = 350 //区分外包和内部人员认领爬虫的优先级界限
+	ClaimPriorityOut   = 200
+	ClaimPriorityIn    = 700
 	ClaimQueryFields   = map[string]interface{}{
 		"site":            1,
 		"code":            1,
@@ -81,7 +83,7 @@ func (f *Front) ClaimCode() {
 		q["priority"] = map[string]interface{}{
 			"$gte": ClaimPriorityLimit,
 		}
-	} else { //外包指定优先级范围且简单的爬虫（若爬虫导入同站点爬虫有重点和非重点的爬虫，会被外包认领）
+	} else { //外包指定优先级范围且简单的爬虫
 		q["priority"] = map[string]interface{}{
 			"$gt": 0,
 			"$lt": ClaimPriorityLimit,
@@ -145,6 +147,8 @@ func (f *Front) ReturnCode() {
 					"claimtype":   CLAIMTYPEUNCLAIMED,
 					"claimtime":   int64(0),
 					"recovertime": int64(0),
+					"priority":    ClaimPriorityIn,
+					"state":       0,
 					"grade":       1, //外包归还爬虫后，爬虫难易度更新为困难
 				}},
 			}

src/front/front.go

@@ -98,7 +98,8 @@ var spinfos sync.Map = sync.Map{}
 var SessMap map[string]*httpsession.Session
 var AutoTpl map[string]interface{}
 
-// var Mails *util.Mail
+var Mails *util.Mail
+
 // var Reg = regexp.MustCompile(`(http|https)://([\w]+\.)+[\w]+`)
 var ProjectHrefReg = regexp.MustCompile("projecthref")
 var Transfercode map[string]interface{}
@@ -438,7 +439,7 @@ func saveLua(o map[string]interface{}) bool {
 		} else {
 			param["recovertime"] = int64(0)       //回收时间
 			param["claimtime"] = int64(0)         //认领时间
-			param["claimtype"] = CLAIMTYPEHISTORY //爬虫认领状态
+			param["claimtype"] = CLAIMTYPEHISTORY //爬虫认领状态（历史爬虫）
 		}
 	} else { //未指定人
 		param["recovertime"] = int64(0)         //回收时间
@@ -711,6 +712,7 @@ func (f *Front) Assign() {
 	codesarr := strings.Split(codes, ",") //被分配的爬虫
 	user, _ := u.MgoEB.FindById("user", userid, nil)
 	if user != nil && len(*user) > 0 {
+		platform := (*user)["s_platform"]
 		name := qu.ObjToString((*user)["s_name"])
 		query := map[string]interface{}{
 			"code": map[string]interface{}{
@@ -726,7 +728,8 @@ func (f *Front) Assign() {
 			claimtype := qu.IntAll(l["claimtype"])
 			priority := qu.IntAll(l["priority"])
 			spiderimportant, _ := l["spiderimportant"].(bool)
-			if claimtype <= CLAIMTYPECLAIMED {
+
+			if claimtype <= CLAIMTYPECLAIMED { //非历史爬虫分配
 				recovertime := CreateRecovertime(spiderimportant, priority)
 				set = map[string]interface{}{
 					"claimtype":       CLAIMTYPECLAIMED,
@@ -737,9 +740,9 @@ func (f *Front) Assign() {
 					"createuserid":    userid,
 					"modifyuser":      name,
 					"modifyuserid":    userid,
-					"platform":        (*user)["s_platform"],
+					"platform":        platform,
 				}
-				if claimtype == CLAIMTYPECLAIMED {
+				if claimtype == CLAIMTYPECLAIMED { //已认领的爬虫增加回收日志
 					//回收日志
 					recovelog := map[string]interface{}{
 						"site":             l["site"],
@@ -758,23 +761,30 @@ func (f *Front) Assign() {
 					}
 					save = append(save, recovelog)
 				}
-				//认领日志
-				claimlog := map[string]interface{}{
-					"site":             l["site"],
-					"code":             l["code"],
-					"channel":          l["channel"],
-					"modifyuser":       name,
-					"priority":         priority,
-					"stype":            "认领",
-					"comeintime":       now,
-					"claimtime":        now,
-					"recovertime":      recovertime,
-					"returntime":       int64(0),
-					"important":        spiderimportant,
-					"returnreason":     "",
-					"claimrecovertype": 0,
+				//被重新分配的爬虫视为重新认领（注意：lua爬虫分配到除chrome外的平台，变为历史爬虫）
+				if platform == "golua平台" { //爬虫分配到非lua平台，不视为认领
+					//认领日志
+					claimlog := map[string]interface{}{
+						"site":             l["site"],
+						"code":             l["code"],
+						"channel":          l["channel"],
+						"modifyuser":       name,
+						"priority":         priority,
+						"stype":            "认领",
+						"comeintime":       now,
+						"claimtime":        now,
+						"recovertime":      recovertime,
+						"returntime":       int64(0),
+						"important":        spiderimportant,
+						"returnreason":     "",
+						"claimrecovertype": 0,
+					}
+					save = append(save, claimlog)
+				} else if platform != "golua平台" && platform != "chrome" { //分配到其他平台改为历史爬虫
+					set["claimtype"] = CLAIMTYPEHISTORY
+					set["claimtime"] = 0
+					set["recovertime"] = 0
 				}
-				save = append(save, claimlog)
 			} else { //历史爬虫
 				set = map[string]interface{}{
 					"createuserid":    userid,
@@ -988,41 +998,8 @@ func (f *Front) UpdateESP() {
 		set["event"] = 7000
 		set["spidertype"] = "history"
 		go ModifyLogs_UpdateCodeState(code)
-		//更新、新建任务
-		//task, _ := u.MgoEB.FindOne("task", map[string]interface{}{"s_code": code, "i_state": map[string]interface{}{"$in": []int{0, 1, 2, 3, 5}}})
-		//text := "指定追加描述：------------------------------\n无发布转待完成\n"
-		//if len(*task) > 0 { //已有任务，更新
-		//	u.MgoEB.UpdateById("task", (*task)["_id"],
-		//		map[string]interface{}{
-		//			"$set": map[string]interface{}{
-		//				"i_state":      2,
-		//				"s_descript":   qu.ObjToString((*task)["s_descript"]) + text,
-		//				"l_updatetime": time.Now().Unix(),
-		//			},
-		//		})
-		//} else {
-		//	save := map[string]interface{}{
-		//		"s_channel":           (*one)["channel"],
-		//		"i_event":             (*one)["event"],
-		//		"i_frequencyerrtimes": 0,
-		//		"i_state":             2,
-		//		"s_descript":          text,
-		//		"l_complete":          time.Now().Unix() + int64(24*3600),
-		//		"s_urgency":           "4",
-		//		"i_pendstate":         0,
-		//		"s_modifyid":          (*one)["modifyuserid"],
-		//		"s_source":            "人工",
-		//		"i_times":             0,
-		//		"l_comeintime":        time.Now().Unix(),
-		//		"i_num":               0,
-		//		"s_modify":            (*one)["modifyuser"],
-		//		"s_code":              code,
-		//		"s_site":              (*one)["site"],
-		//		"s_type":              "0",
-		//	}
-		//	u.MgoEB.Save("task", save)
-		//}
 	} else if w == "platform" {
+		//常规字段更新
 		set["platform"] = val
 		set["comeintime"] = time.Now().Unix()
 		if val != "golua平台" && val != "chrome" {
@@ -1055,21 +1032,38 @@ func (f *Front) UpdateESP() {
 				f.Write("n")
 				return
 			}
-			//qu.Debug("下架:", upresult, code)
-			//b, err := UpStateAndUpSpider(code, "", "", "", Sp_state_6) //线上爬虫下架
-			//qu.Debug("爬虫下架成功：", b)
-			//if !b || err != nil {
-			//	f.Write("n")
-			//	return
-			//}
+			//lua转python，爬虫认领状态更新
+			UpdateLuaClaimtype(code)
+			//换平台爬虫回收，保存日志
+			recovelog := map[string]interface{}{
+				"site":             (*one)["site"],
+				"code":             (*one)["code"],
+				"channel":          (*one)["channel"],
+				"modifyuser":       (*one)["modifyuser"],
+				"priority":         (*one)["priority"],
+				"stype":            "回收",
+				"comeintime":       time.Now().Unix(),
+				"claimtime":        (*one)["claimtime"],
+				"recovertime":      (*one)["recovertime"],
+				"returntime":       int64(0),
+				"important":        (*one)["spiderimportant"],
+				"returnreason":     "转平台",
+				"claimrecovertype": 0,
+			}
+			u.MgoEB.Save("lua_logs_claim", recovelog)
+		}
+	} else if w == "priority" { //调整优先级
+		priority := qu.IntAll(val)
+		if priority < 0 {
+			priority = 0
+		} else if priority > 1000 {
+			priority = 1000
 		}
+		set["priority"] = priority
 	} else { //修改节点
 		event, _ := strconv.Atoi(val)
 		set["event"] = event
 		set["incrementevent"] = event
-		//state := f.GetString("s")
-		//if state == "5" { //已上架状态改为下架
-		code := f.GetString("c")
 		set["state"] = 6
 		b, err := UpStateAndUpSpider(code, "", "", "", Sp_state_6) //线上爬虫下架
 		qu.Debug("爬虫下架成功：", b)
@@ -1077,9 +1071,7 @@ func (f *Front) UpdateESP() {
 			f.Write("n")
 			return
 		}
-		//}
 	}
-	//if mgdb.Update("luaconfig", query, update, false, false) {
 	if u.MgoEB.Update("luaconfig", query, update, false, false) {
 		log.Println("Id:", id, "	Update", w, val, "Success")
 		f.Write("y")

src/front/site.go

@@ -355,7 +355,9 @@ func UpdateSiteCodeBaseInfo(site string, important int) {
 		ok := util.MgoEB.Del("site_code_baseinfo", query)
 		qu.Debug("删除重点网站爬虫信息：", site, ok)
 		set["spiderimportant"] = false
+		set["priority"] = ClaimPriorityOut
 	} else { //变为重点网站
+		set["priority"] = ClaimPriorityIn
 		set["spiderimportant"] = true
 		list, _ := util.MgoEB.Find("luaconfig", query, nil, map[string]interface{}{"site": 1, "code": 1, "channel": 1, "state": 1, "modifyuser": 1, "platform": 1}, false, -1, -1)
 		if len(*list) > 0 {

src/web/templates/index.html

@@ -367,7 +367,27 @@ $(function(){
 					  ttable.cell(cell).data(text);
 					  cellData = text;
 				  });
-			  }}
+			  }},
+			//更新优先级
+			{"targets":[9], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
+					$(cell).click(function () {
+						$(this).html("<input type='text' style='width: 50px'></<input>");
+						var aInput = $(this).find(":input");
+						aInput.focus().val(cellData);
+					});
+					$(cell).on("click", ":input", function (e) {
+						e.stopPropagation();
+					});
+					$(cell).on("change", ":input", function () {
+						$(this).blur();
+					});
+					$(cell).on("blur", ":input", function () {
+						var text = $(this).val();
+						ttable.cell(cell).data(text);
+						cellData = text;
+						updateesp(text,"priority","",rowData.code,rowData._id)
+					});
+				}}
 		],
 		"order": [[10,"desc"]],
 		"processing":true,
@@ -980,24 +1000,24 @@ $(function(){
 				"</select>";
 	};
 	function updateesp(val,w,s,c,id){
-	$.ajax({
-		url:"/center/spider/updateesp",
-		type:"post",
-		data:{
-			"val":val,
-			"w":w,
-			"id":id,
-			"s":s,
-			"c":c
-		},
-		success:function(r){
-			if(r=="n"){
-				showTip("修改失败", 1000, function() {});
+		$.ajax({
+			url:"/center/spider/updateesp",
+			type:"post",
+			data:{
+				"val":val,
+				"w":w,
+				"id":id,
+				"s":s,
+				"c":c
+			},
+			success:function(r){
+				if(r=="n"){
+					showTip("修改失败", 1000, function() {});
+				}
+				ttable.ajax.reload();
 			}
-			ttable.ajax.reload();
-		}
-	});
-}
+		});
+	}
 
 	{{if gt (session "auth") 1}}
 	function refuse(me){

src/web/templates/sitetask.html

@@ -56,9 +56,9 @@
 				
        		 },
 			"columnDefs": [
-				{ "orderable": false, "targets": [0,1,2,3,4,5,6,7,10,11,12,13] } //设置列不可排序
+				{ "orderable": false, "targets": [0,1,2,3,4,5,6,7,11,12,13] } //设置列不可排序
 			],
-			"order": [[9,"desc"]], //默认排序列
+			"order": [[10,"desc"]], //默认排序列
 			"lengthChange":false,
 			"serverSide": true,
 			"searching": true,
@@ -137,8 +137,11 @@
 				{"data": "_id","width":"7%",render:function(val,a,row){
 					var div=$("<div><div class=\"btn-group\"></div></div>");
 					{{if eq (session "auth") 1}}
-						var buttonWatch=$('<a type="button" target="_blank" class="btn btn-sm btn-primary">编辑</a>');
-						buttonWatch.attr("onclick","editSiteTask('"+row['s_spidercode']+"','"+row['encode']+"')");
+						var buttonWatch=$('<a type="button" target="_blank" disabled="true" class="btn btn-sm btn-primary">编辑</a>');
+						if (!row.b_repair){
+							var buttonWatch=$('<a type="button" target="_blank" class="btn btn-sm btn-primary">编辑</a>');
+							buttonWatch.attr("onclick","editSiteTask('"+row['s_spidercode']+"','"+row['encode']+"')");
+						}
 						div.find(".btn-group").append(buttonWatch);
 					{{else}}
 						var buttonEdit=$('<a type="button" target="_blank" class="btn btn-sm btn-primary">编辑</a>');