Merge branch 'dev2.0' of http://192.168.3.207:8080/data_processing/data_validation into dev2.0

* 'dev2.0' of http://192.168.3.207:8080/data_processing/data_validation:
  标注

src/front/front.go

@@ -6,6 +6,7 @@ import (
 	"mongodb"
 	qu "qfw/util"
 	"strconv"
+	"time"
 	. "util"
 )
 
@@ -66,17 +67,24 @@ type Front struct {
 	projectGroupTaskClose    xweb.Mapper `xweb:"/front/project/task/close"`        //用户组任务关闭
 	projectGetEntnameList    xweb.Mapper `xweb:"/front/project/getEntnameList"`    //模糊查询公司名称
 
+	//user task
 	userTaskSave     xweb.Mapper `xweb:"/front/user/task/save"`     //用户任务分发
 	userTaskList     xweb.Mapper `xweb:"/front/user/task/list"`     //用户任务列表
 	userTaskRetrieve xweb.Mapper `xweb:"/front/user/task/retrieve"` //用户任务收回
 	userTaskClose    xweb.Mapper `xweb:"/front/user/task/close"`    //用户任务关闭
 
+	//group task
 	groupTaskListByAdmin xweb.Mapper `xweb:"/front/group/admin/task/list"` //用户组任务列表(系统管理员权限)
 	groupTaskListByGroup xweb.Mapper `xweb:"/front/group/task/list"`       //用户组任务列表(用户组权限)
 	groupTaskDeliver     xweb.Mapper `xweb:"/front/group/task/deliver"`    //用户组任务交付
 	groupTaskExport      xweb.Mapper `xweb:"/front/group/task/export"`     //用户组任务导出
 	groupUserTaskList    xweb.Mapper `xweb:"/front/group/user/task/list"`  //用户组下用户任务列表
 
+	//mark
+	dataMark xweb.Mapper `xweb:"/front/data/mark"` //数据标注
+
+	//check
+
 	remarkList   xweb.Mapper `xweb:"/front/user/remark/list"`
 	remarkDetail xweb.Mapper `xweb:"/center/detail/(.*).html"`
 }
@@ -124,3 +132,31 @@ func GetUserMenu(role int) []map[string]interface{} {
 	}
 	return list
 }
+
+// SaveLog 标注日志保存
+func SaveLog(s_infoid, s_usertaskid, s_stype string, tagFields map[string]interface{}) (success bool) {
+	defer qu.Debug()
+	task, _ := Mgo.FindById(TASKCOLLNAME, s_usertaskid, nil)
+	if task != nil && len(*task) > 0 {
+		v_before := map[string]interface{}{}
+		v_after := map[string]interface{}{}
+		save := map[string]interface{}{
+			"s_projectname": qu.ObjToString((*task)["s_projectname"]),
+			"s_taskid":      s_usertaskid,
+			"s_infoid":      s_infoid,
+			"s_stype":       s_stype,
+			"s_userid":      qu.ObjToString((*task)["s_personid"]),
+			"s_username":    qu.ObjToString((*task)["s_personname"]),
+			"i_createtime":  time.Now().Unix(),
+			"v_before":      v_before,
+			"v_after":       v_after,
+		}
+		id := Mgo.Save(LOGCOLLNAME, save)
+		if id != "" {
+			success = true
+		}
+	} else {
+		qu.Debug("Find Data Failed：", s_infoid)
+	}
+	return
+}

src/front/mark.go

@@ -0,0 +1,463 @@
+package front
+
+import (
+	"encoding/json"
+	"fmt"
+	qu "qfw/util"
+	"strings"
+	"time"
+	"util"
+)
+
+// DataMark 数据标注
+func (f *Front) DataMark() {
+	defer qu.Catch()
+	success := false
+	msg := ""
+	//user := f.GetSession("user").(map[string]interface{})
+	//username := qu.ObjToString(user["s_login"]) //当前登录用户
+	obj := []map[string]interface{}{}
+	infoId := f.GetString("s_infoid")
+	userTaskId := f.GetString("s_usertaskid")
+	data := f.GetString("data")
+	err := json.Unmarshal([]byte(data), &obj)
+	if err != nil {
+		f.ServeJson(map[string]interface{}{"success": success, "msg": "解析数据失败"})
+		return
+	}
+	tagSet := map[string]interface{}{}    //被标注字段状态
+	baseSet := map[string]interface{}{}   //要修改的字段信息
+	baseUnset := map[string]interface{}{} //要删除的字段信息
+
+	//isSaveMarked := false
+	if len(obj) == 1 { //单独保存某个一级
+		content, ok := obj[0]["content"].([]interface{})
+		if !ok || len(content) == 0 {
+			f.ServeJson(map[string]interface{}{"success": success, "msg": "解析数据失败"})
+			return
+		}
+		title := qu.ObjToString(obj[0]["title"])
+		istag, _ := obj[0]["checkAllTag"].(bool)
+		status := qu.IntAll(obj[0]["status"])
+		switch title {
+		case "基本字段":
+			BzJBZD_New(content, tagSet, baseSet, baseUnset)
+		case "时间地点":
+			BzSJDD_New(content, tagSet, baseSet, baseUnset)
+		case "标的信息":
+			BzBDXX_New(content, tagSet, baseSet, baseUnset, istag, status)
+		case "多包信息":
+			BzDBXX_New(content, tagSet, baseSet, baseUnset, status)
+		case "中标候选人信息":
+			BzZBHXRXX_New(content, tagSet, baseSet, baseUnset, status)
+		case "其余信息":
+			BzQYXX_New(content, tagSet, baseSet, baseUnset)
+		}
+	} else {
+		for j, val := range obj {
+			content, ok := val["content"].([]interface{})
+			status := qu.IntAll(val["status"])
+			if !ok {
+				qu.Debug("Content Error")
+				continue
+			}
+			istag, _ := val["checkAllTag"].(bool)
+			if j == 0 { //基本信息
+				BzJBZD_New(content, tagSet, baseSet, baseUnset)
+			} else if j == 1 { //时间地点
+				BzSJDD_New(content, tagSet, baseSet, baseUnset)
+			} else if j == 2 { //标的物
+				BzBDXX_New(content, tagSet, baseSet, baseUnset, istag, status)
+			} else if j == 3 { //多包
+				BzDBXX_New(content, tagSet, baseSet, baseUnset, status)
+			} else if j == 4 { //候选人
+				BzZBHXRXX_New(content, tagSet, baseSet, baseUnset, status)
+			} else { //其余信息
+				BzQYXX_New(content, tagSet, baseSet, baseUnset)
+			}
+		}
+	}
+	userTask, _ := util.Mgo.FindById(util.TASKCOLLNAME, userTaskId, map[string]interface{}{"s_personid": 1, "s_personname": 1, "s_projectname": 1, "s_sourceinfo": 1})
+	if sourceInfo := qu.ObjToString((*userTask)["s_sourceinfo"]); sourceInfo != "" { //数据源表
+		dataInfo, _ := util.Mgo.FindById(sourceInfo, infoId, map[string]interface{}{"v_baseinfo": 1, "v_taginfo": 1}) //查询标注保存前的原始信息
+		if tagInfo, ok := (*dataInfo)["v_taginfo"].(map[string]interface{}); ok && len(tagInfo) > 0 {
+			for field, tmpStatus := range tagSet { //比对本次标注信息和历史标注信息
+				status := qu.IntAll(tmpStatus)            //此次被标注字段的状态
+				markedStatus := qu.IntAll(tagInfo[field]) //历史标注状态
+				if status == 1 && markedStatus != 0 {     //此次标注结果为正确，且有历史标注记录，不做修改
+					qu.Debug("已标注字段field---", field)
+					delete(tagSet, field)
+					delete(baseSet, field)
+					continue
+				} else {
+					qu.Debug("未标注字段field---", field, status)
+				}
+			}
+		}
+		if len(tagSet) > 0 {
+			//1、更新数据源信息
+			setResult := map[string]interface{}{ //更新字段集
+				"i_updatetime": time.Now().Unix(),
+				"i_ckdata":     2,
+				"b_istag":      true,
+			}
+			for field, val := range tagSet { //更新标注字段
+				setResult["v_taginfo."+field] = val
+			}
+			for field, val := range baseSet { //更新基本字段
+				setResult["v_baseinfo."+field] = val
+			}
+			baseUnsetResult := map[string]interface{}{} //删除字段集
+			for field, _ := range baseUnset {           //删除基本字段
+				baseUnsetResult["v_baseinfo."+field] = ""
+			}
+			set := map[string]interface{}{
+				"$set": setResult,
+			}
+			if len(baseUnsetResult) > 0 {
+				set["$unset"] = baseUnsetResult
+			}
+			qu.Debug("set---", set)
+			//success = util.Mgo.UpdateById(sourceInfo, infoId, set)
+			//2、更新marked表
+
+			//3、保存标注记录
+		}
+
+	}
+	f.ServeJson(map[string]interface{}{"success": success, "msg": msg})
+}
+
+func BzJBZD_New(content []interface{}, tagSet, baseSet, baseUnset map[string]interface{}) {
+	defer qu.Catch()
+	info, _ := content[0].(map[string]interface{})
+	if uInputs, ok := info["uInput"].([]interface{}); ok {
+		for _, tmp := range uInputs {
+			if tmpMap, ok := tmp.(map[string]interface{}); ok {
+				if status := qu.IntAll(tmpMap["status"]); status != -1 {
+					key := qu.ObjToString(tmpMap["key"]) //字段
+					if key == "" {
+						continue
+					}
+					if status == 2 { //新增、修改、删除
+						input := tmpMap["input"]                                                                                               //值
+						if key == "bidamounttype" || key == "subtype" || key == "attach_discern" || key == "attach_ext" || key == "isrepeat" { //附件识别、抽取select
+							input = tmpMap["select"]
+						}
+						if input == "" { //删除原字段
+							baseUnset[key] = ""
+						} else { //修改原字段
+							if key == "budget" || key == "bidamount" || key == "biddiscount" {
+								input = qu.Float64All(input)
+								//input, _ = strconv.ParseFloat(qu.ObjToString(input), 32)
+							}
+							if key == "subtype" {
+								if topsubtype := strings.Split(qu.ObjToString(input), "-"); len(topsubtype) == 2 {
+									baseSet["toptype"] = topsubtype[0]
+									baseSet[key] = topsubtype[1]
+								}
+							} else {
+								baseSet[key] = input
+							}
+						}
+					}
+					tagSet[key] = status //记录被标注状态
+				}
+			}
+		}
+	}
+	qu.Debug("tagSet===", tagSet)
+	qu.Debug("baseSet===", baseSet)
+	qu.Debug("baseUnset===", baseUnset)
+}
+
+func BzSJDD_New(content []interface{}, tagSet, baseSet, baseUnset map[string]interface{}) {
+	info, _ := content[0].(map[string]interface{})
+	if uInputs, ok := info["uInput"].([]interface{}); ok {
+		for _, tmp := range uInputs {
+			if tmpMap, ok := tmp.(map[string]interface{}); ok {
+				if status := qu.IntAll(tmpMap["status"]); status != -1 {
+					key := qu.ObjToString(tmpMap["key"]) //字段
+					if key == "" {
+						continue
+					}
+					if status == 2 { //新增、修改、删除
+						input := tmpMap["input"] //值
+						if input == "" {
+							baseUnset[key] = ""
+						} else {
+							if key == "bidopentime" || key == "publishtime" || key == "bidendtime" || key == "project_startdate" || key == "project_completedate" {
+								inputTmp, _ := time.ParseInLocation(qu.Date_Full_Layout, input.(string), time.Local)
+								input = inputTmp.Unix()
+							} else if key == "project_duration" {
+								input = qu.IntAll(input)
+							}
+							baseSet[key] = input
+						}
+					}
+					tagSet[key] = status
+				}
+			}
+		}
+	}
+	qu.Debug("tagSet===", tagSet)
+	qu.Debug("baseSet===", baseSet)
+	qu.Debug("baseUnset===", baseUnset)
+}
+
+func BzBDXX_New(content []interface{}, tagSet, baseSet, baseUnset map[string]interface{}, istag bool, status int) {
+	if status == -1 {
+		return
+	}
+	baseSet["purchasinglist_alltag"] = istag //标的信息是否标注完全
+	purchasinglist := []interface{}{}
+	delpclson := 0
+	for _, con := range content {
+		info, _ := con.(map[string]interface{})
+		isNew, _ := info["isnew"].(bool) //是否是新增子包
+		pclSonStatus := qu.IntAll(info["status"])
+		if pclSonStatus == 4 {
+			delpclson++
+		}
+		if uInputs, ok := info["uInput"].([]interface{}); ok {
+			result := map[string]interface{}{
+				"isnew": isNew,
+			}
+			for _, tmp := range uInputs {
+				if tmpMap, ok := tmp.(map[string]interface{}); ok {
+					key := qu.ObjToString(tmpMap["key"]) //字段
+					input := tmpMap["input"]             //值
+					//status := qu.IntAll(tmpMap["status"])
+					isNull := false
+					if input == "" { //判断前台页面是否填值，无值不进行字段存储
+						isNull = true
+					} else if key == "number" || key == "unitprice" || key == "totalprice" {
+						input = qu.Float64All(input)
+					}
+					if !isNull { //避免数字类型的字段在没有填写值的情况默认给0
+						result[key] = input
+					}
+				}
+			}
+			if pclSonStatus != -1 { //没有标注的标的信息不打标记
+				result["purchasinglist_son"] = pclSonStatus
+			}
+			if len(result) > 0 && pclSonStatus != 4 {
+				purchasinglist = append(purchasinglist, result)
+			}
+		}
+	}
+	qu.Debug("purchasinglist", len(purchasinglist))
+	if len(purchasinglist)+delpclson == len(content) {
+		if len(purchasinglist) > 0 {
+			baseSet["purchasinglist"] = purchasinglist
+		}
+		if len(content) > 0 && delpclson == len(content) { //只有删除
+			baseUnset["purchasinglist"] = ""
+		}
+		tagSet["purchasinglist"] = status
+	} else {
+		qu.Debug("Purchasinglist Tag Error")
+	}
+	qu.Debug("tagSet===", tagSet)
+	qu.Debug("baseSet===", baseSet)
+	qu.Debug("baseUnset===", baseUnset)
+}
+
+func BzDBXX_New(content []interface{}, tagSet, baseSet, baseUnset map[string]interface{}, status int) {
+	if status == -1 {
+		return
+	}
+	pkgs := map[string]interface{}{}
+	newNum := 1
+	delpkgson := 0 //记录子包删除个数
+	for _, con := range content {
+		info, _ := con.(map[string]interface{})
+		pkgSonStatus := qu.IntAll(info["status"])
+		if pkgSonStatus == 4 {
+			delpkgson++
+		}
+		num := fmt.Sprint(info["num"])   //包号
+		isNew, _ := info["isnew"].(bool) //是否是新增子包
+		if isNew {                       //新增子包新建包名
+			num = "new" + fmt.Sprint(newNum)
+			newNum++
+		}
+		if uInputs, ok := info["uInput"].([]interface{}); ok {
+			result := map[string]interface{}{
+				"isnew": isNew,
+			}
+			winnerArr := []interface{}{}
+			bidamountArr := []interface{}{}
+			for _, tmp := range uInputs {
+				if tmpMap, ok := tmp.(map[string]interface{}); ok {
+					key := qu.ObjToString(tmpMap["key"]) //字段
+					input := tmpMap["input"]             //值
+					isNull := false                      //记录字段是否有值
+					if key == "bidamounttype" {
+						input = tmpMap["select"]
+					} else {
+						if input == "" { //判断前台页面是否填值，无值不进行字段存储
+							isNull = true
+						} else if key == "bidamount" || key == "budget" {
+							input = qu.Float64All(input)
+							//input, _ = strconv.ParseFloat(qu.ObjToString(input), 32)
+						}
+					}
+					if key == "winner" {
+						if isNull {
+							winnerArr = append(winnerArr, nil)
+						} else {
+							winnerArr = append(winnerArr, input)
+						}
+						continue
+					} else if key == "bidamount" {
+						if isNull {
+							bidamountArr = append(bidamountArr, nil)
+						} else {
+							bidamountArr = append(bidamountArr, input)
+						}
+						continue
+					}
+
+					if !isNull { //避免数字类型的字段在没有填写值的情况默认给0
+						result[key] = input
+					}
+				}
+			}
+			winner_all := []interface{}{}
+			if len(winnerArr) == len(bidamountArr) {
+				for i, w := range winnerArr {
+					b := bidamountArr[i]
+					wbMap := map[string]interface{}{}
+					if w != nil {
+						wbMap["winner"] = w
+					}
+					if b != nil {
+						wbMap["bidamount"] = b
+					}
+					if len(wbMap) > 0 {
+						winner_all = append(winner_all, wbMap)
+					}
+				}
+			}
+			if len(winner_all) > 0 {
+				result["winner_all"] = winner_all
+			}
+			result["package_son"] = pkgSonStatus
+
+			if len(result) > 0 && pkgSonStatus != 4 { //要删除的子包不再保存
+				pkgs[num] = result
+			}
+		}
+	}
+	qu.Debug("pkgs", len(pkgs))
+	if len(pkgs)+delpkgson == len(content) {
+		if len(pkgs) > 0 {
+			baseSet["package"] = pkgs
+		}
+		if len(content) > 0 && delpkgson == len(content) { //只有删除
+			baseUnset["package"] = ""
+		}
+		tagSet["package"] = status
+	} else {
+		qu.Debug("Package Tag Error")
+	}
+	qu.Debug("tagSet===", tagSet)
+	qu.Debug("baseSet===", baseSet)
+	qu.Debug("baseUnset===", baseUnset)
+}
+
+func BzZBHXRXX_New(content []interface{}, tagSet, baseSet, baseUnset map[string]interface{}, status int) {
+	if status == -1 {
+		return
+	}
+	winnerorder := []interface{}{}
+	delwodrson := 0
+	for _, con := range content {
+		info, _ := con.(map[string]interface{})
+		isNew, _ := info["isnew"].(bool) //是否是新增子包
+		wodrSonStatus := qu.IntAll(info["status"])
+		if wodrSonStatus == 4 {
+			delwodrson++
+		}
+		if uInputs, ok := info["uInput"].([]interface{}); ok {
+			result := map[string]interface{}{
+				"isnew": isNew,
+			}
+			for _, tmp := range uInputs {
+				if tmpMap, ok := tmp.(map[string]interface{}); ok {
+					key := qu.ObjToString(tmpMap["key"]) //字段
+					input := tmpMap["input"]             //值
+					isNull := false
+					if input == "" { //判断前台页面是否填值，无值不进行字段存储
+						isNull = true
+					} else if key == "price" {
+						input = qu.Float64All(input)
+						//input, _ = strconv.ParseFloat(qu.ObjToString(input), 32)
+					}
+					if !isNull { //避免数字类型的字段在没有填写值的情况默认给0
+						result[key] = input
+					}
+					result["winnerorder_son"] = wodrSonStatus
+				}
+			}
+			if len(result) > 0 && wodrSonStatus != 4 {
+				winnerorder = append(winnerorder, result)
+			}
+		}
+	}
+	qu.Debug("winnerorder", len(winnerorder))
+	if len(winnerorder)+delwodrson == len(content) {
+		if len(winnerorder) > 0 {
+			baseSet["winnerorder"] = winnerorder
+		}
+		if len(content) > 0 && delwodrson == len(content) { //只有删除
+			baseUnset["winnerorder"] = ""
+		}
+		tagSet["winnerorder"] = status
+	} else {
+		qu.Debug("Winnerorder Tag Error")
+	}
+	qu.Debug("tagSet===", tagSet)
+	qu.Debug("baseSet===", baseSet)
+	qu.Debug("baseUnset===", baseUnset)
+}
+
+func BzQYXX_New(content []interface{}, tagSet, baseSet, baseUnset map[string]interface{}) {
+	info, _ := content[0].(map[string]interface{})
+	if uInputs, ok := info["uInput"].([]interface{}); ok {
+		for _, tmp := range uInputs {
+			if tmpMap, ok := tmp.(map[string]interface{}); ok {
+				if status := qu.IntAll(tmpMap["status"]); status != -1 {
+					key := qu.ObjToString(tmpMap["key"]) //字段
+					if key == "" {
+						continue
+					}
+					if status == 2 { //新增、修改、删除
+						input := tmpMap["input"] //值
+						if key == "isppp" || key == "contract_guarantee" || key == "bid_guarantee" {
+							input = tmpMap["select"]
+						}
+						if input == "" {
+							baseUnset[key] = ""
+						} else {
+							if key == "signaturedate" {
+								inputTmp, _ := time.ParseInLocation(qu.Date_Full_Layout, input.(string), time.Local)
+								input = inputTmp.Unix()
+							} else if key == "bid_bond" || key == "contract_bond" || key == "supervisorrate" || key == "agencyrate" || key == "docamount" || key == "agencyfee" {
+								input = qu.Float64All(input)
+								//input, _ = strconv.ParseFloat(qu.ObjToString(input), 32)
+							}
+							baseSet[key] = input
+						}
+					}
+					tagSet[key] = status
+				}
+			}
+		}
+	}
+	qu.Debug("tagSet===", tagSet)
+	qu.Debug("baseSet===", baseSet)
+	qu.Debug("baseUnset===", baseUnset)
+}

src/front/quality.go

@@ -79,10 +79,9 @@ func calculateFieldScore(tmp map[string]interface{}, field_tag []string) map[str
 	}
 
 	//综合比对是否正确 field_tag   - 指定字段
-	isUse, v_taginfo := true, make(map[string]interface{}, 0)
+	isUse := true
 	b_isfield_tag := make(map[string]interface{}, 0)
 	for _, key := range field_tag {
-		v_taginfo[key] = 1
 		value := *qu.ObjToMap(update_dict[key])
 		score := qu.Int64All(value["score"])
 		if score < 99 {
@@ -101,16 +100,20 @@ func calculateFieldScore(tmp map[string]interface{}, field_tag []string) map[str
 	}
 	b_istagging := !isUse
 
+	ckdata := 0
+	if isUse { //达标数据ck_data默认未2，未达标为0
+		ckdata = 2
+	}
 	//计算标的物-有效状态
 	b_isprchasing := calculatePrchasinglist(tmp)
 
 	return map[string]interface{}{
 		"$set": map[string]interface{}{
 			"v_fieldscore":  update_dict,
-			"v_taginfo":     v_taginfo,
 			"b_isfield_tag": b_isfield_tag,
 			"b_istagging":   b_istagging,
 			"b_isprchasing": b_isprchasing,
+			"i_ckdata":      ckdata,
 		},
 	}
 }

src/main.go

@@ -34,6 +34,16 @@ func init() {
 }
 
 func main() {
+	set := map[string]interface{}{
+		"$set": map[string]interface{}{
+			"test.aa": "aa",
+		},
+		"$unset": map[string]interface{}{
+			"test.cc": "",
+		},
+	}
+	Mgo.UpdateById("test", "60f0e69c1a75b8f446056be1", set)
+
 	//qu.Debug(SE.EncodeString("60b99c2d72c25c51c492af6a"))
 	//return
 	//go front.QuaFieldScore([]string{"buyer"},"zktest_data") //临时测试-质量

src/util/config.go

@@ -55,6 +55,7 @@ var SE = qu.SimpleEncrypt{Key: "topJYBX2019"}
 const BIDDINGSTARTID = "5a862f0640d2d9bbe88e3cec"
 const PROJECTCOLLNAME = "f_project"
 const TASKCOLLNAME = "f_task"
+const LOGCOLLNAME = "l_taglog"
 
 func InitConfig() {
 	Mgo = &mongodb.MongodbSim{