Merge branch 'dev2.0' of http://192.168.3.207:8080/data_processing/data_validation into dev2.0

* 'dev2.0' of http://192.168.3.207:8080/data_processing/data_validation:
  xg

src/config.json

@@ -41,7 +41,7 @@
         "行业归属": "buyer_type",
         "采购方式": "bidtype",
         "项目标识": "projectId",
-        "中标人模型": "winner_type",
+        "中标人类型": "winner_type",
         "中标类型": "winner_style",
         "标签": "tagname",
         "附加词": "tagname",

src/front/front.go

@@ -68,6 +68,7 @@ type Front struct {
 	projectGroupTaskRepulse  xweb.Mapper `xweb:"/front/project/task/repulse"`      //用户组任务打回
 	projectGroupTaskClose    xweb.Mapper `xweb:"/front/project/task/close"`        //用户组任务关闭
 	projectGetEntnameList    xweb.Mapper `xweb:"/front/project/getEntnameList"`    //模糊查询公司名称
+	projectIsExists          xweb.Mapper `xweb:"/front/project/isExists"`          //判断项目名称是否存在
 
 	//user task
 	userTaskSave     xweb.Mapper `xweb:"/front/user/task/save"`     //用户任务分发

src/front/group.go

@@ -96,7 +96,7 @@ func (f *Front) GroupTaskListByAdmin() {
 			query["$or"] = []interface{}{
 				map[string]interface{}{"s_projectname": map[string]interface{}{"$regex": search}},
 				map[string]interface{}{"s_entname": map[string]interface{}{"$regex": search}},
-				map[string]interface{}{"s_rule": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"s_rulename": map[string]interface{}{"$regex": search}},
 				map[string]interface{}{"s_departname": map[string]interface{}{"$regex": search}},
 			}
 		}

src/front/mark.go

@@ -261,7 +261,6 @@ func (f *Front) JyUserDataMark() {
 
 // UserDataMark 标注人员数据标注(基于任务的标注)
 func (f *Front) UserDataMark() {
-	qu.Debug("------------------")
 	defer qu.Catch()
 	success := false
 	msg := ""

src/front/project.go

@@ -18,6 +18,18 @@ import (
 	"util"
 )
 
+// ProjectIsExists 获取所有项目名称
+func (f *Front) ProjectIsExists() {
+	defer qu.Catch()
+	name := f.GetString("s_name")
+	exists := false
+	project, _ := util.Mgo.FindOne(util.PROJECTCOLLNAME, map[string]interface{}{"s_name": name})
+	if project != nil && len(*project) > 0 {
+		exists = true
+	}
+	f.ServeJson(map[string]interface{}{"exists": exists})
+}
+
 // ProjectList 项目列表
 func (f *Front) ProjectList() {
 	defer qu.Catch()
@@ -70,6 +82,10 @@ func (f *Front) ProjectSave() {
 	stype := f.GetString("s_type")                 //新建项目类型：数据库导入、excel导入
 	s_sourceinfoTmp := f.GetString("s_sourceinfo") //数据表
 	s_sourceinfo := "f_sourceinfo_" + s_sourceinfoTmp
+	if !util.Mgo.CreateIndex(s_sourceinfo, util.SourceInfoIndexArr) { //创建数据源表同时生成字段索引
+		f.ServeJson(map[string]interface{}{"success": false, "msg": "数据源表创建失败"})
+		return
+	}
 	if stype != "edit" && s_sourceinfo == "" {
 		f.ServeJson(map[string]interface{}{"success": false, "msg": "缺少数据存储表名"})
 		return
@@ -79,7 +95,6 @@ func (f *Front) ProjectSave() {
 		"s_name": s_name,
 	}
 	set := map[string]interface{}{}
-	qu.Debug(stype)
 	//导入数据
 	if stype == "excel" { //excel导入
 		s_entname = f.GetString("s_entname") //公司名称
@@ -369,7 +384,6 @@ func (f *Front) ProjectGroupTaskSave() {
 	projectname := qu.ObjToString((*project)["s_name"])      //项目名称
 	sourceinfo := qu.ObjToString((*project)["s_sourceinfo"]) //源数据表
 	group := f.GetString("s_group")
-	qu.Debug(group)
 	stype := f.GetString("s_type")
 	qu.Debug("项目id：", projectid, " 项目名称：", projectname)
 	if err := json.Unmarshal([]byte(group), &groupArr); err != nil {
@@ -909,7 +923,9 @@ func ImportDataByColl(s_sourceinfo, historyid string, success *bool, msg *string
 			}
 			rname := qu.ObjToString(tmp["rulename"])
 			lock.Lock()
-			rulenameMap[rname] = true
+			for _, r := range strings.Split(rname, ",") {
+				rulenameMap[r] = true
+			}
 			//rulename = append(rulename, qu.ObjToString(tmp["rulename"])) //规则名称
 			idInfoMap[id] = needField
 			lock.Unlock()
@@ -969,9 +985,9 @@ func GetDataById(idsInfo map[string]map[string]interface{}, importType, s_source
 			bidData, _ := util.MgoB.FindById(tmpBidColl, id, nil)
 			if bidData != nil && len(*bidData) > 0 { //bidding表数据存在
 				//2.查extract
-				extData, _ := util.MgoE.FindById(util.ExtColl1, id, nil)
+				extData, _ := util.MgoE.FindById(util.ExtColl1, id, map[string]interface{}{"attach_text": 0})
 				if extData == nil || len(*extData) == 0 {
-					extData, _ = util.MgoE.FindById(util.ExtColl2, id, nil)
+					extData, _ = util.MgoE.FindById(util.ExtColl2, id, map[string]interface{}{"attach_text": 0})
 				}
 				//抽取表字段合并到bidding
 				if extData != nil && len(*extData) > 0 {
@@ -1044,36 +1060,40 @@ func GetDataById(idsInfo map[string]map[string]interface{}, importType, s_source
 				//esQ := `{"query":{"bool":{"must":[{"term":{"ids":"` + id + `"}}]}}}`
 				//info := util.Es.Get("projectset", "projectset", esQ)
 				projectId := qu.ObjToString((*bidData)["projectId"])
-				project, _ := util.MgoE.FindById(util.ProjectColl, projectId, map[string]interface{}{"ids": 1})
-				if project != nil && len(*project) > 0 {
-					ids := qu.ObjArrToStringArr((*project)["ids"].([]interface{}))
-					if len(ids) > 0 {
-						var infolist []map[string]interface{}
-						for _, v := range ids {
-							if v == id { // 当前公告
-								continue
-							}
-							if v < util.BIDDINGSTARTID {
-								tmpBidColl = util.BidColl2 //bidding_back
-							}
-							bid, b := util.MgoB.FindById(tmpBidColl, v, nil)
-							if b && len(*bid) > 0 {
-								tmp := make(map[string]interface{})
-								tmp["id"] = v
-								tmp["title"] = (*bid)["title"]
-								tmp["href"] = (*bid)["href"]
-								tmp["toptype"] = (*bid)["toptype"]
-								tmp["subtype"] = (*bid)["subtype"]
-								tmp["publishtime"] = (*bid)["publishtime"]
-								tmp["detail"] = (*bid)["detail"]
-								tmp["filetext"] = util.GetFileText(*bid)
-								infolist = append(infolist, tmp)
+				if projectId != "" && len(projectId) == 24 { //舍弃项目id为空和合并错误的
+					project, _ := util.MgoE.FindById(util.ProjectColl, projectId, map[string]interface{}{"ids": 1})
+					if project != nil && len(*project) > 0 {
+						ids := qu.ObjArrToStringArr((*project)["ids"].([]interface{}))
+						if len(ids) > 0 {
+							var infolist []map[string]interface{}
+							for _, v := range ids {
+								if v == id { // 当前公告
+									continue
+								}
+								if v < util.BIDDINGSTARTID {
+									tmpBidColl = util.BidColl2 //bidding_back
+								}
+								bid, b := util.MgoB.FindById(tmpBidColl, v, nil)
+								if b && len(*bid) > 0 {
+									tmp := make(map[string]interface{})
+									tmp["id"] = v
+									tmp["title"] = (*bid)["title"]
+									tmp["href"] = (*bid)["href"]
+									tmp["toptype"] = (*bid)["toptype"]
+									tmp["subtype"] = (*bid)["subtype"]
+									tmp["publishtime"] = (*bid)["publishtime"]
+									tmp["detail"] = (*bid)["detail"]
+									tmp["filetext"] = util.GetFileText(*bid)
+									infolist = append(infolist, tmp)
+								}
 							}
+							(*bidData)["info"] = infolist
 						}
-						(*bidData)["info"] = infolist
+					} else {
+						qu.Debug("Projectset Find Error", projectId)
 					}
 				} else {
-					qu.Debug("Projectset Find Error", projectId, ", tmp--", tmp)
+					qu.Debug("ProjectId Is Null", id)
 				}
 				baseInfoMap["id"] = id
 				_id := (*bidData)["_id"]

src/front/user.go

@@ -597,7 +597,7 @@ func (f *Front) UserTaskList() {
 		query := map[string]interface{}{
 			"s_stype": "user",
 		}
-		if s_role != "0" && s_role != "1" {
+		if s_role == "4" {
 			query["s_personid"] = s_personid
 		}
 		if status != "-1" { //任务状态

src/main.go

@@ -34,16 +34,6 @@ func init() {
 }
 
 func main() {
-	set := map[string]interface{}{
-		"$set": map[string]interface{}{
-			"test.aa": "aa",
-		},
-		"$unset": map[string]interface{}{
-			"test.cc": "",
-		},
-	}
-	Mgo.UpdateById("test", "60f0e69c1a75b8f446056be1", set)
-
 	//qu.Debug(SE.EncodeString("60b99c2d72c25c51c492af6a"))
 	//return
 	//go front.QuaFieldScore([]string{"buyer"},"zktest_data") //临时测试-质量

src/util/config.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"go.mongodb.org/mongo-driver/mongo"
 	"mongodb"
 	qu "qfw/util"
 	"sort"
@@ -48,6 +49,18 @@ var (
 )
 
 var SE = qu.SimpleEncrypt{Key: "topJYBX2019"}
+var SourceInfoIndexArr = []mongo.IndexModel{
+	mongo.IndexModel{Keys: map[string]interface{}{"s_grouptaskid": 1}},               //独立索引
+	mongo.IndexModel{Keys: map[string]interface{}{"s_grouptaskid": 1, "b_istag": 1}}, //联合索引
+	mongo.IndexModel{Keys: map[string]interface{}{"s_grouptaskid": 1, "s_isgiveuser": 1}},
+	mongo.IndexModel{Keys: map[string]interface{}{"b_istagging": 1}},
+	mongo.IndexModel{Keys: map[string]interface{}{"b_istagging": 1, "b_istag": 1}},
+	mongo.IndexModel{Keys: map[string]interface{}{"b_isgivegroup": 1, "b_istagging": 1}},
+	mongo.IndexModel{Keys: map[string]interface{}{"b_isgivegroup": 1, "s_isgiveuser": 1}},
+	mongo.IndexModel{Keys: map[string]interface{}{"b_isgivegroup": 1, "s_isgiveuser": 1, "b_istag": 1}},
+	mongo.IndexModel{Keys: map[string]interface{}{"b_isgivegroup": 1, "s_isgiveuser": 1, "b_istagging": 1}},
+	mongo.IndexModel{Keys: map[string]interface{}{"b_isgivegroup": 1, "s_isgiveuser": 1, "i_ckdata": 1}},
+}
 
 const JYHREFPRE = "https://www.jianyu360.cn/article/content/"
 const BIDDINGSTARTID = "5a862f0640d2d9bbe88e3cec"