任务分发

src/config.json

@@ -23,7 +23,7 @@
     },
     "jy": {
         "addr":"192.168.3.207:27092",
-        "db": "qfw_data",
+        "db": "qfw",
         "history": "usermail_history",
         "user": "user",
         "size": 10
@@ -70,5 +70,4 @@
         "企业id列表": "entidlist",
         "项目唯一标识": "projectId"
     }
-
 }

src/front/front.go

@@ -55,9 +55,14 @@ type Front struct {
 	roleSecondEdit xweb.Mapper `xweb:"/front/role/second/edit"` //二级权限编辑
 
 	//project
-	projectList xweb.Mapper `xweb:"/front/project"`      //项目列表
-	projectSave xweb.Mapper `xweb:"/front/project/save"` //新增项目
+	projectList           xweb.Mapper `xweb:"/front/project"`                //项目列表
+	projectSave           xweb.Mapper `xweb:"/front/project/save"`           //新增项目
+	projectClear          xweb.Mapper `xweb:"/front/project/clear"`          //项目清洗
+	projectTaskSave       xweb.Mapper `xweb:"/front/project/task/save"`      //用户组任务分发
+	projectTaskList       xweb.Mapper `xweb:"/front/project/task/list"`      //用户组任务分发列表
+	projectGetEntnameList xweb.Mapper `xweb:"/front/project/getEntnameList"` //模糊查询公司名称
 
+	//task
 }
 
 func (f *Front) Index() {

src/front/project.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"mime/multipart"
 	qu "qfw/util"
+	"sort"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -19,6 +20,7 @@ func (f *Front) ProjectList() {
 	defer qu.Catch()
 	if f.Method() == "POST" {
 		start, _ := f.GetInteger("start")
+		limit, _ := f.GetInteger("length")
 		draw, _ := f.GetInteger("draw")
 		status := f.GetString("status")
 		searchStr := f.GetString("search[value]")
@@ -36,8 +38,8 @@ func (f *Front) ProjectList() {
 				map[string]interface{}{"s_departname": map[string]interface{}{"$regex": search}},
 			}
 		}
-		list, _ := util.Mgo.Find("f_project", query, nil, nil, false, start, draw)
-		count := util.Mgo.Count("f_project", query)
+		list, _ := util.Mgo.Find(util.PROJECTCOLLNAME, query, nil, nil, false, start, limit)
+		count := util.Mgo.Count(util.PROJECTCOLLNAME, query)
 		f.ServeJson(map[string]interface{}{"draw": draw, "data": *list, "recordsFiltered": count, "recordsTotal": count})
 	} else {
 		_ = f.Render("project/project_list.html", &f.T)
@@ -47,9 +49,11 @@ func (f *Front) ProjectList() {
 // ProjectSave 项目保存
 func (f *Front) ProjectSave() {
 	defer qu.Catch()
-	success := false
-	msg := ""
-	successNum := int64(0)
+	success := false        //导入数据是否成功
+	msg := ""               //异常信息
+	successNum := int64(0)  //导入成功条数
+	importDataNum := 0      //查询数量
+	var s_rulename []string //规则
 	user := f.GetSession("user").(map[string]interface{})
 	username := qu.ObjToString(user["s_name"])  //当前登录用户
 	stype := f.GetString("s_type")              //新建项目类型：数据库导入、excel导入
@@ -57,8 +61,6 @@ func (f *Front) ProjectSave() {
 	s_sourceinfo := f.GetString("s_sourceinfo") //数据表
 	s_sourceinfo = "s_sourceinfo_" + s_sourceinfo
 	s_departname, s_entname := "", ""
-	importDataNum := 0
-	var s_rulename []string
 	query := map[string]interface{}{
 		"s_name": s_name,
 	}
@@ -84,6 +86,7 @@ func (f *Front) ProjectSave() {
 			"s_createname": username,                      //创建人
 			"s_status":     "未开始",                         //项目状态
 			"i_createtime": time.Now().Unix(),             //创建时间
+			"s_importtype": "excel",                       //导入类型
 		}
 	} else if stype == "coll" { //数据库导入
 		historyid := f.GetString("s_historyid")
@@ -100,21 +103,25 @@ func (f *Front) ProjectSave() {
 			"s_createname": username,                      //创建人
 			"s_status":     "未开始",                         //项目状态
 			"i_createtime": time.Now().Unix(),             //创建时间
+			"s_importtype": "coll",                        //导入类型
+			"s_historyid":  historyid,                     //源数据集标识
 		}
 	} else if stype == "edit" { //编辑保存
-		s_entname = f.GetString("s_entname")       //公司名称
+		//s_entname = f.GetString("s_entname")       //公司名称
 		s_departname = f.GetString("s_departname") //部门名称
 		rulename := f.GetString("s_rulename")      //规则名称
 		s_rulename = strings.Split(rulename, ",")
 		s_personname := f.GetString("s_personname")
 		fields := f.GetString("v_fields")
-		var v_fields []string
-		if json.Unmarshal([]byte(fields), &v_fields) != nil {
-			qu.Debug("标注字段解析失败")
+		v_fields := map[string]interface{}{}
+		if err := json.Unmarshal([]byte(fields), &v_fields); err != nil {
+			qu.Debug("V_Filelds Unmarshal Failed：", err)
+			f.ServeJson(map[string]interface{}{"success": false})
+			return
 		}
 		set = map[string]interface{}{
-			"s_name":       s_name,                        //项目名称
-			"s_entname":    s_entname,                     //公司名称
+			//"s_name":       s_name,                        //项目名称
+			//"s_entname":    s_entname,                     //公司名称
 			"s_departname": s_departname,                  //部门名称
 			"s_rulename":   strings.Join(s_rulename, ","), //规则名称
 			"v_fields":     v_fields,                      //标注字段
@@ -125,7 +132,154 @@ func (f *Front) ProjectSave() {
 			//"i_completetime",//结束时间
 		}
 	}
-	util.Mgo.Update("f_project", query, map[string]interface{}{"$set": set}, true, false)
+	b := util.Mgo.Update(util.PROJECTCOLLNAME, query, map[string]interface{}{"$set": set}, true, false)
+	qu.Debug("Create Project：", b)
+	//返回信息
+	if stype == "edit" {
+		f.ServeJson(map[string]interface{}{"success": b})
+	} else {
+		f.ServeJson(map[string]interface{}{"success": success, "msg": msg, "importnum": importDataNum, "successnum": successNum, "failnum": int64(importDataNum) - successNum})
+	}
+}
+
+// ProjectClear 项目清洗
+func (f *Front) ProjectClear() {
+	defer qu.Catch()
+	projectid := f.GetString("s_projectid") //项目id
+	project, _ := util.Mgo.FindById(util.PROJECTCOLLNAME, projectid, map[string]interface{}{"s_status": 1})
+	if project != nil && len(*project) > 0 {
+		if status := qu.ObjToString((*project)["s_status"]); status == "未开始" {
+			//TODO:调用数据质量评估接口
+			//点击清洗更新项目状态为进行中
+			b := util.Mgo.UpdateById(util.PROJECTCOLLNAME, projectid, map[string]interface{}{"$set": map[string]interface{}{"s_status": "进行中", "i_starttime": time.Now().Unix()}})
+			qu.Debug("Update Porject："+projectid+"	Status Success：", b)
+		}
+	} else {
+		qu.Debug("Search Porject Failed：", projectid)
+		f.ServeJson("查询项目信息失败")
+		return
+	}
+	sourceinfo := f.GetString("s_sourceinfo")                                                                      //数据源表
+	noTagAllDataNum := util.Mgo.Count(sourceinfo, map[string]interface{}{"b_istagging": false})                    //达标数据总量
+	noTagGiveDataNum := util.Mgo.Count(sourceinfo, map[string]interface{}{"b_istagging": false, "b_isgive": true}) //达标数据已分发量
+	noTagNoGiveDataNum := noTagAllDataNum - noTagGiveDataNum                                                       //达标待分发量
+	tagAllDataNum := util.Mgo.Count(sourceinfo, map[string]interface{}{"b_istagging": true})                       //未达标数据总量
+	tagGiveDataNum := util.Mgo.Count(sourceinfo, map[string]interface{}{"b_istagging": true, "b_isgive": true})    //未达标数据已分发量
+	tagNoGiveDataNum := tagAllDataNum - tagGiveDataNum                                                             //未达标待分发量
+	allGiveDataNum := noTagGiveDataNum + tagGiveDataNum                                                            //总分发量
+	allNoGiveDataNum := noTagNoGiveDataNum + tagNoGiveDataNum                                                      //总待分发量
+	allDataNum := allGiveDataNum + allNoGiveDataNum
+	f.T["allDataNum"] = allDataNum
+	f.T["allGiveDataNum"] = allGiveDataNum
+	f.T["allNoGiveDataNum"] = allNoGiveDataNum
+	f.T["noTagAllDataNum"] = noTagAllDataNum
+	f.T["noTagGiveDataNum"] = noTagGiveDataNum
+	f.T["noTagNoGiveDataNum"] = noTagNoGiveDataNum
+	f.T["tagAllDataNum"] = tagAllDataNum
+	f.T["tagGiveDataNum"] = tagGiveDataNum
+	f.T["tagNoGiveDataNum"] = tagNoGiveDataNum
+	f.T["s_projectid"] = projectid
+	_ = f.Render("user/user_group.html", &f.T)
+}
+
+// ProjectTaskList 用户组任务分发列表
+func (f *Front) ProjectTaskList() {
+	defer qu.Catch()
+	projectid := f.GetString("s_projectid") //项目id
+	status := f.GetString("s_status")       //任务状态
+	searchStr := f.GetString("search[value]")
+	search := strings.TrimSpace(searchStr)
+	start, _ := f.GetInteger("start")
+	limit, _ := f.GetInteger("length")
+	draw, _ := f.GetInteger("draw")
+	query := map[string]interface{}{
+		"s_projectid": projectid,
+	}
+	if status != "-1" {
+		query["s_status"] = status
+	}
+	if search != "" {
+		query["$or"] = []interface{}{
+			map[string]interface{}{"s_groupname": map[string]interface{}{"$regex": search}},
+		}
+	}
+	list, _ := util.Mgo.Find(util.TASKCOLLNAME, query, nil, nil, false, start, limit)
+	count := util.Mgo.Count(util.TASKCOLLNAME, query)
+	f.ServeJson(map[string]interface{}{"draw": draw, "data": *list, "recordsFiltered": count, "recordsTotal": count})
+}
+
+// ProjectTaskSave 用户组任务分发
+func (f *Front) ProjectTaskSave() {
+	defer qu.Catch()
+	var groupArr []map[string]interface{}
+	var taskArr []map[string]interface{}
+	var groupIdArr []string
+	success := false
+	msg := ""
+	user := f.GetSession("user").(map[string]interface{})
+	username := qu.ObjToString(user["s_name"])                                                  //当前登录用户
+	projectid := f.GetString("s_projectid")                                                     //项目标识
+	projectname := f.GetString("s_projectname")                                                 //项目名称
+	sourceinfo := f.GetString("s_sourceinfo")                                                   //源数据表
+	sourcetaskinfo := "s_sourcetaskinfo_" + strings.ReplaceAll(sourceinfo, "s_sourceinfo_", "") //任务日志表
+	group := f.GetString("s_group")
+	if err := json.Unmarshal([]byte(group), &groupArr); err != nil {
+		qu.Debug("V_Filelds Unmarshal Failed：", err)
+	} else {
+		for _, groupInfo := range groupArr {
+			groupId := qu.ObjToString(groupInfo["s_groupid"])
+			groupIdArr = append(groupIdArr, groupId)
+			task := map[string]interface{}{
+				"s_projectid":      projectid,                                 //项目标识
+				"s_projectname":    projectname,                               //项目名称
+				"s_status":         "未开始",                                     //任务状态
+				"s_personid":       qu.ObjToString(groupInfo["s_personid"]),   //任务负责人标识
+				"s_personname":     qu.ObjToString(groupInfo["s_personname"]), //任务负责人
+				"s_groupname":      qu.ObjToString(groupInfo["s_groupname"]),  //用户组名称
+				"s_groupid":        groupId,                                   //用户组标识
+				"i_givenum":        qu.ObjToString(groupInfo["i_givenum"]),    //分发数据量
+				"s_createname":     username,                                  //创建人
+				"i_createtime":     time.Now().Unix(),                         //创建时间
+				"s_progress":       "0%",                                      //完成进度
+				"s_sourceinfo":     sourceinfo,                                //源数据表
+				"s_sourcetaskinfo": sourcetaskinfo,                            //任务日志表
+			}
+			taskArr = append(taskArr, task)
+		}
+	}
+	//分发数据后更新项目中用户组标识信息
+	success = util.Mgo.UpdateById(util.PROJECTCOLLNAME, projectid, map[string]interface{}{
+		"$push": map[string]interface{}{
+			"v_groupids": map[string]interface{}{
+				"$each": groupIdArr,
+			},
+		},
+	})
+	if !success {
+		msg = "更新项目:" + projectname + "用户组标识失败"
+	} else { //分发任务
+		success = util.Mgo.SaveBulk(util.TASKCOLLNAME, taskArr...)
+		msg = "任务分发成功"
+	}
+	qu.Debug("Msg：", msg)
+	f.ServeJson(map[string]interface{}{"success": success, "msg": msg})
+}
+
+// ProjectGetEntnameList 模糊查询公司名称
+func (f *Front) ProjectGetEntnameList() {
+	defer qu.Catch()
+	var entnameList []string
+	entname := f.GetString("entname")
+	query := map[string]interface{}{
+		"username": map[string]interface{}{
+			"$regex": entname,
+		},
+	}
+	list, _ := util.MgoJy.Find(util.JyUser, query, nil, map[string]interface{}{"username": 1}, false, -1, -1)
+	for _, l := range *list {
+		entnameList = append(entnameList, qu.ObjToString(l["username"]))
+	}
+	f.ServeJson(map[string]interface{}{"entname": entnameList})
 }
 
 //ImportDataByExcel 通过excel获取数据源
@@ -179,7 +333,7 @@ func ImportDataByExcel(s_sourceinfo string, mf multipart.File, success *bool, ms
 		idInfoMap[id] = tmp
 	}
 	importDataNum = len(idInfoMap)
-	qu.Debug("共加载Excel数据：", importDataNum, "条")
+	qu.Debug("Load Excel Count：", importDataNum)
 	if importDataNum > 0 {
 		GetDataById(idInfoMap, "excel", s_sourceinfo, success, msg, successNum)
 	} else {
@@ -193,6 +347,7 @@ func ImportDataByExcel(s_sourceinfo string, mf multipart.File, success *bool, ms
 //ImportDataByColl 通过表获取数据源
 func ImportDataByColl(s_sourceinfo, historyid string, success *bool, msg *string, successNum *int64) (departname, entname string, rulename []string, importDataNum int) {
 	defer qu.Catch()
+	rulenameMap := map[string]bool{}
 	sess := util.MgoJy.GetMgoConn()
 	defer util.MgoJy.DestoryMongoConn(sess)
 	ch := make(chan bool, 3)
@@ -225,8 +380,10 @@ func ImportDataByColl(s_sourceinfo, historyid string, success *bool, msg *string
 				user, _ := util.MgoJy.FindOne(util.JyUser, map[string]interface{}{"appid": appid})
 				entname = qu.ObjToString((*user)["username"]) //公司名称
 			}
+			rname := qu.ObjToString(tmp["rulename"])
 			lock.Lock()
-			rulename = append(rulename, qu.ObjToString(tmp["rulename"])) //规则名称
+			rulenameMap[rname] = true
+			//rulename = append(rulename, qu.ObjToString(tmp["rulename"])) //规则名称
 			idInfoMap[id] = needField
 			lock.Unlock()
 		}(tmp)
@@ -236,6 +393,9 @@ func ImportDataByColl(s_sourceinfo, historyid string, success *bool, msg *string
 		tmp = map[string]interface{}{}
 	}
 	wg.Wait()
+	for r, _ := range rulenameMap {
+		rulename = append(rulename, r)
+	}
 	importDataNum = len(idInfoMap) //查询数据总数
 	if importDataNum > 0 {
 		GetDataById(idInfoMap, "coll", s_sourceinfo, success, msg, successNum)
@@ -249,6 +409,7 @@ func ImportDataByColl(s_sourceinfo, historyid string, success *bool, msg *string
 //GetDataById 通过id集从bidding、extract、project获取数据所有信息
 func GetDataById(idsInfo map[string]map[string]interface{}, importType, s_sourceinfo string, success *bool, msg *string, successNum *int64) {
 	*success = true
+	var msgArr []string
 	wg := &sync.WaitGroup{}
 	lock := &sync.Mutex{}
 	ch := make(chan bool, 10)
@@ -300,22 +461,22 @@ func GetDataById(idsInfo map[string]map[string]interface{}, importType, s_source
 					UpdateMarkColl(bidData, markData, &tagInfoMap, &baseInfoMap) //比对更新数据
 				} else {
 					baseInfoMap["i_ckdata"] = 0 //设置ck_data默认值0
-				}
-				//多包、中标候选人、标的信息是否抽取
-				if packageMap, ok := (*bidData)["package"].(map[string]interface{}); ok && len(packageMap) > 0 {
-					baseInfoMap["pkgisext"] = true
-				} else {
-					baseInfoMap["pkgisext"] = false
-				}
-				if winorderArr, ok := (*bidData)["winnerorder"].([]interface{}); ok && len(winorderArr) > 0 {
-					baseInfoMap["wodrisext"] = true
-				} else {
-					baseInfoMap["wodrisext"] = false
-				}
-				if purchArr, ok := (*bidData)["purchasinglist"].([]interface{}); ok && len(purchArr) > 0 {
-					baseInfoMap["pclisext"] = true
-				} else {
-					baseInfoMap["pclisext"] = false
+					//多包、中标候选人、标的信息是否抽取
+					//if packageMap, ok := (*bidData)["package"].(map[string]interface{}); ok && len(packageMap) > 0 {
+					//	baseInfoMap["b_pkgisext"] = true
+					//} else {
+					//	baseInfoMap["b_pkgisext"] = false
+					//}
+					//if winorderArr, ok := (*bidData)["winnerorder"].([]interface{}); ok && len(winorderArr) > 0 {
+					//	baseInfoMap["b_wodrisext"] = true
+					//} else {
+					//	baseInfoMap["b_wodrisext"] = false
+					//}
+					//if purchArr, ok := (*bidData)["purchasinglist"].([]interface{}); ok && len(purchArr) > 0 {
+					//	baseInfoMap["b_pclisext"] = true
+					//} else {
+					//	baseInfoMap["b_pclisext"] = false
+					//}
 				}
 				//合并导入表中客户所需的字段
 				if len(tmp) > 0 {
@@ -385,14 +546,17 @@ func GetDataById(idsInfo map[string]map[string]interface{}, importType, s_source
 						(*bidData)["info"] = infolist
 					}
 				} else {
-					qu.Debug("projectset find err", projectId)
+					qu.Debug("Projectset Find Error", projectId)
 				}
 				baseInfoMap["id"] = id
-				delete(*bidData, "id")
+				_id := (*bidData)["_id"]
+				delete(*bidData, "_id")
 				//保存数据
-				baseInfoMap["_id"] = (*bidData)["_id"]
+				baseInfoMap["_id"] = _id
 				baseInfoMap["v_datainfo"] = bidData
-				baseInfoMap["v_taginfo"] = tagInfoMap
+				if len(tagInfoMap) > 0 {
+					baseInfoMap["v_taginfo"] = tagInfoMap
+				}
 				baseInfoMap["i_createtime"] = time.Now().Unix()
 				baseInfoMap["b_isgive"] = false //是否分配
 				baseInfoMap["b_istag"] = false  //是否已标注
@@ -403,9 +567,11 @@ func GetDataById(idsInfo map[string]map[string]interface{}, importType, s_source
 					lock.Lock()
 					*success = false
 					if importType == "excel" {
-						*msg += "第" + fmt.Sprint(num+2) + "行未保存成功数据_id：" + id + ";\n"
+						msgArr = append(msgArr, "第"+fmt.Sprint(num+2)+"行未导入id："+id)
+						//*msg += "第" + fmt.Sprint(num+2) + "行未保存成功数据_id：" + id + ";\n"
 					} else {
-						*msg += "未保存成功数据_id:" + id + ";\n"
+						msgArr = append(msgArr, "未导入id:"+id)
+						//*msg += "未保存成功数据_id:" + id + ";\n"
 					}
 					lock.Unlock()
 				}
@@ -413,18 +579,22 @@ func GetDataById(idsInfo map[string]map[string]interface{}, importType, s_source
 				lock.Lock()
 				*success = false
 				if importType == "excel" {
-					*msg += "第" + fmt.Sprint(num+2) + "行未查询到数据：" + id + ";\n"
+					msgArr = append(msgArr, "第"+fmt.Sprint(num+2)+"行未查询id："+id)
+					//*msg += "第" + fmt.Sprint(num+2) + "行未查询到数据：" + id + ";\n"
 				} else {
-					*msg += "未查询到数据_id:" + id + ";\n"
+					msgArr = append(msgArr, "未查询id:"+id)
+					//*msg += "未查询到数据_id:" + id + ";\n"
 				}
 				lock.Unlock()
 			}
 		}(id, info)
 	}
 	wg.Wait()
+	sort.Strings(msgArr)
+	*msg = strings.Join(msgArr, ";\n")
 }
 
-//更新数据
+// UpdateMarkColl 更新数据
 func UpdateMarkColl(bidData, markData, tagInfoMap, baseInfoMap *map[string]interface{}) {
 	defer qu.Catch()
 	ckdata := qu.IntAll((*markData)["i_ckdata"])
@@ -435,8 +605,8 @@ func UpdateMarkColl(bidData, markData, tagInfoMap, baseInfoMap *map[string]inter
 			(*bidData)[fk] = v_datainfo[fk] //字段更新
 		}
 	}
-	(*tagInfoMap)["v_taginfo"] = v_taginfo //marked中已有的标注信息保存到新数据上
-	if ckdata == 2 {                       //某些字段已标注
+	(*tagInfoMap) = v_taginfo //marked中已有的标注信息保存到新数据上
+	if ckdata == 2 {          //某些字段已标注
 		(*baseInfoMap)["i_ckdata"] = 0 //marked表中该条数据如果为字段验证，临时表ck_data:0;若为数据验证ck_data:1
 	} else if ckdata == 1 {
 		(*baseInfoMap)["i_ckdata"] = 1

src/front/task.go

@@ -0,0 +1 @@
+package front

src/main.go

@@ -33,5 +33,11 @@ func init() {
 }
 
 func main() {
+	success := false
+	msg := ""
+	successNum := int64(0)
+	s_departname, s_entname, s_rulename, importDataNum := front.ImportDataByColl("test", "615278a07f395572067fc599", &success, &msg, &successNum)
+	qu.Debug(s_departname, s_entname, s_rulename, importDataNum)
+	qu.Debug(success, msg, successNum)
 	xweb.Run(":" + qu.ObjToString(Sysconfig["port"]))
 }

src/util/common.go

@@ -8,10 +8,6 @@ import (
 	qu "qfw/util"
 )
 
-var SE = qu.SimpleEncrypt{Key: "topJYBX2019"}
-
-const BIDDINGSTARTID = "5a862f0640d2d9bbe88e3cec"
-
 //oss
 var (
 	ossEndpoint        = "oss-cn-beijing.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com

src/util/config.go

@@ -31,6 +31,11 @@ var (
 	CustomerFieldMap_EH map[string]string
 	CustomerFieldMap_HE map[string]string
 )
+var SE = qu.SimpleEncrypt{Key: "topJYBX2019"}
+
+const BIDDINGSTARTID = "5a862f0640d2d9bbe88e3cec"
+const PROJECTCOLLNAME = "f_project"
+const TASKCOLLNAME = "f_task"
 
 func InitConfig() {
 	Mgo = &mongodb.MongodbSim{
@@ -93,7 +98,7 @@ func InitConfig() {
 	CustomerFieldMap_HE = map[string]string{}
 	for k, v := range Sysconfig["customerfield"].(map[string]interface{}) {
 		val := v.(string)
-		CustomerFieldMap_EH[k] = val
-		CustomerFieldMap_HE[val] = k
+		CustomerFieldMap_HE[k] = val
+		CustomerFieldMap_EH[val] = k
 	}
 }