pici 任务修改

src/config.json

@@ -31,10 +31,10 @@
         "size": 10
     },
     "es":{
-        "addr": "http://127.0.0.1:19805",
+        "addr": "http://192.168.3.241:9205",
         "index": "bidding",
-        "user": "es_all",
-        "password": "TopJkO2E_d1x",
+        "user": "",
+        "password": "",
         "pool": 12
     },
     "customerfield":{

src/front/front.go

@@ -67,6 +67,7 @@ type Front struct {
 	projectQualityAssessment xweb.Mapper `xweb:"/front/project/qualityAssessment"` //数据质量评估
 	projectGroupTaskSave     xweb.Mapper `xweb:"/front/project/task/save"`         //用户组任务分发
 	projectGroupTaskList     xweb.Mapper `xweb:"/front/project/task/list"`         //用户组任务分发列表
+	projectPiciList          xweb.Mapper `xweb:"/front/project/pici/list"`         //项目 pici 列表
 	projectGroupTaskRetrieve xweb.Mapper `xweb:"/front/project/task/retrieve"`     //用户组任务收回
 	projectGroupTaskRepulse  xweb.Mapper `xweb:"/front/project/task/repulse"`      //用户组任务打回
 	dataRepulse              xweb.Mapper `xweb:"/front/task/data/repulse"`

src/front/group.go

@@ -286,33 +286,39 @@ func (f *Front) GroupTaskDeliver() {
 func (f *Front) TaskSyncJy() {
 	defer qu.Catch()
 	pid := f.GetString("pid")
-	groupTaskId := f.GetString("taskid")
-	c := syncJyData(groupTaskId, pid)
-	qu.Debug(util.DATACOLLNAME, groupTaskId)
-	//info, _ := util.Mgo.FindById(util.TASKCOLLNAME, groupTaskId, bson.M{"i_givenum": 1})
-	if c > 0 {
-		util.Mgo.UpdateById(util.TASKCOLLNAME, groupTaskId, bson.M{"$set": bson.M{"sendflag": true}})
-		f.ServeJson(map[string]interface{}{"success": true, "msg": "数据推送成功", "count": c})
+	appid := f.GetString("appid")
+	pici, _ := f.GetInt("pici")
+	project, _ := util.Mgo.FindById(util.PROJECTCOLLNAME, pid, bson.M{"v_add_data": 1})
+	importNum := 0
+	for _, v := range (*project)["v_add_data"].([]interface{}) {
+		v1 := v.(map[string]interface{})
+		if qu.Int64All(v1["pici"]) == pici {
+			importNum = qu.IntAll(v1["i_importnum"])
+			break
+		}
+	}
+	tagNum := util.Mgo.Count(util.DATACOLLNAME, bson.M{"appid": appid, "pici": pici, "b_istag": true})
+	if importNum > 0 && importNum == tagNum {
+		c := syncJyData(pici, appid)
+		if c > 0 {
+			//util.Mgo.UpdateById(util.TASKCOLLNAME, groupTaskId, bson.M{"$set": bson.M{"sendflag": true}})
+			f.ServeJson(map[string]interface{}{"success": true, "msg": "数据推送成功", "count": c})
+		} else {
+			f.ServeJson(map[string]interface{}{"success": false, "msg": "数据推送失败"})
+		}
 	} else {
-		f.ServeJson(map[string]interface{}{"success": false, "msg": "数据推送失败"})
+		f.ServeJson(map[string]interface{}{"success": false, "msg": "数据未清洗完"})
 	}
-
 }
 
-func syncJyData(tid, pid string) int {
+func syncJyData(pici int64, appid string) int {
 	sess := util.Mgo.GetMgoConn()
 	defer util.Mgo.DestoryMongoConn(sess)
 
 	ch := make(chan bool, 5)
 	wg := &sync.WaitGroup{}
 
-	var q bson.M
-	if pid == "650c310bc88c29b90a54b1c4" {
-		// 联通中标数据 特殊处理
-		q = bson.M{"s_grouptaskid": tid, "is_push": 1}
-	} else {
-		q = bson.M{"s_grouptaskid": tid, "sendflag": nil}
-	}
+	q := bson.M{"pici": pici, "appid": appid}
 	query := sess.DB(util.Mgo.DbName).C(util.DATACOLLNAME).Find(q).Select(nil).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); query.Next(&tmp); count++ {
@@ -799,14 +805,15 @@ func (f *Front) CheckResultWb() {
 
 func (f *Front) GroupExportData() {
 	defer qu.Catch()
-	tid := f.GetString("taskid")
-	q := bson.M{"s_grouptaskid": tid}
+	appid := f.GetString("appid")
+	pici, _ := f.GetInt("pici")
+	q := bson.M{"appid": appid, "pici": pici}
 	count := util.Mgo.Count(util.DATACOLLNAME, q)
 	if count > 0 {
 		file, err := xlsx.OpenFile("web/model/taskexportdata.xlsx")
 		if err != nil {
 			qu.Debug("Load Excel Model Error")
-			f.ServeJson("加载脚本失败")
+			f.ServeJson("加载文件失败")
 			return
 		}
 		sheet := file.Sheets[0]

src/front/project.go

@@ -67,6 +67,19 @@ func (f *Front) ProjectList() {
 	}
 }
 
+type ImportTask struct {
+	success       bool        //导入数据是否成功
+	msg           string      //异常信息
+	successNum    int         //导入成功条数
+	importDataNum int         //查询数量
+	appId         string      //客户标识(客户管理平台)
+	pici          int64       //导入批次时间
+	newCreate     bool        //新建项目
+	importType    string      // excel、coll
+	idInfoArr     []util.Data //记录数据id及需要保存的字段信息
+
+}
+
 // ProjectSave 项目保存
 func (f *Front) ProjectSave() {
 	defer qu.Catch()
@@ -76,11 +89,6 @@ func (f *Front) ProjectSave() {
 		f.ServeJson(map[string]interface{}{"success": false, "msg": "缺少项目名称字段"})
 		return
 	}
-	success := false        //导入数据是否成功
-	msg := ""               //异常信息
-	successNum := int64(0)  //导入成功条数
-	importDataNum := 0      //查询数量
-	appid := ""             // 客户标识(客户管理平台)
 	var s_rulename []string //规则
 	user := f.GetSession("user").(map[string]interface{})
 	username := qu.ObjToString(user["s_login"]) //当前登录用户
@@ -90,6 +98,11 @@ func (f *Front) ProjectSave() {
 		"s_name": s_name,
 	}
 	set := map[string]interface{}{}
+	task := &ImportTask{
+		newCreate:  true,
+		importType: stype,
+		pici:       time.Now().Unix(),
+	}
 	//导入数据
 	if stype == "excel" { //excel导入
 		s_entname = f.GetString("s_entname") //公司名称
@@ -103,26 +116,35 @@ func (f *Front) ProjectSave() {
 		mf, _, err := f.GetFile("xlsx")
 		qu.Debug(s_entname, s_departname, s_rulename)
 		if err == nil {
-			importDataNum, appid = ImportDataByExcel(mf, &success, &msg, &successNum, true, appid)
+			task.ImportDataByExcel(mf)
 		}
-		if importDataNum == 0 {
+		if task.importDataNum == 0 {
 			f.ServeJson(map[string]interface{}{"success": false, "msg": "文档缺少企业客户id字段"})
 			return
 		}
+		var addDataTag []map[string]interface{}
+		addDataTag = append(addDataTag, map[string]interface{}{
+			"s_importtype":   stype,
+			"i_importnum":    task.importDataNum,
+			"s_updateperson": username,
+			"i_updatetime":   time.Now().Unix(),
+			"pici":           task.pici,
+		})
 		//保存项目信息
 		set = map[string]interface{}{
 			"s_name":         s_name,                        //项目名称
 			"s_entname":      s_entname,                     //公司名称
 			"s_departname":   s_departname,                  //部门名称
 			"s_rulename":     strings.Join(s_rulename, ","), //规则名称
-			"i_importnum":    importDataNum,                 //导入数量
+			"i_importnum":    task.importDataNum,            //导入数量
 			"s_cleartype":    clearType,
-			"appid":          appid,
+			"appid":          task.appId,
 			"s_createname":   username,          //创建人
 			"s_status":       "未开始",             //项目状态
 			"i_createtime":   time.Now().Unix(), //创建时间
 			"s_importtype":   "excel",           //导入类型
 			"b_isassessment": false,             //是否进行了质量评估
+			"v_add_data":     addDataTag,
 		}
 	} else if stype == "coll" { //数据库导入
 		historyid := f.GetString("s_historyid")
@@ -145,41 +167,39 @@ func (f *Front) ProjectSave() {
 			f.ServeJson(map[string]interface{}{"success": false, "msg": "查询数据量为0！"})
 			return
 		}
-		s_departname, s_entname, appid, s_rulename, importDataNum = ImportDataByColl(collName, query1, &success, &msg, &successNum, true)
-		appidNum := ""
-		if util.Appid[appid] != 0 {
-			i := util.Appid[appid]
-			i++
-			appidNum = fmt.Sprintf("%s-%03d", appid, i)
-			util.Appid[appid] = i
-		} else {
-			appidNum = fmt.Sprintf("%s-%03d", appid, 1)
-			util.Appid[appid] = 1
-		}
-		qu.Debug(s_departname, s_entname, appidNum, s_rulename, importDataNum)
+		s_departname, s_entname, s_rulename = task.ImportDataByColl(collName, query1)
+		qu.Debug(s_departname, s_entname, task.appId, s_rulename, task.importDataNum)
+		var addDataTag []map[string]interface{}
+		addDataTag = append(addDataTag, map[string]interface{}{
+			"s_importtype":   stype,
+			"i_importnum":    task.importDataNum,
+			"s_updateperson": username,
+			"i_updatetime":   time.Now().Unix(),
+			"pici":           task.pici,
+		})
 		//保存项目信息
 		set = map[string]interface{}{
 			"s_name":         s_name,                        //项目名称
 			"s_entname":      s_entname,                     //公司名称
 			"s_departname":   s_departname,                  //部门名称
 			"s_rulename":     strings.Join(s_rulename, ","), //规则名称
-			"i_importnum":    importDataNum,                 //导入数量
-			"appid":          appid,
+			"i_importnum":    task.importDataNum,            //导入数量
+			"appid":          task.appId,
 			"s_createname":   username,          //创建人
 			"s_status":       "未开始",             //项目状态
 			"i_createtime":   time.Now().Unix(), //创建时间
 			"s_importtype":   "coll",            //导入类型
 			"s_historyid":    historyid,         //源数据集标识
 			"b_isassessment": true,              //是否进行了质量评估
+			"v_add_data":     addDataTag,
 		}
 	} else if stype == "edit" { //编辑保存
-		success = true
+		task.success = true
 		//s_entname = f.GetString("s_entname")       //公司名称
 		s_departname = f.GetString("s_departname") //部门名称
 		rulename := f.GetString("s_rulename")      //规则名称
 		s_rulename = strings.Split(rulename, ",")
 		s_personname := f.GetString("s_personname")
-
 		set = map[string]interface{}{
 			//"s_name":       s_name,                        //项目名称
 			//"s_entname":    s_entname,                     //公司名称
@@ -192,21 +212,21 @@ func (f *Front) ProjectSave() {
 			//"i_completetime",//结束时间
 		}
 	}
-	if success {
-		success = util.Mgo.Update(util.PROJECTCOLLNAME, query, map[string]interface{}{"$set": set}, true, false)
-		if !success { //保存项目失败
-			msg = "新建项目失败\n" + msg
+	if task.success {
+		task.success = util.Mgo.Update(util.PROJECTCOLLNAME, query, map[string]interface{}{"$set": set}, true, false)
+		if !task.success { //保存项目失败
+			task.msg = "新建项目失败\n" + task.msg
 		} else {
-			msg = "保存项目成功"
+			task.msg = "保存项目成功"
 		}
 	}
-	//qu.Debug("Msg：", msg)
+
 	//返回信息
 	if stype == "edit" {
-		f.ServeJson(map[string]interface{}{"success": success, "msg": msg})
+		f.ServeJson(map[string]interface{}{"success": task.success, "msg": task.msg})
 	} else {
-		qu.Debug("Create Project：", success, "importnum：", importDataNum, "successnum：", successNum, "failnum：", int64(importDataNum)-successNum)
-		f.ServeJson(map[string]interface{}{"success": success, "msg": msg, "importnum": importDataNum, "successnum": successNum, "failnum": int64(importDataNum) - successNum})
+		qu.Debug("Create Project：", task.success, "importnum：", task.importDataNum, "successnum：", task.successNum, "failnum：", task.importDataNum-task.successNum)
+		f.ServeJson(map[string]interface{}{"success": task.success, "msg": task.msg, "importnum": task.importDataNum, "successnum": task.successNum, "failnum": task.importDataNum - task.successNum})
 	}
 }
 
@@ -219,13 +239,15 @@ func (f *Front) ProjectAddData() {
 		stype := f.GetString("s_type")
 		info, _ := util.Mgo.FindById(util.PROJECTCOLLNAME, projectid, nil)
 		if len(*info) > 0 {
-			success := false                         //导入数据是否成功
-			msg := ""                                //异常信息
-			importDataNum, successNum := 0, int64(0) //导入成功条数
+			task := &ImportTask{
+				success:    false,
+				pici:       time.Now().Unix(),
+				importType: stype,
+			}
 			if stype == "excel" {
 				mf, _, err := f.GetFile("xlsx")
 				if err == nil {
-					importDataNum, _ = ImportDataByExcel(mf, &success, &msg, &successNum, false, qu.ObjToString((*info)["appid"]))
+					task.ImportDataByExcel(mf)
 					var addDataTag []map[string]interface{}
 					if (*info)["v_add_tag"] != nil {
 						arr := qu.ObjArrToMapArr((*info)["v_add_data"].([]interface{}))
@@ -233,9 +255,10 @@ func (f *Front) ProjectAddData() {
 					} else {
 						addDataTag = append(addDataTag, map[string]interface{}{
 							"s_importtype":   stype,
-							"i_importnum":    importDataNum,
+							"i_importnum":    task.importDataNum,
 							"s_updateperson": username,
 							"i_updatetime":   time.Now().Unix(),
+							"pici":           task.pici,
 						})
 					}
 					s_status := ""
@@ -245,10 +268,11 @@ func (f *Front) ProjectAddData() {
 						s_status = "进行中"
 					}
 					set := map[string]interface{}{
-						"i_importnum":  importDataNum + qu.IntAll((*info)["i_importnum"]), //导入数量
-						"s_status":     s_status,                                          //项目状态
+						"i_importnum":  task.importDataNum + qu.IntAll((*info)["i_importnum"]), //导入数量
+						"s_status":     s_status,                                               //项目状态
 						"i_updatetime": time.Now().Unix(),
 						"v_add_data":   addDataTag,
+						"pici":         task.pici,
 					}
 					util.Mgo.UpdateById(util.PROJECTCOLLNAME, projectid, map[string]interface{}{"$set": set})
 				}
@@ -274,9 +298,9 @@ func (f *Front) ProjectAddData() {
 					f.ServeJson(map[string]interface{}{"success": false, "msg": "查询数据量为0！"})
 					return
 				}
-				_, _, _, _, importDataNum = ImportDataByColl(collName, query1, &success, &msg, &successNum, false)
-				if !success {
-					f.ServeJson(map[string]interface{}{"success": success, "msg": msg})
+				task.ImportDataByColl(collName, query1)
+				if !task.success {
+					f.ServeJson(map[string]interface{}{"success": task.success, "msg": task.msg})
 					return
 				}
 				var addDataTag []map[string]interface{}
@@ -286,9 +310,10 @@ func (f *Front) ProjectAddData() {
 				} else {
 					addDataTag = append(addDataTag, map[string]interface{}{
 						"s_importtype":   stype,
-						"i_importnum":    importDataNum,
+						"i_importnum":    task.importDataNum,
 						"s_updateperson": username,
 						"i_updatetime":   time.Now().Unix(),
+						"pici":           task.pici,
 					})
 				}
 				s_status := ""
@@ -299,14 +324,15 @@ func (f *Front) ProjectAddData() {
 				}
 				//保存项目信息
 				set := map[string]interface{}{
-					"i_importnum":  importDataNum + qu.IntAll((*info)["i_importnum"]), //导入数量
-					"s_status":     s_status,                                          //项目状态
+					"i_importnum":  task.importDataNum + qu.IntAll((*info)["i_importnum"]), //导入数量
+					"s_status":     s_status,                                               //项目状态
 					"i_updatetime": time.Now().Unix(),
 					"v_add_data":   addDataTag,
+					"pici":         task.pici,
 				}
 				util.Mgo.UpdateById(util.PROJECTCOLLNAME, projectid, map[string]interface{}{"$set": set})
 			}
-			f.ServeJson(map[string]interface{}{"success": success, "msg": msg, "importnum": importDataNum, "successnum": successNum, "failnum": int64(importDataNum) - successNum})
+			f.ServeJson(map[string]interface{}{"success": task.success, "msg": task.msg, "importnum": task.importDataNum, "successnum": task.successNum, "failnum": task.importDataNum - task.successNum})
 		} else {
 			f.ServeJson(map[string]interface{}{"success": false, "msg": "项目查询失败"})
 		}
@@ -486,6 +512,32 @@ func (f *Front) ProjectGroupTaskList() {
 	}
 }
 
+func (f *Front) ProjectPiciList() {
+	defer qu.Catch()
+	projectid := f.GetString("pid") //项目id
+	if f.Method() == "POST" {
+		project, _ := util.Mgo.FindById(util.PROJECTCOLLNAME, projectid, nil)
+		if (*project)["v_add_data"] != nil {
+			record := (*project)["v_add_data"].([]interface{})
+			var data []map[string]interface{}
+			for _, v := range record {
+				v1 := v.(map[string]interface{})
+				v1["projectname"] = (*project)["s_name"]
+				v1["projectid"] = projectid
+				v1["appid"] = (*project)["appid"]
+				q := map[string]interface{}{
+					"appid":   (*project)["appid"],
+					"pici":    v1["pici"],
+					"b_istag": true,
+				}
+				v1["i_tagnum"] = util.Mgo.Count(util.DATACOLLNAME, q)
+				data = append(data, v1)
+			}
+			f.ServeJson(map[string]interface{}{"data": data, "recordsFiltered": len(record), "recordsTotal": len(record)})
+		}
+	}
+}
+
 // ProjectGroupTaskSave 用户组任务分发
 func (f *Front) ProjectGroupTaskSave() {
 	defer qu.Catch()
@@ -504,7 +556,6 @@ func (f *Front) ProjectGroupTaskSave() {
 	appid := qu.ObjToString((*project)["appid"])
 	group := f.GetString("s_group")
 	stype := f.GetString("s_type")
-	qu.Debug("项目id：", projectid, " 项目名称：", projectname)
 	if err := json.Unmarshal([]byte(group), &groupArr); err != nil {
 		qu.Debug("GroupInfo Unmarshal Failed：", err)
 		msg = "用户组信息解析失败"
@@ -1015,7 +1066,7 @@ func UpdateSourceInfoByGroup(stype, appid string, groupIdInfo map[string]util.Ta
 }
 
 // ImportDataByExcel 通过excel获取数据源
-func ImportDataByExcel(mf multipart.File, success *bool, msg *string, successNum *int64, newCreate bool, appid string) (importDataNum int, appidNum string) {
+func (task *ImportTask) ImportDataByExcel(mf multipart.File) {
 	defer qu.Catch()
 	binary, _ := io.ReadAll(mf)
 	xls, _ := xlsx.OpenBinary(binary)
@@ -1024,7 +1075,7 @@ func ImportDataByExcel(mf multipart.File, success *bool, msg *string, successNum
 	idcolnum := -1
 	appidColnum := -1
 	cellFieldName := map[int]string{} //记录客户需求字段所在的列
-	idInfoArr := []util.Data{}        //记录数据id及需要保存的字段信息
+	idInfoArr := []util.Data{}
 	for rn, row := range rows {
 		if rn == 0 {
 			for index, cell := range row.Cells {
@@ -1042,10 +1093,10 @@ func ImportDataByExcel(mf multipart.File, success *bool, msg *string, successNum
 			if idcolnum == -1 {
 				break
 			}
-			if newCreate && appidColnum == -1 {
-				*success = false
-				*msg = "新增数据，企业客户id不能为空"
-				return 0, ""
+			if task.newCreate && appidColnum == -1 {
+				task.success = false
+				task.msg = "新增数据，企业客户id不能为空"
+				return
 			}
 			continue
 		}
@@ -1071,42 +1122,28 @@ func ImportDataByExcel(mf multipart.File, success *bool, msg *string, successNum
 			break
 		}
 		id = util.SE.DecodeString(id) //解密后id
-		idInfoArr = append(idInfoArr, util.Data{
+		task.idInfoArr = append(task.idInfoArr, util.Data{
 			ID:   id,
 			Info: tmp,
 		})
-		if appid == "" {
-			appid = qu.ObjToString(tmp["appid"])
+		if task.appId == "" {
+			task.appId = qu.ObjToString(tmp["appid"])
 		}
 	}
-	importDataNum = len(idInfoArr)
-	if newCreate {
-		if util.Appid[appid] != 0 {
-			i := util.Appid[appid]
-			i++
-			appidNum = fmt.Sprintf("%s-%03d", appid, i)
-			util.Appid[appid] = i
-		} else {
-			appidNum = fmt.Sprintf("%s-%03d", appid, 1)
-			util.Appid[appid] = 1
-		}
+	task.importDataNum = len(idInfoArr)
+	qu.Debug("Load Excel Count：", task.importDataNum, task.appId)
+	if task.importDataNum > 0 {
+		task.GetDataById()
 	} else {
-		appidNum = appid
-	}
-
-	qu.Debug("Load Excel Count：", importDataNum, appid)
-	if importDataNum > 0 {
-		GetDataById(idInfoArr, appidNum, "excel", success, msg, successNum)
-	} else {
-		*success = false
-		*msg = "查询数据失败"
+		task.success = false
+		task.msg = "查询数据失败"
 	}
 	idInfoArr = []util.Data{}
 	return
 }
 
 // ImportDataByColl 通过表获取数据源
-func ImportDataByColl(collName string, query map[string]interface{}, success *bool, msg *string, successNum *int64, newCreate bool) (departname, entname, appid string, rulename []string, importDataNum int) {
+func (task *ImportTask) ImportDataByColl(collName string, query map[string]interface{}) (departname, entname string, rulename []string) {
 	defer qu.Catch()
 	rulenameMap := map[string]bool{}
 	sess := util.MgoJy.GetMgoConn()
@@ -1114,7 +1151,6 @@ func ImportDataByColl(collName string, query map[string]interface{}, success *bo
 	ch := make(chan bool, 3)
 	wg := &sync.WaitGroup{}
 	lock := &sync.Mutex{}
-	idInfoArr := []util.Data{} //记录数据id及需要保存的字段信息
 	it := sess.DB(util.MgoJy.DbName).C(collName).Find(&query).Iter()
 	n := 0
 	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
@@ -1126,8 +1162,8 @@ func ImportDataByColl(collName string, query map[string]interface{}, success *bo
 				wg.Done()
 			}()
 			id := qu.ObjToString(tmp["id"]) //bidding id
-			if appid == "" {
-				appid = qu.ObjToString(tmp["appid"]) //根据appid查user表获取公司名称
+			if task.appId == "" {
+				task.appId = qu.ObjToString(tmp["appid"]) //根据appid查user表获取公司名称
 			}
 			departname = qu.ObjToString(tmp["departname"]) //部门名称。所有数据都应部门名称，若不一致，随机取
 			needField := map[string]interface{}{}
@@ -1137,7 +1173,7 @@ func ImportDataByColl(collName string, query map[string]interface{}, success *bo
 				}
 			}
 			if entname == "" { //获取一次公司名称即可
-				user, _ := util.MgoJy.FindOne(util.JyUser, map[string]interface{}{"appid": appid})
+				user, _ := util.MgoJy.FindOne(util.JyUser, map[string]interface{}{"appid": task.appId})
 				entname = qu.ObjToString((*user)["username"]) //公司名称
 			}
 			rname := qu.ObjToString(tmp["rulename"])
@@ -1145,9 +1181,8 @@ func ImportDataByColl(collName string, query map[string]interface{}, success *bo
 			for _, r := range strings.Split(rname, ",") {
 				rulenameMap[r] = true
 			}
-			//rulename = append(rulename, qu.ObjToString(tmp["rulename"])) //规则名称
-			//idInfoMap[id] = needField
-			idInfoArr = append(idInfoArr, util.Data{
+
+			task.idInfoArr = append(task.idInfoArr, util.Data{
 				ID:   id,
 				Info: needField,
 			})
@@ -1162,36 +1197,24 @@ func ImportDataByColl(collName string, query map[string]interface{}, success *bo
 	for r := range rulenameMap {
 		rulename = append(rulename, r)
 	}
-	importDataNum = len(idInfoArr) //查询数据总数
-	if util.Appid[appid] != 0 {
-		i := util.Appid[appid]
-		if newCreate { // 新增
-			i++
-		}
-		appid = fmt.Sprintf("%s-%03d", appid, i)
-		util.Appid[appid] = i
+	task.importDataNum = len(task.idInfoArr) //查询数据总数
+	if task.importDataNum > 0 {
+		task.GetDataById()
 	} else {
-		appid = fmt.Sprintf("%s-%03d", appid, 1)
-		util.Appid[appid] = 1
+		task.msg = "查询数据失败"
 	}
-	if importDataNum > 0 {
-		GetDataById(idInfoArr, appid, "coll", success, msg, successNum)
-	} else {
-		*msg = "查询数据失败"
-	}
-	idInfoArr = []util.Data{}
 	return
 }
 
 // GetDataById 通过id集从bidding、extract、project获取数据所有信息
-func GetDataById(idInfoArr []util.Data, appid, importType string, success *bool, msg *string, successNum *int64) {
-	*success = true
+func (task *ImportTask) GetDataById() {
+	var successNum int64
 	var msgArr []string
 	wg := &sync.WaitGroup{}
 	lock := &sync.Mutex{}
 	ch := make(chan bool, 10)
 	//num := int64(0)                                                   //计数
-	for i, data := range idInfoArr {
+	for i, data := range task.idInfoArr {
 		wg.Add(1)
 		ch <- true
 		go func(index int, tmpData util.Data) {
@@ -1216,14 +1239,11 @@ func GetDataById(idInfoArr []util.Data, appid, importType string, success *bool,
 			m, _ := util.Mgo.FindById(util.DATACOLLNAME, id, nil)
 			if len(*m) > 0 {
 				appids := qu.ObjArrToStringArr((*m)["appid"].([]interface{}))
-				if !strings.Contains(strings.Join(appids, ","), appid) {
-					util.Mgo.UpdateById(util.DATACOLLNAME, id, bson.M{"$push": bson.M{"appid": appid}})
+				if !strings.Contains(strings.Join(appids, ","), task.appId) {
+					util.Mgo.UpdateById(util.DATACOLLNAME, id, bson.M{"$push": bson.M{"appid": task.appId, "pici": task.pici}})
 				}
 			} else {
-				//markData, _ := util.MgoHM.FindById("bidding", id, nil)
-				//if len(*markData) > 0 {
-				//
-				//}
+
 				//1.查bidding
 				tmpBidColl := util.BidColl1 //bidding
 				if id < util.BIDDINGSTARTID {
@@ -1332,7 +1352,7 @@ func GetDataById(idInfoArr []util.Data, appid, importType string, success *bool,
 						baseInfoMap["v_taginfo"] = tagInfoMap
 					}
 					baseInfoMap["i_createtime"] = time.Now().Unix()
-					baseInfoMap["appid"] = []string{appid}
+					baseInfoMap["appid"] = []string{task.appId}
 					baseInfoMap["b_isgivegroup"] = false //是否分配给用户组
 					baseInfoMap["b_istag"] = false       //是否已标注
 					//baseInfoMap["b_cleartag"] = false    //是否清理标注信息
@@ -1340,12 +1360,12 @@ func GetDataById(idInfoArr []util.Data, appid, importType string, success *bool,
 					baseInfoMap["b_check"] = false      // 质检标记
 					baseInfoMap["b_isEff"] = false      // 标的物有效性
 					baseInfoMap["b_istagging"] = true
+					baseInfoMap["pici"] = []int64{task.pici}
 					if util.Mgo.SaveByOriID(util.DATACOLLNAME, baseInfoMap) {
-						atomic.AddInt64(successNum, 1) //保存成功计数
+						atomic.AddInt64(&successNum, 1) //保存成功计数
 					} else {
 						lock.Lock()
-						//*success = false
-						if importType == "excel" {
+						if task.importType == "excel" {
 							msgArr = append(msgArr, "第"+fmt.Sprint(index+2)+"行未导入id："+id)
 							//*msg += "第" + fmt.Sprint(num+2) + "行未保存成功数据_id：" + id + ";\n"
 						} else {
@@ -1356,9 +1376,8 @@ func GetDataById(idInfoArr []util.Data, appid, importType string, success *bool,
 					}
 				} else {
 					lock.Lock()
-					*success = false
 					qu.Debug(id)
-					if importType == "excel" {
+					if task.importType == "excel" {
 						msgArr = append(msgArr, "第"+fmt.Sprint(index+2)+"行，未查询到id："+id)
 						//*msg += "第" + fmt.Sprint(num+2) + "行未查询到数据：" + id + ";\n"
 					} else {
@@ -1372,8 +1391,7 @@ func GetDataById(idInfoArr []util.Data, appid, importType string, success *bool,
 	}
 	wg.Wait()
 	sort.Strings(msgArr)
-	*msg = strings.Join(msgArr, ";<br>")
-
+	task.msg = strings.Join(msgArr, ";<br>")
 }
 
 func (f *Front) ProjectCheckSuc() {

src/util/config.go

@@ -6,8 +6,6 @@ import (
 	"mongodb"
 	qu "qfw/util"
 	"sort"
-	"strconv"
-	"strings"
 	"sync"
 )
 
@@ -41,8 +39,6 @@ var (
 	MgoHM               *mongodb.MongodbSim //高质量库
 
 	MgoBulkSize = 200
-
-	Appid map[string]int
 )
 
 var (
@@ -246,13 +242,4 @@ func InitConfig() {
 	for _, f := range tmpAllField {
 		AllFieldArr = append(AllFieldArr, map[string]string{AllFieldMap[f]: f})
 	}
-
-	Appid = make(map[string]int, 50000)
-	info1, _ := Mgo.Find(PROJECTCOLLNAME, nil, nil, bson.M{"appid": 1}, false, -1, -1)
-	for _, m := range *info1 {
-		aid := qu.ObjToString(m["appid"])
-		ss := strings.Split(aid, "-")
-		intVar, _ := strconv.Atoi(ss[1])
-		Appid[ss[0]] = intVar
-	}
 }

src/web/templates/project/project_clear.html

@@ -110,6 +110,23 @@
                                 <option value="已关闭">已关闭</option>
                             </select></label>
                         </div>
+                        <div class="box-body margin">
+                            <h4><i class="glyphicon glyphicon-tasks" style="margin-right: 6px"></i>数据导入列表</h4>
+                            <hr>
+                            <table id="importTable" class="table table-bordered table-hover">
+                                <thead>
+                                <tr>
+                                    <th>序号</th>
+                                    <th>项目名称</th>
+                                    <th>导入时间</th>
+                                    <th>导入数量</th>
+                                    <th>清洗完成数量</th>
+                                    <th>操作</th>
+                                </tr>
+                                </thead>
+                            </table>
+                            <iframe srcdoc="<form id='uploadform' method='post' enctype='multipart/form-data' action='/center/taskfile'><input type='file' name='xlsx' /></form>" height=0 scrolling=no class="hide"  id="taskfileframe"></iframe>
+                        </div>
                         <div class="box-body margin">
                             <h4><i class="glyphicon glyphicon-tasks" style="margin-right: 6px"></i>数据清洗任务列表</h4>
                             <hr>
@@ -213,6 +230,45 @@
     let stype = ""                                  // 分发类型：所有数据:all 达标数据：notag 未达标：tag
 
     $(function () {
+        ttable1 = $('#importTable').dataTable({
+            "paging": false,
+            "lengthChange": false,
+            "searching": false,
+            "ordering": false,
+            "info": true,
+            "autoWidth": false,
+            "language": {
+                "url": "/dist/js/dataTables.chinese.lang"
+            },
+            "fnDrawCallback": function () {
+                this.api().column(0).nodes().each(function (cell, i) {
+                    cell.innerHTML = i + 1;
+                });
+            },
+            "columns": [
+                {"data": null, width: "1%"},
+                {"data": "projectname", width: "5%"},
+                {
+                    "data": "_id", width: "4%", render: function (val, a, row) {
+                        let str = ""
+                        let dt = new Date()
+                        dt.setTime(parseInt(row['pici']) * 1000);
+                        str = dt.format("yyyy-MM-dd hh:mm:ss")
+                        return str
+                    }
+                },
+                {"data": "i_importnum", width: "4%"},
+                {"data": "i_tagnum", width: "4%"},
+                {"data": "pici", width: "11%", render: function (val, a, row, pos) {
+                        return '<div>' +
+                            '<a class="btn btn-sm btn-primary" onclick="syncJyCm(' + val + ',\'' + row.appid + '\')">推送</a>&nbsp;&nbsp;' +
+                            '<a class="btn btn-sm btn-primary" onclick="exportData(' + val + ',\'' + row.appid + '\')">导出</a>&nbsp;&nbsp;' +
+                            '<a class="btn btn-sm btn-primary" onclick="importData(' + val + ')">导入</a>&nbsp;&nbsp;' +
+                            '</div>'
+                    }
+                }
+            ]
+        });
         ttable = $('#dataTable').dataTable({
             "paging": true,
             "lengthChange": false,
@@ -228,7 +284,7 @@
                 $('#dataTable-btn').click(function (e) {
                     var redirectpage = 0
                     if ($("#changePage").val() && $("#changePage").val() > 0) {
-                        var redirectpage = $("#changePage").val() - 1;
+                        redirectpage = $("#changePage").val() - 1;
                     }
                     ttable.api().page(redirectpage).draw(false);
                 });
@@ -277,9 +333,9 @@
                                 tmp = '<div>' +
                                     '<a class="btn btn-sm btn-primary" href="/front/group/user/task/list?pid='+projectid+'&grouptaskid='+val+'&s_groupid='+row.s_groupid+'">查看</a>&nbsp;&nbsp;' +
                                     '<a class="btn btn-sm btn-primary" onclick="checkMethod1(\'' + row.s_projectid + '\',\'' + val + '\')">质检</a>&nbsp;&nbsp;' +
-                                    '<a class="btn btn-sm btn-primary" onclick="syncJyCm(\'' + val + '\',\'' + row.s_status + '\')">推送</a>&nbsp;&nbsp;' +
-                                    '<a class="btn btn-sm btn-primary" onclick="exportData(\'' + val + '\')">导出</a>&nbsp;&nbsp;' +
-                                    '<a class="btn btn-sm btn-primary" onclick="importData(\'' + val + '\')">导入</a>&nbsp;&nbsp;' +
+                                    // '<a class="btn btn-sm btn-primary" onclick="syncJyCm(\'' + val + '\',\'' + row.s_status + '\')">推送</a>&nbsp;&nbsp;' +
+                                    // '<a class="btn btn-sm btn-primary" onclick="exportData(\'' + val + '\')">导出</a>&nbsp;&nbsp;' +
+                                    // '<a class="btn btn-sm btn-primary" onclick="importData(\'' + val + '\')">导入</a>&nbsp;&nbsp;' +
                                     '<a class="btn btn-sm btn-primary" onclick="StatsTask(\'' + val + '\',\'' + row.s_status + '\')">信息统计</a>&nbsp;&nbsp;' +
                                     '</div>';
                             }
@@ -316,6 +372,17 @@
                 }
             }
         })
+        $.ajax({
+            url: "/front/project/pici/list",
+            type: "POST",
+            data: {"pid": projectid},
+            success: function (r) {
+                if (r.data.length > 0) {
+                    ttable1.fnClearTable();
+                    ttable1.fnAddData(r.data);
+                }
+            }
+        })
         $.ajax({
             url: "/front/group/list",
             type: "POST",
@@ -508,41 +575,37 @@
     }
 
     // 推送数据
-    function syncJyCm(id, status) {
-        if (status === "已完成") {
-            showConfirm("确认要否推送当前数据到剑鱼客户管理服务平台？", function () {
-                showLoading("正在推送数据，请稍候...")
-                $.ajax({
-                    url: "/front/project/task/syncjycm",
-                    type: 'POST',
-                    data: {"taskid": id, "pid": projectid},
-                    success: function (r) {
-                        hideLoading()
-                        if (r.success) {
-                            let msg = r.msg+"<br>"+"一共推送"+r.count+"条数据"
-                            showMsg(msg, function (){
-                                location.reload()
-                            })
-                        } else {
-                            showMsg(r.msg, function (){
-                                location.reload()
-                            })
-                        }
+    function syncJyCm(pici, appid) {
+        showConfirm("确认要否推送当前数据到剑鱼客户管理服务平台？", function () {
+            showLoading("正在推送数据，请稍候...")
+            $.ajax({
+                url: "/front/project/task/syncjycm",
+                type: 'POST',
+                data: {"pici": pici, "appid": appid, "pid": projectid},
+                success: function (r) {
+                    hideLoading()
+                    if (r.success) {
+                        let msg = r.msg+"<br>"+"一共推送"+r.count+"条数据"
+                        showMsg(msg, function (){
+                            location.reload()
+                        })
+                    } else {
+                        showMsg(r.msg, function (){
+                            location.reload()
+                        })
                     }
-                })
+                }
             })
-        }else {
-            showTip("操作不允许")
-        }
+        })
     }
 
-    function exportData(id) {
+    function exportData(pici, appid) {
         showConfirm("确认从标注平台导出任务数据？", function () {
             showLoading("正在导出数据，请稍候...")
             $.ajax({
                 url: "/front/project/task/exportdata",
                 type: 'POST',
-                data: {"taskid": id},
+                data: {"pici": pici, "appid": appid, "pid": projectid},
                 xhrFields: { responseType: 'arraybuffer'},
                 success: function (r, status, xhr) {
                     hideLoading()