data_validation/src/front/project.go

package front

import (
	"encoding/json"
	"fmt"
	"github.com/tealeg/xlsx"
	"io/ioutil"
	"mime/multipart"
	qu "qfw/util"
	"strings"
	"sync"
	"sync/atomic"
	"time"
	"util"
)

// ProjectList 项目列表
func (f *Front) ProjectList() {
	defer qu.Catch()
	if f.Method() == "POST" {
		start, _ := f.GetInteger("start")
		draw, _ := f.GetInteger("draw")
		status := f.GetString("status")
		searchStr := f.GetString("search[value]")
		search := strings.TrimSpace(searchStr)
		//data := util.GetPostForm(f.Request)
		query := map[string]interface{}{}
		if status != "-1" {
			query["s_status"] = status
		}
		if search != "" {
			query["$or"] = []interface{}{
				map[string]interface{}{"s_name": map[string]interface{}{"$regex": search}},
				map[string]interface{}{"s_entname": map[string]interface{}{"$regex": search}},
				map[string]interface{}{"s_rule": map[string]interface{}{"$regex": search}},
				map[string]interface{}{"s_departname": map[string]interface{}{"$regex": search}},
			}
		}
		list, _ := util.Mgo.Find("f_project", query, nil, nil, false, start, draw)
		count := util.Mgo.Count("f_project", query)
		f.ServeJson(map[string]interface{}{"draw": draw, "data": *list, "recordsFiltered": count, "recordsTotal": count})
	} else {
		query := map[string]interface{}{"s_type": "tag"}
		info, _ := util.Mgo.Find("v_field", query, nil, map[string]interface{}{"s_name": 1, "s_code": 1}, false, -1, -1)
		f.T["fields"] = *info
		_ = f.Render("project/project_list.html", &f.T)
	}
}

// ProjectSave 项目保存
func (f *Front) ProjectSave() {
	defer qu.Catch()
	success := false
	msg := ""
	successNum := int64(0)
	user := f.GetSession("user").(map[string]interface{})
	username := qu.ObjToString(user["s_name"])  //当前登录用户
	stype := f.GetString("s_type")              //新建项目类型：数据库导入、excel导入
	s_name := f.GetString("s_name")             //项目名称
	s_sourceinfo := f.GetString("s_sourceinfo") //数据表
	s_sourceinfo = "s_sourceinfo_" + s_sourceinfo
	s_departname, s_entname := "", ""
	importDataNum := 0
	var s_rulename []string
	query := map[string]interface{}{
		"s_name": s_name,
	}
	set := map[string]interface{}{}
	//导入数据
	if stype == "excel" { //excel导入
		s_entname = f.GetString("s_entname")       //公司名称
		s_departname = f.GetString("s_departname") //部门名称
		rulename := f.GetString("s_rulename")      //规则名称
		s_rulename = strings.Split(rulename, ",")
		mf, _, err := f.GetFile("xlsx")
		if err == nil {
			importDataNum = ImportDataByExcel(s_sourceinfo, mf, &success, &msg, &successNum)
		}
		//保存项目信息
		set = map[string]interface{}{
			"s_name":       s_name,                        //项目名称
			"s_entname":    s_entname,                     //公司名称
			"s_departname": s_departname,                  //部门名称
			"s_rulename":   strings.Join(s_rulename, ","), //规则名称
			"i_importnum":  importDataNum,                 //导入数量
			"s_sourceinfo": s_sourceinfo,                  //源数据表
			"s_createname": username,                      //创建人
			"s_status":     "未开始",                         //项目状态
			"i_createtime": time.Now().Unix(),             //创建时间
		}
	} else if stype == "coll" { //数据库导入
		historyid := f.GetString("s_historyid")
		s_departname, s_entname, s_rulename, importDataNum = ImportDataByColl(s_sourceinfo, historyid, &success, &msg, &successNum)
		qu.Debug(s_departname, s_entname, s_rulename)
		//保存项目信息
		set = map[string]interface{}{
			"s_name":       s_name,                        //项目名称
			"s_entname":    s_entname,                     //公司名称
			"s_departname": s_departname,                  //部门名称
			"s_rulename":   strings.Join(s_rulename, ","), //规则名称
			"i_importnum":  importDataNum,                 //导入数量
			"s_sourceinfo": s_sourceinfo,                  //源数据表
			"s_createname": username,                      //创建人
			"s_status":     "未开始",                         //项目状态
			"i_createtime": time.Now().Unix(),             //创建时间
		}
	} else if stype == "edit" { //编辑保存
		s_entname = f.GetString("s_entname")       //公司名称
		s_departname = f.GetString("s_departname") //部门名称
		rulename := f.GetString("s_rulename")      //规则名称
		s_rulename = strings.Split(rulename, ",")
		s_personname := f.GetString("s_personname")
		fields := f.GetString("v_fields")
		var v_fields []string
		if json.Unmarshal([]byte(fields), &v_fields) != nil {
			qu.Debug("标注字段解析失败")
		}
		set = map[string]interface{}{
			"s_name":       s_name,                        //项目名称
			"s_entname":    s_entname,                     //公司名称
			"s_departname": s_departname,                  //部门名称
			"s_rulename":   strings.Join(s_rulename, ","), //规则名称
			"v_fields":     v_fields,                      //标注字段
			"i_updatetime": username,                      //更新人
			"i_createtime": time.Now().Unix(),             //更新时间
			"s_personname": s_personname,                  //售后人员
			//"i_starttime":,//开始时间
			//"i_completetime",//结束时间
		}
	}
	util.Mgo.Update("f_project", query, map[string]interface{}{"$set": set}, true, false)
}

//ImportDataByExcel 通过excel获取数据源
func ImportDataByExcel(s_sourceinfo string, mf multipart.File, success *bool, msg *string, successNum *int64) (importDataNum int) {
	defer qu.Catch()
	binary, _ := ioutil.ReadAll(mf)
	xls, _ := xlsx.OpenBinary(binary)
	sheet := xls.Sheets[0]
	rows := sheet.Rows
	idcolnum := -1
	cellFieldName := map[int]string{}                //记录客户需求字段所在的列
	idInfoMap := map[string]map[string]interface{}{} //记录数据id及需要保存的字段信息
	for rn, row := range rows {
		if rn == 0 {
			for index, cell := range row.Cells {
				title := cell.Value
				if fieldName := util.CustomerFieldMap_HE[title]; fieldName != "" { //客户需求字段
					cellFieldName[index] = fieldName
				}
				if title == "唯一标识" || title == "信息标识" { //id所在列
					idcolnum = index
				}
			}
			if idcolnum == -1 {
				break
			}
			continue
		}
		if len(row.Cells) < len(rows[0].Cells) {
			break
		}
		tmp := map[string]interface{}{}
		for index, f := range cellFieldName {
			if val := row.Cells[index].Value; val != "" {
				if f == "capital" { //注册资金（万元）
					cf, _ := row.Cells[index].Float()
					tmp[f] = cf
				} else if f == "createtime" { //创建时间
					ci, _ := row.Cells[index].Int64()
					tmp[f] = ci
				} else {
					tmp[f] = val
				}
			}
		}
		id := row.Cells[idcolnum].String() //加密的id
		if id == "" {
			break
		}
		id = util.SE.DecodeString(id) //解密后id
		idInfoMap[id] = tmp
	}
	importDataNum = len(idInfoMap)
	qu.Debug("共加载Excel数据：", importDataNum, "条")
	if importDataNum > 0 {
		GetDataById(idInfoMap, "excel", s_sourceinfo, success, msg, successNum)
	} else {
		*success = false
		*msg = "查询数据失败"
	}
	idInfoMap = map[string]map[string]interface{}{}
	return
}

//ImportDataByColl 通过表获取数据源
func ImportDataByColl(s_sourceinfo, historyid string, success *bool, msg *string, successNum *int64) (departname, entname string, rulename []string, importDataNum int) {
	defer qu.Catch()
	sess := util.MgoJy.GetMgoConn()
	defer util.MgoJy.DestoryMongoConn(sess)
	ch := make(chan bool, 3)
	wg := &sync.WaitGroup{}
	lock := &sync.Mutex{}
	idInfoMap := map[string]map[string]interface{}{} //记录数据id及需要保存的字段信息
	query := map[string]interface{}{
		"historyId": historyid,
	}
	it := sess.DB(util.MgoJy.DbName).C(util.JyHistory).Find(&query).Iter()
	n := 0
	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
		ch <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			id := qu.ObjToString(tmp["id"])                //bidding id
			appid := qu.ObjToString(tmp["appid"])          //根据appid查user表获取公司名称
			departname = qu.ObjToString(tmp["departname"]) //部门名称。所有数据都应部门名称，若不一致，随机取
			needField := map[string]interface{}{}
			for f, _ := range util.CustomerFieldMap_EH {
				if tmp[f] != nil {
					needField[f] = tmp[f]
				}
			}
			if entname == "" { //获取一次公司名称即可
				user, _ := util.MgoJy.FindOne(util.JyUser, map[string]interface{}{"appid": appid})
				entname = qu.ObjToString((*user)["username"]) //公司名称
			}
			lock.Lock()
			rulename = append(rulename, qu.ObjToString(tmp["rulename"])) //规则名称
			idInfoMap[id] = needField
			lock.Unlock()
		}(tmp)
		if n%1000 == 0 {
			qu.Debug("current:", n)
		}
		tmp = map[string]interface{}{}
	}
	wg.Wait()
	importDataNum = len(idInfoMap) //查询数据总数
	if importDataNum > 0 {
		GetDataById(idInfoMap, "coll", s_sourceinfo, success, msg, successNum)
	} else {
		*msg = "查询数据失败"
	}
	idInfoMap = map[string]map[string]interface{}{}
	return
}

//GetDataById 通过id集从bidding、extract、project获取数据所有信息
func GetDataById(idsInfo map[string]map[string]interface{}, importType, s_sourceinfo string, success *bool, msg *string, successNum *int64) {
	*success = true
	wg := &sync.WaitGroup{}
	lock := &sync.Mutex{}
	ch := make(chan bool, 10)
	num := int64(0) //计数
	for id, info := range idsInfo {
		wg.Add(1)
		ch <- true
		go func(id string, tmp map[string]interface{}) {
			defer func() {
				wg.Done()
				<-ch
			}()
			/*
				1.查bidding
				2.查extract
				3.extract合并到bidding(删除item字段与客户需要的item不是一个含义)
				4.对比marked表,替换已标注过的字段值,补充标记
				5.合并客户所需字段信息,补充id字段
				//6.若为同步时,删除原有id对应的信息,新增该id对应的_id信息
				6.mgo查询项目信息
			*/
			tagInfoMap := map[string]interface{}{}  //记录数据已标注过的信息
			baseInfoMap := map[string]interface{}{} //记录其他信息
			//1.查bidding
			tmpBidColl := util.BidColl1 //bidding
			//查询bidding
			if id < util.BIDDINGSTARTID {
				tmpBidColl = util.BidColl2 //bidding_back
			}
			bidData, _ := util.MgoB.FindById(tmpBidColl, id, nil)
			if bidData != nil && len(*bidData) > 0 { //bidding表数据存在
				//2.查extract
				extData, _ := util.MgoE.FindById(util.ExtColl1, id, nil)
				if extData == nil || len(*extData) == 0 {
					extData, _ = util.MgoE.FindById(util.ExtColl2, id, nil)
				}
				//抽取表字段合并到bidding
				if extData != nil && len(*extData) > 0 {
					for k, v := range *extData {
						(*bidData)[k] = v
					}
				}
				//3.删除item
				//删除item
				delete((*bidData), "item")
				//4.对比marked表，对比marked表是否已标注该数据
				markData, _ := util.Mgo.FindById(util.AllToColl, id, nil)
				if markData != nil && len(*markData) > 0 {
					UpdateMarkColl(bidData, markData, &tagInfoMap, &baseInfoMap) //比对更新数据
				} else {
					baseInfoMap["i_ckdata"] = 0 //设置ck_data默认值0
				}
				//多包、中标候选人、标的信息是否抽取
				if packageMap, ok := (*bidData)["package"].(map[string]interface{}); ok && len(packageMap) > 0 {
					baseInfoMap["pkgisext"] = true
				} else {
					baseInfoMap["pkgisext"] = false
				}
				if winorderArr, ok := (*bidData)["winnerorder"].([]interface{}); ok && len(winorderArr) > 0 {
					baseInfoMap["wodrisext"] = true
				} else {
					baseInfoMap["wodrisext"] = false
				}
				if purchArr, ok := (*bidData)["purchasinglist"].([]interface{}); ok && len(purchArr) > 0 {
					baseInfoMap["pclisext"] = true
				} else {
					baseInfoMap["pclisext"] = false
				}
				//合并导入表中客户所需的字段
				if len(tmp) > 0 {
					for k, v := range tmp {
						(*bidData)[k] = v
					}
				}
				//补充id
				//(*bidData)["id"] = id

				//if stype == "syncoll" { //同步数据时删除原始数据
				//	if util.MgoM.Delete(coll, `{"id":"`+id+`"}`) == 0 {
				//		lock.Lock()
				//		*msg += "同步未删除成功数据id:" + id + ";\n"
				//		*success = false
				//		lock.Unlock()
				//	}
				//}
				// 处理	package winner_all
				if p, o1 := (*bidData)["package"].(map[string]interface{}); o1 {
					for _, v := range p {
						v1 := v.(map[string]interface{})
						t := make(map[string]interface{})
						if v1["winner"] != nil {
							t["winner"] = v1["winner"]
						}
						if v1["bidamount"] != nil {
							t["bidamount"] = qu.Float64All(v1["bidamount"])
						}
						if len(t) > 0 {
							v1["winner_all"] = append([]map[string]interface{}{}, t)
						}
					}
				}
				// 补充filetext
				(*bidData)["filetext"] = util.GetFileText(*bidData)
				// 6.es查询项目合并信息
				//esQ := `{"query":{"bool":{"must":[{"term":{"ids":"` + id + `"}}]}}}`
				//info := util.Es.Get("projectset", "projectset", esQ)
				projectId := qu.ObjToString((*bidData)["projectId"])
				project, _ := util.MgoE.FindById(util.ProjectColl, projectId, map[string]interface{}{"ids": 1})
				if project != nil && len(*project) > 0 {
					ids := qu.ObjArrToStringArr((*project)["ids"].([]interface{}))
					if len(ids) > 0 {
						var infolist []map[string]interface{}
						for _, v := range ids {
							if v == id { // 当前公告
								continue
							}
							if v < util.BIDDINGSTARTID {
								tmpBidColl = util.BidColl2 //bidding_back
							}
							bid, b := util.MgoB.FindById(tmpBidColl, v, nil)
							if b && len(*bid) > 0 {
								tmp := make(map[string]interface{})
								tmp["id"] = v
								tmp["title"] = (*bid)["title"]
								tmp["href"] = (*bid)["href"]
								tmp["toptype"] = (*bid)["toptype"]
								tmp["subtype"] = (*bid)["subtype"]
								tmp["publishtime"] = (*bid)["publishtime"]
								tmp["detail"] = (*bid)["detail"]
								tmp["filetext"] = util.GetFileText(*bid)
								infolist = append(infolist, tmp)
							}
						}
						(*bidData)["info"] = infolist
					}
				} else {
					qu.Debug("projectset find err", projectId)
				}
				baseInfoMap["id"] = id
				delete(*bidData, "id")
				//保存数据
				baseInfoMap["_id"] = (*bidData)["_id"]
				baseInfoMap["v_datainfo"] = bidData
				baseInfoMap["v_taginfo"] = tagInfoMap
				baseInfoMap["i_createtime"] = time.Now().Unix()
				baseInfoMap["b_isgive"] = false //是否分配
				baseInfoMap["b_istag"] = false  //是否已标注

				if util.Mgo.SaveByOriID(s_sourceinfo, baseInfoMap) {
					atomic.AddInt64(successNum, 1) //保存成功计数
				} else {
					lock.Lock()
					*success = false
					if importType == "excel" {
						*msg += "第" + fmt.Sprint(num+2) + "行未保存成功数据_id：" + id + ";\n"
					} else {
						*msg += "未保存成功数据_id:" + id + ";\n"
					}
					lock.Unlock()
				}
			} else {
				lock.Lock()
				*success = false
				if importType == "excel" {
					*msg += "第" + fmt.Sprint(num+2) + "行未查询到数据：" + id + ";\n"
				} else {
					*msg += "未查询到数据_id:" + id + ";\n"
				}
				lock.Unlock()
			}
		}(id, info)
	}
	wg.Wait()
}

//更新数据
func UpdateMarkColl(bidData, markData, tagInfoMap, baseInfoMap *map[string]interface{}) {
	defer qu.Catch()
	ckdata := qu.IntAll((*markData)["i_ckdata"])
	v_taginfo := (*markData)["v_taginfo"].(map[string]interface{})   //标注信息
	v_datainfo := (*markData)["v_datainfo"].(map[string]interface{}) //基本信息
	for fk, _ := range v_taginfo {
		if v_datainfo[fk] != nil {
			(*bidData)[fk] = v_datainfo[fk] //字段更新
		}
	}
	(*tagInfoMap)["v_taginfo"] = v_taginfo //marked中已有的标注信息保存到新数据上
	if ckdata == 2 {                       //某些字段已标注
		(*baseInfoMap)["i_ckdata"] = 0 //marked表中该条数据如果为字段验证，临时表ck_data:0;若为数据验证ck_data:1
	} else if ckdata == 1 {
		(*baseInfoMap)["i_ckdata"] = 1
	}
}