data_validation/src/front/server.go

// front
package front

import (
	"context"
	"encoding/json"
	"io/ioutil"

	//"container/list"
	"fmt"
	qu "qfw/util"
	"qfw/util/redis"

	mgo "mongodb"
	"strings"
	"sync"
	"time"
	"util"

	"github.com/go-xweb/xweb"
	"go.mongodb.org/mongo-driver/bson"

	"github.com/tealeg/xlsx"
	es "gopkg.in/olivere/elastic.v7"
)

type Front struct {
	*xweb.Action
	// elist         xweb.Mapper `xweb:"/elist"`
	login         xweb.Mapper `xweb:"/"`
	saveUser      xweb.Mapper `xweb:"/center/saveuser"`
	listInfo      xweb.Mapper `xweb:"/center/list"`
	checkId       xweb.Mapper `xweb:"/center/checkid"`
	detail        xweb.Mapper `xweb:"/center/detail/(.*).html"`
	biaozhu       xweb.Mapper `xweb:"/center/biaozhu"`
	importByExcel xweb.Mapper `xweb:"/center/importbyexcel"`
	importByEs    xweb.Mapper `xweb:"/center/importbyes"`
	getEsCount    xweb.Mapper `xweb:"/center/getescount"`
	finishCheck   xweb.Mapper `xweb:"/center/finishcheck"`
	errCheck      xweb.Mapper `xweb:"/center/errcheck"`
	syncMarked    xweb.Mapper `xweb:"/center/syncmarked"`
	markedInit    xweb.Mapper `xweb:"/center/markedinit"`

	review       xweb.Mapper `xweb:"/center/review"`     // 质检
	reviewList   xweb.Mapper `xweb:"/center/reviewlist"` // 质检数据列表
	reviewDetail xweb.Mapper `xweb:"/center/reviewdetail/(.*).html"`
	reviewSave   xweb.Mapper `xweb:"/center/reviewsave"`
	reviewStat   xweb.Mapper `xweb:"/center/review/result"`
	reviewNext   xweb.Mapper `xweb:"/center/review/next"`

	logout xweb.Mapper `xweb:"/front/logout"`
}

var preKey = "ck_"
var preErr = "err_"
var SE = qu.SimpleEncrypt{Key: "topJYBX2019"}
var CheckLock = &sync.Mutex{}
var LABELER, AUDITOR, ADMIN = 1, 2, 3 //标注人员，审核人员，管理员

/*
	status
		-1:初始未验证
		1:正确
		2:新增
		3:修改
		4:删除
*/

//列表
func (i *Front) ListInfo() error {
	//loginuser := i.GetSession("loginuser").(string)
	spidercode := i.GetString("spidercode")
	coll := i.GetString("coll")
	field := i.GetString("field")
	stype := i.GetString("type")
	min := i.GetString("minval") //min max不用int类型接收，以免有默认值0
	max := i.GetString("maxval")
	hasno, _ := i.GetBool("hasno")
	notag, _ := i.GetBool("notag")
	pagenum, _ := i.GetInteger("pagenum")
	labeler := i.GetString("labeler")
	if pagenum == 0 { //页码
		pagenum = 1
	}
	qu.Debug("coll:", coll, "stype:", stype, "field:", field, "hasno:", hasno, "notag:", notag, "max:", max, "min:", min, "pagenum:", pagenum, "labeler:", labeler)
	query := map[string]interface{}{}
	if spidercode != "" { //爬虫代码
		query["spidercode"] = spidercode
	}
	if stype != "-1" && stype != "" { //类型
		if stype == util.SPECIALTYPE {
			query["subtype"] = map[string]interface{}{
				"$exists": false,
			}
		} else {
			subtype := strings.Split(stype, "-")[1]
			query["subtype"] = subtype
		}
	}
	fieldScreen := false
	if field != "-1" && field != "" { //字段
		fieldScreen = true
		queryfield := map[string]interface{}{
			"$exists": !hasno,
		}
		numMap := map[string]interface{}{}
		if field == "budget" || field == "bidamount" { //金额区间
			if min != "" {
				minint := qu.IntAll(min)
				numMap["$gte"] = minint
			}
			if max != "" {
				maxint := qu.IntAll(max)
				numMap["$lte"] = maxint
			}
			//if len(numMap) > 0 { //给定了区间，不再判断字段存在
			//	queryfield = numMap
			//}
		}
		if len(numMap) > 0 {
			if !hasno {
				query[field] = numMap
			} else {
				query["$or"] = []map[string]interface{}{
					{field: queryfield},
					{field: numMap},
				}
			}
		} else {
			query[field] = queryfield
		}
	}
	if fieldScreen && notag {
		query[preKey+field] = map[string]interface{}{
			"$exists": false,
		}
	} else if !fieldScreen && notag {
		query["ck_data"] = 0
	}

	//标注人员
	if labeler != "" {
		query["modifyuser"] = labeler
	}
	qu.Debug("query:", coll, query)
	if coll == "" {
		coll = util.Config.Fromtable
		//coll = "markwork_wb_zcz"
	}
	i.SetSession("coll", coll)   //session中存入查询表
	i.SetSession("query", query) //session中存入查询条件

	listData := getListInfo(coll, query, pagenum)
	i.T["list"] = listData
	i.T["type"] = stype
	i.T["field"] = field
	i.T["spidercode"] = spidercode
	i.T["coll"] = coll
	i.T["topsubtype"] = util.TopSubStypeArr
	i.T["allfield"] = util.AllFieldArr
	i.T["fromtable"] = util.Config.Fromtable
	i.T["dbname"] = util.Config.Dbname
	i.T["max"] = max
	i.T["min"] = min
	i.T["hasno"] = hasno
	i.T["notag"] = notag
	checkedNum, allNum := GetCheckedAndAllDataInfo(query, coll) //已标和总数信息
	if notag {
		checkedNum = 0
	}
	i.T["checkednum"] = checkedNum
	i.T["allnum"] = allNum
	i.T["pagenum"] = pagenum
	role := qu.IntAll(i.GetSession("role"))
	i.T["role"] = role
	i.T["labeler"] = labeler
	if role >= AUDITOR {
		i.T["labelers"] = GetLabeler(coll)
	}
	return i.Render("list.html", &i.T)
}

//判断id是否正在被标记
func (i *Front) CheckId() {
	defer qu.Catch()
	CheckLock.Lock()
	defer CheckLock.Unlock()
	msg := ""
	id := i.GetString("id")
	coll := i.GetSession("coll").(string)
	if coll != "" {
		exists, err := redis.Exists("extcheck", coll+"_"+id)
		if err == nil {
			if exists {
				id, exists = GetNoCheckedId(id, coll)
			}
		} else {
			msg = "校验数据出错！"
		}
		i.ServeJson(map[string]interface{}{"msg": msg, "id": id, "exists": exists})
		return
	}
	i.ServeJson(map[string]interface{}{"msg": "数据校验表出错！"})
}

//标注
func (i *Front) Biaozhu() error {
	b := false
	obj := []map[string]interface{}{}
	//ispackage := i.GetString("ispackage")
	key := i.GetString("key")
	_id := i.GetString("_id")
	stype, _ := i.GetInteger("stype")
	err := json.Unmarshal([]byte(key), &obj)
	if err != nil {
		i.ServeJson(b)
		return nil
	}
	//base := map[string]interface{}{}
	set := map[string]interface{}{}    //更新、新增字段
	unset := map[string]interface{}{}  //删除字段
	errset := map[string]interface{}{} //记录修改的字段信息
	isSaveMarked := false
	if len(obj) == 1 { //单独保存某个一级
		content, ok := obj[0]["content"].([]interface{})
		if !ok || len(content) == 0 {
			i.ServeJson(b)
			return nil
		}
		title := qu.ObjToString(obj[0]["title"])
		isext, _ := obj[0]["checkType"].(bool)
		istag, _ := obj[0]["checkAllTag"].(bool)
		status := qu.IntAll(obj[0]["status"])
		switch title {
		case "基本字段":
			BzJBZD(content, set, unset, errset)
		case "时间地点":
			BzSJDD(content, set, unset, errset)
		case "标的信息":
			BzBDXX(content, set, unset, errset, isext, istag, status)
		case "多包信息":
			BzDBXX(content, set, unset, errset, isext, status)
		case "中标候选人信息":
			BzZBHXRXX(content, set, unset, errset, isext, status)
		case "其余信息":
			BzQYXX(content, set, unset, errset)
		}
	} else {
		isSaveMarked = true
		for j, val := range obj {
			content, ok := val["content"].([]interface{})
			status := qu.IntAll(val["status"])
			if !ok {
				continue
				// i.ServeJson(b)
				// return nil
			}
			isext, _ := val["checkType"].(bool)
			istag, _ := val["checkAllTag"].(bool)
			if j == 0 { //基本信息
				BzJBZD(content, set, unset, errset)
			} else if j == 1 { //时间地点
				BzSJDD(content, set, unset, errset)
			} else if j == 2 { //标的物
				BzBDXX(content, set, unset, errset, isext, istag, status)
			} else if j == 3 { //多包
				BzDBXX(content, set, unset, errset, isext, status)
			} else if j == 4 { //候选人
				BzZBHXRXX(content, set, unset, errset, isext, status)
			} else { //其余信息
				BzQYXX(content, set, unset, errset)
			}
		}
	}
	qu.Debug("errset---", errset)
	qu.Debug("set---", set)
	qu.Debug("unset---", unset)
	userInfo := map[string]interface{}{}     //记录本次标注日志
	modifyFields := map[string]interface{}{} //本次标注所有字段
	unerrset := map[string]interface{}{}     //记录errdata错误信息表删除

	loginuser := i.GetSession("loginuser").(string)
	coll := i.GetSession("coll").(string)
	data, _ := util.MgoM.FindById(coll, _id, nil) //查询标注保存前的原始信息
	//modifyuser := qu.ObjToString((*data)["modifyuser"])
	//标注多次操作处理
	if len(set) > 0 { //set中为本次标注保存的数据(ck_bidopentime:1;ck_buyer:2;buyer:"XXX")
		for s, sv := range set {
			//特殊字段处理
			//if s == "ck_pclistag" && (*data)["ck_pclistag"] != nil {
			//	delete(set, s)
			//	continue
			//}
			//区分是标记字段，还是普通字段(标记字段:ck_buyer;普通字段:buyer)
			if strings.HasPrefix(s, preKey) {
				status := qu.IntAll(sv)                                        //标注字段状态
				field := s[3:]                                                 //去除前缀，被标注字段
				checkedStatus := qu.IntAll((*data)[s])                         //数据库被标注状态
				if status == 1 && (checkedStatus == 2 || checkedStatus == 1) { //数据库已有标记,且页面标注正确,此次该字段不做标注保存
					qu.Debug("已标注字段field---", field)
					delete(set, field)
					delete(set, s)
					continue
				} else {
					qu.Debug("未标注字段field---", field, status)
					if set[field] == nil && status != 2 { //表示该字段没有要修改的值，此时标记为1或2(亦或是ck_wodrisext;ck_pkgisext;ck_pclisext)
						modifyFields[field] = set[s] //目前会有pclisext、pclistag、pkgisext、wodrisext
						continue
					} else { //此时有修改的值和标记2
						modifyFields[field] = status //记录标注的字段和状态
						if checkedStatus != 0 {      //已有标注信息的状态
							if status == 2 && checkedStatus == 2 { //字段已被修改且本次也为修改
								errdata, _ := util.MgoM.FindById(util.Config.Totablel, _id, map[string]interface{}{"userinfo": 0})
								//if field == "package" || field == "purchasinglist" || field == "winnerorder" {
								//}
								if errdata != nil && len(*errdata) != 0 {
									qu.Debug(field, (*errdata)[field] != nil, fmt.Sprint((*errdata)[field]))
									delete(errset, field) //errdata中有该字段的错误记录，不再更新errdata
									//对比errdata错误值:相等变为1，errdata删除;不等变为2，标注表修改该字段
									errText := fmt.Sprint((*errdata)[field])
									text := fmt.Sprint(set[field])
									qu.Debug("errText---", errText, text)
									if errText == text {
										set[s] = 1           //更新标注表字段状态
										unerrset[field] = "" //删除errdata表字段
										//delete(errset, field)
									}
								}
							}
						}
					}
				}
			}
		}
	}
	qu.Debug("errset---", errset)
	qu.Debug("set---", set)
	qu.Debug("unset---", unset)
	qu.Debug("unerrset---", unerrset)
	//1、errdata操作
	//存储原错误信息
	errUpdata := map[string]interface{}{} //errdata所有要更新内容
	for f, v := range errset {
		qu.Debug("err filed---", f, v)
		//if f != "package" && f != "purchasinglist" && f != "winnerorder" {
		errset[f] = (*data)[f]
		//}
	}
	if len(modifyFields) > 0 { //有标注字段，记录标注信息
		errset["updatetime"] = time.Now().Unix()
		errset["modifyuser"] = loginuser
		userInfo["fields"] = modifyFields
		userInfo["updatetime"] = time.Now().Unix()
		userInfo["modifyuser"] = loginuser
		errUpdata["$push"] = map[string]interface{}{
			"userinfo": map[string]interface{}{
				"$each":     []interface{}{userInfo},
				"$position": 0,
			},
		}
	}
	qu.Debug("set---", set)
	qu.Debug("errset---", errset)
	qu.Debug("unerrset---", unerrset)
	qu.Debug("userInfo---", userInfo)
	if len(errset) > 0 {
		errUpdata["$set"] = errset
	}
	if len(unerrset) > 0 {
		errUpdata["$unset"] = unerrset
	}
	if len(errUpdata) > 0 {
		util.MgoM.Update(util.Config.Totablel, `{"_id":"`+_id+`"}`, errUpdata, true, false)
	}
	//更新表操作
	//qu.Debug("set---", set)
	//更新正确信息
	update := map[string]interface{}{}
	if len(set) > 0 {
		set["ck_data"] = stype
		set["updatetime"] = time.Now().Unix()
		set["modifyuser"] = loginuser
		update["$set"] = set
	}
	qu.Debug("unset---", unset)
	if len(unset) > 0 {
		update["$unset"] = unset
	}
	qu.Debug("update---", update)
	if len(update) > 0 {
		b = util.MgoM.Update(coll, `{"_id":"`+_id+`"}`, update, false, false)
		if b {
			if coll != util.Config.Fromtable && isSaveMarked { //util.coll此时的标注表  util.Config.Fromtable默认标注表
				data, _ := util.MgoM.FindById(coll, _id, nil)
				delete(*data, "modifyuser")
				if data != nil && len(*data) > 0 {
					update := map[string]interface{}{"$set": *data}
					if len(unset) > 0 {
						update["$unset"] = unset
					}
					if !util.MgoM.Update(util.Config.Fromtable, map[string]interface{}{"_id": (*data)["_id"]}, update, true, false) {
						qu.Debug("同步marked失败，", _id)
					}
				}
			}
		}
		i.ServeJson(b)
	} else {
		i.ServeJson(true)
	}

	return nil
}

//查询信息
func (i *Front) Detail(id string) error {
	coll, _ := i.GetSession("coll").(string)
	//if coll != "" {
	//	exists, _ := redis.Exists("extcheck", coll+"_"+id)
	//	if exists {
	//		tmpId, exists := GetNoCheckedId(id, coll)
	//		if exists { //标注完成
	//			return i.Render("finish.html", &i.T)
	//		} else {
	//			id = tmpId
	//		}
	//	}
	//} else {
	//	i.T["err"] = "数据查询表为空！"
	//	return i.Render("err.html", &i.T)
	//}
	query, _ := i.GetSession("query").(map[string]interface{})
	rep := getDetail(id, coll)          //获取本条公告的信息
	i.T["otherInfo"] = rep["otherInfo"] //展示关联公告信息
	i.T["moreInfo"] = rep["moreInfo"]   //更多关联公告信息
	//i.T["jyhref"] = util.JYHREFPRE + qu.CommonEncodeArticle("content", id) + ".html"
	i.T["info"] = rep["info"]
	i.T["common"] = rep["common"]
	i.T["worder"] = rep["worder"]
	i.T["packs"] = rep["packs"]
	i.T["packskey"] = rep["packskey"]
	//i.T["ck_pclisext"] = rep["ck_pclisext"]
	i.T["ck_pclistag"] = rep["ck_pclistag"]
	//i.T["ck_wodrisext"] = rep["ck_wodrisext"]
	//i.T["ck_pkgisext"] = rep["ck_pkgisext"]
	i.T["timeplace"] = rep["timeplace"]
	i.T["purchasinglist"] = rep["purchasinglist"]
	i.T["other"] = rep["other"]
	i.T["PurchasinglistField"] = util.PurchasinglistField
	i.T["WinnerorderField"] = util.WinnerorderField
	i.T["PackageField"] = util.PackageField
	i.T["topsubtype"] = util.TopSubStypeArr2
	i.T[preKey+"purchasinglist"] = rep[preKey+"purchasinglist"]
	i.T[preKey+"package"] = rep[preKey+"package"]
	i.T[preKey+"winnerorder"] = rep[preKey+"winnerorder"]
	i.T["worder_new"] = rep["worder_new"]
	i.T["pcl_new"] = rep["pcl_new"]
	i.T["pkg_new"] = rep["pkg_new"]
	i.T["nextid"] = GetNextDataId(id, coll, query)              //下一条id
	checkedNum, allNum := GetCheckedAndAllDataInfo(query, coll) //已标和总数信息
	i.T["checkednum"] = checkedNum
	i.T["allnum"] = allNum
	i.T["fields"] = util.Config.Fields
	//存入Redis
	redis.Put("extcheck", coll+"_"+id, "", util.Config.RedisTimeout*60) //正在标注的数据存入redis避免多人同时标注(加上coll左前缀避免不同表相同id数据不能同时标注)
	return i.Render("detail.html", &i.T)
}

//通过excel表格导入
func (i *Front) ImportByExcel() {
	defer qu.Catch()
	//success := false
	msg := ""
	importNum := 0
	successNum := int64(0)
	coll := i.GetString("excelcoll")
	if coll == "" {
		i.ServeJson(map[string]interface{}{"msg": "表名错误"})
		return
	}
	mf, _, err := i.GetFile("xlsx")
	if err == nil {
		binary, _ := ioutil.ReadAll(mf)
		xls, _ := xlsx.OpenBinary(binary)
		sheet := xls.Sheets[0]
		rows := sheet.Rows
		idcolnum := -1
		ids := []string{}
		cellFieldName := map[int]string{}             //记录客户需求字段所在的列
		tmpMap := map[string]map[string]interface{}{} //excel表中需要保存字段集合
		for rn, row := range rows {
			if rn == 0 {
				for j, cell := range row.Cells {
					title := cell.Value
					if fieldName := util.Config.CustomerField[title]; fieldName != "" { //客户需求字段
						cellFieldName[j] = fieldName
					}
					if title == "唯一标识" || title == "信息标识" { //id所在列
						idcolnum = j
					}
				}
				if idcolnum == -1 {
					break
				}
				continue
			}
			if len(row.Cells) < len(rows[0].Cells) {
				break
			}
			tmp := map[string]interface{}{}
			for j, f := range cellFieldName {
				if val := row.Cells[j].Value; val != "" {
					if f == "capital" { //注册资金（万元）
						cf, _ := row.Cells[j].Float()
						tmp[f] = cf
					} else if f == "createtime" { //创建时间
						ci, _ := row.Cells[j].Int64()
						tmp[f] = ci
					} else {
						tmp[f] = val
					}
				}
			}
			id := row.Cells[idcolnum].String() //加密的id
			if id == "" {
				break
			}
			id = SE.DecodeString(id) //解密后id
			tmpMap[id] = tmp
			ids = append(ids, id)
		}
		importNum = len(ids) //excel表数据个数
		if importNum > 0 {
			_, msg, successNum = GetDataById1(coll, ids, "excel", tmpMap)
			tmpMap = map[string]map[string]interface{}{}
			ids = []string{}
		}
	}
	msg = fmt.Sprintf("共查询%d条，导入成功%d条\n", importNum, successNum) + msg
	i.ServeJson(map[string]interface{}{"msg": msg})
}

func (i *Front) GetEsCount() {
	defer qu.Catch()
	msg := ""
	count := int64(0)
	estext := i.GetString("estext") //es查询语句
	esJson := map[string]interface{}{}
	if json.Unmarshal([]byte(estext), &esJson) != nil || len(esJson) == 0 {
		msg = "Es语句错误"
	} else {
		count = util.Es.Count(util.Index, util.Itype, estext)
	}
	i.ServeJson(map[string]interface{}{"count": count, "msg": msg})
}

//通过es语句导入
func (i *Front) ImportByEs() {
	defer qu.Catch()
	//success := false
	msg := ""
	successNum := int64(0)
	estext := i.GetString("estext") //es查询语句
	coll := i.GetString("coll")     //导入表
	if coll == "" {
		i.ServeJson(map[string]interface{}{"msg": "表名错误"})
		return
	}
	client := util.Es.GetEsConn()
	defer util.Es.DestoryEsConn(client)
	ch := make(chan bool, 5)
	wg := &sync.WaitGroup{}
	lock := &sync.Mutex{}
	escount := util.Es.Count(util.Index, util.Itype, estext)
	qu.Debug("查询总数:", escount)
	if escount > 0 {
		//查询条件类型转换
		q := es.NewQueryStringQuery(estext)
		//游标查询，index不支持别名，只能写索引库的名称
		res, err := client.Scroll(util.Index).Query(q).Size(200).Do(context.TODO()) //查询一条获取游标
		ids := []string{}                                                           //id数据
		if err == nil {
			numDocs := 0
			scrollId := res.ScrollId
			for {
				if scrollId == "" {
					qu.Debug("ScrollId Is Error")
					break
				}
				searchResult, err := client.Scroll(util.Index).Size(200).ScrollId(scrollId).Do(context.TODO()) //查询
				if err != nil {
					if err.Error() == "EOS" { //迭代完毕
						qu.Debug("Es Search Data Over：", err)
					} else {
						qu.Debug("Es Search Data Error：", err)
					}
					break
				}
				for _, hit := range searchResult.Hits.Hits {
					//开始处理数据
					wg.Add(1)
					ch <- true
					go func(tmpHit *es.SearchHit) {
						defer func() {
							<-ch
							wg.Done()
						}()
						tmp := make(map[string]interface{})
						if json.Unmarshal(tmpHit.Source, &tmp) == nil {
							id := qu.ObjToString(tmp["_id"])
							tmp["id"] = id //记录数据原有id
							lock.Lock()
							ids = append(ids, id)
							lock.Unlock()
						}
					}(hit)
					numDocs += 1
					if numDocs%500 == 0 {
						qu.Debug("Current：", numDocs)
					}
				}
				scrollId = searchResult.ScrollId
			}
			wg.Wait()
			client.ClearScroll().ScrollId(scrollId).Do(context.TODO()) //清理游标
			//qu.Debug("Result Data Count：", numDocs)
		} else {
			qu.Debug("Es Search Data Error")
		}
		//判断数量
		if int64(len(ids)) != escount {
			msg = "查询数据和结果不一致"
		} else { //入库
			_, msg, successNum = GetDataById1(coll, ids, "es", map[string]map[string]interface{}{})
		}
	} else {
		msg = "无查询数据"
	}
	msg = fmt.Sprintf("共查询%d条，导入成功%d条\n", escount, successNum) + msg
	i.ServeJson(map[string]interface{}{"msg": msg})
}

//同步数据
func (i *Front) SyncMarked() {
	syncColl := i.GetString("coll")
	sess := util.MgoM.GetMgoConn()
	defer util.MgoM.DestoryMongoConn(sess)
	it := sess.DB(util.Config.Dbname).C(syncColl).Find(nil).Iter()
	count, _ := sess.DB(util.Config.Dbname).C(syncColl).Find(nil).Count()
	qu.Debug(syncColl, count)
	n := 0
	lock := &sync.Mutex{}
	wg := &sync.WaitGroup{}
	ch := make(chan bool, 3)
	result := map[string]map[string]interface{}{}
	idArr := []string{}
	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
		wg.Add(1)
		ch <- true
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			id := qu.ObjToString(tmp["id"])
			if id == "" {
				return
			}
			tmpMap := map[string]interface{}{}
			for _, f := range util.Config.CustomerField {
				if val := tmp[f]; val != nil {
					tmpMap[f] = val
				}
			}
			tmpMap["info"] = tmp["info"]
			lock.Lock()
			idArr = append(idArr, id)
			if result[id] != nil {
				qu.Debug("id exist!!!", id)
			}
			result[id] = tmpMap
			lock.Unlock()
		}(tmp)
		tmp = map[string]interface{}{}
	}
	wg.Wait()
	if count != int64(len(result)) {
		i.ServeJson(map[string]interface{}{"msg": "同步失败", "flag": false})
		return
	}
	//util.MgoM.C.Database(util.Config.Dbname).Collection(syncColl).Drop(util.MgoM.Ctx) //删除syncColl表(清空数据)
	success, msg, successNum := GetDataById(syncColl, idArr, "syncoll", result)
	result = map[string]map[string]interface{}{}
	idArr = []string{}
	msg = fmt.Sprintf("共查询%d条，同步成功%d条\n", count, successNum) + msg
	i.ServeJson(map[string]interface{}{"msg": msg, "flag": success})
}

//将marked表中ck_data：2更新为0
func (i *Front) MarkedInit() {
	set := map[string]interface{}{
		"$set": map[string]interface{}{
			"ck_data": 0,
		},
	}
	b := util.MgoM.Update(util.Config.Fromtable, `{"ck_data":2}`, set, false, true)
	i.ServeJson(map[string]interface{}{"success": b})
}

//标注完成
func (i *Front) FinishCheck() {
	i.Render("finish.html")
}

//错误页面
func (i *Front) ErrCheck() {
	i.Render("err.html")
}

//统计抽查
func (i *Front) Tj() error {
	comm := map[string]map[string]int{}
	comm_win := map[string]map[string]int{}
	pack := map[string]map[string]int{}
	pack_win := map[string]map[string]int{}
	list, _ := util.MgoM.Find(util.Config.Totablel, "{}", nil, nil, false, -1, -1)
	for _, tmp := range *list {
		for k, val := range tmp {
			if len(k) > 3 && k[:3] == preKey {
				comm = mapIntAdd(k, qu.ObjToString(val), comm)
			}
		}
		if winnerorder, ok := tmp["winnerorder"].([]interface{}); ok {
			for _, wd := range winnerorder {
				if winner, ok := wd.(map[string]interface{}); ok {
					for k, val := range winner {
						if len(k) > 3 && k[:3] == preKey {
							comm_win = mapIntAdd(k, qu.ObjToString(val), comm_win)
						}
					}
				}
			}
		}
		if ptmp, ok := tmp["package"].(map[string]interface{}); ok {
			for _, pktmp := range ptmp {
				if pkage, ok := pktmp.(map[string]interface{}); ok {
					for k, val := range pkage {
						if len(k) > 3 && k[:3] == preKey {
							pack = mapIntAdd(k, qu.ObjToString(val), pack)
						}
						if k == "winnerorder" {
							if wtmp, ok := val.([]interface{}); ok {
								for _, winner := range wtmp {
									if win, ok := winner.(map[string]interface{}); ok {
										for wk, wval := range win {
											if len(k) > 3 && k[:3] == preKey {
												pack_win = mapIntAdd(wk, qu.ObjToString(wval), pack_win)
											}
										}
									}
								}
							}
						}
					}
				}
			}
		}
	}
	//data := map[string]interface{}{"total": len(list), "comm": comm, "pack": pack, "pack_win": pack_win}
	//mongodb.Save("extcheck_tj", data)
	i.T["comm"] = comm
	i.T["pack"] = pack
	i.T["comm_win"] = comm_win
	i.T["pack_win"] = pack_win
	i.T["total"] = len(*list)
	return i.Render("tj.html", &i.T)
}

//标错列表
func (i *Front) Elist() error {
	attrname := i.GetString("attrname")
	common := util.Config.Biaozhu["common"]
	elist, _ := util.MgoM.Find(util.Config.Totablel, `{"ck_`+attrname+`":"0"}`, `{"_id":1}`, `{"_id":1}`, false, -1, -1)
	for _, v := range *elist {
		v["_id"] = mgo.BsonIdToSId(v["_id"])
	}
	i.T["elist"] = *elist
	i.T["attrname"] = attrname
	i.T["common"] = common
	return i.Render("elist.html", &i.T)
}

func (i *Front) Review() error {
	defer qu.Catch()
	var labeler []map[string]interface{}
	coll := i.GetString("coll")
	sess := util.MgoM.GetMgoConn()
	defer util.MgoM.DestoryMongoConn(sess)
	sess.DB(util.MgoM.DbName).C(coll).Pipe([]map[string]interface{}{
		//查询条件
		{
			"$match": bson.M{
				"ck_data": bson.M{"$gte": 1},
			},
		},
		// 按照modifyuser分组，并统计
		{
			"$group": bson.M{
				"_id":   "$modifyuser",
				"count": bson.M{"$sum": 1},
			},
		},
		// 排序
		{
			"$sort": bson.M{"count": -1},
		},
	}).All(&labeler)

	var maps []string
	maps = append(maps, "")
	for k, v := range util.Config.Fields {
		if v {
			maps = append(maps, k)
		}
	}
	maps = append(maps, "全部数据")
	i.T["fields"] = maps
	i.T["users"] = labeler
	i.T["coll"] = coll
	return i.Render("review.html", &i.T)
}

func (i *Front) ReviewList() {
	defer qu.Catch()
	if i.Method() == "POST" {
		coll := i.GetSession("coll").(string)
		user := i.GetString("user")
		q := make(map[string]interface{})
		q["ck_data"] = bson.M{"$gte": 1}
		if user != "0" && user != "-1" {
			q["modifyuser"] = user
		}
		i.SetSession("query1", q)
		datas, b := util.MgoM.Find(coll, q, bson.M{"_id": 1}, nil, false, -1, -1)
		if b && len(*datas) > 0 {
			i.ServeJson(map[string]interface{}{
				"rep":  b,
				"data": *datas,
			})
		} else {
			i.ServeJson(map[string]interface{}{
				"rep": false,
				"msg": "未查询到数据",
			})
		}
	}
}

func (i *Front) ReviewDetail(id string) error {
	defer qu.Catch()
	coll, _ := i.GetSession("coll").(string)
	query, _ := i.GetSession("query1").(map[string]interface{})
	rep := getDetail(id, coll)          //获取本条公告的信息
	i.T["otherInfo"] = rep["otherInfo"] //展示关联公告信息
	i.T["moreInfo"] = rep["moreInfo"]   //更多关联公告信息
	i.T["info"] = rep["info"]
	i.T["common"] = rep["common"]
	i.T["worder"] = rep["worder"]
	i.T["packs"] = rep["packs"]
	i.T["packskey"] = rep["packskey"]
	i.T["ck_pclisext"] = rep["ck_pclisext"]
	i.T["ck_wodrisext"] = rep["ck_wodrisext"]
	i.T["ck_pkgisext"] = rep["ck_pkgisext"]
	i.T["timeplace"] = rep["timeplace"]
	i.T["purchasinglist"] = rep["purchasinglist"]
	i.T["other"] = rep["other"]
	i.T["PurchasinglistField"] = util.PurchasinglistField
	i.T["WinnerorderField"] = util.WinnerorderField
	i.T["PackageField"] = util.PackageField
	i.T["topsubtype"] = util.TopSubStypeArr2
	i.T[preKey+"purchasinglist"] = rep[preKey+"purchasinglist"]
	i.T[preKey+"package"] = rep[preKey+"package"]
	i.T[preKey+"winnerorder"] = rep[preKey+"winnerorder"]
	i.T["worder_new"] = rep["worder_new"]
	i.T["pcl_new"] = rep["pcl_new"]
	i.T["pkg_new"] = rep["pkg_new"]
	i.T["fields"] = util.Config.Fields
	i.T["nextid"] = GetNextDataId(id, coll, query) //下一条id
	return i.Render("re_detail.html", &i.T)
}

func (i *Front) ReviewSave() error {
	defer qu.Catch()
	b := false
	obj := []map[string]interface{}{}
	key := i.GetString("key")
	_id := i.GetString("_id")
	//stype, _ := i.GetInteger("stype")
	err := json.Unmarshal([]byte(key), &obj)
	if err != nil {
		i.ServeJson(b)
		return nil
	}
	set := map[string]interface{}{}    //更新、新增字段
	unset := map[string]interface{}{}  //删除字段
	errset := map[string]interface{}{} //记录修改的字段信息
	isSaveMarked := false
	if len(obj) == 1 { //单独保存某个一级
		content, ok := obj[0]["content"].([]interface{})
		if !ok || len(content) == 0 {
			i.ServeJson(b)
			return nil
		}
		title := qu.ObjToString(obj[0]["title"])
		isext, _ := obj[0]["checkType"].(bool)
		istag, _ := obj[0]["checkAllTag"].(bool)
		status := qu.IntAll(obj[0]["status"])
		switch title {
		case "基本字段":
			BzJBZD(content, set, unset, errset)
		case "时间地点":
			BzSJDD(content, set, unset, errset)
		case "标的信息":
			BzBDXX(content, set, unset, errset, isext, istag, status)
		case "多包信息":
			BzDBXX(content, set, unset, errset, isext, status)
		case "中标候选人信息":
			BzZBHXRXX(content, set, unset, errset, isext, status)
		case "其余信息":
			BzQYXX(content, set, unset, errset)
		}
	} else {
		isSaveMarked = true
		for j, val := range obj {
			content, ok := val["content"].([]interface{})
			status := qu.IntAll(val["status"])
			if !ok {
				continue
				// i.ServeJson(b)
				// return nil
			}
			isext, _ := val["checkType"].(bool)
			istag, _ := val["checkAllTag"].(bool)
			if j == 0 { //基本信息
				BzJBZD(content, set, unset, errset)
			} else if j == 1 { //时间地点
				BzSJDD(content, set, unset, errset)
			} else if j == 2 { //标的物
				BzBDXX(content, set, unset, errset, isext, istag, status)
			} else if j == 3 { //多包
				BzDBXX(content, set, unset, errset, isext, status)
			} else if j == 4 { //候选人
				BzZBHXRXX(content, set, unset, errset, isext, status)
			} else { //其余信息
				BzQYXX(content, set, unset, errset)
			}
		}
	}
	userInfo := map[string]interface{}{}     //记录本次标注日志
	modifyFields := map[string]interface{}{} //本次标注所有字段
	unerrset := map[string]interface{}{}     //记录errdata错误信息表删除

	loginuser := i.GetSession("loginuser").(string)
	coll := i.GetSession("coll").(string)
	data, _ := util.MgoM.FindById(coll, _id, nil) //查询标注保存前的原始信息
	//modifyuser := qu.ObjToString((*data)["modifyuser"])

	remap := make(map[string]interface{}) // 质检信息
	reField := make(map[string]interface{})
	remap["user"] = loginuser
	remap["updatetime"] = time.Now().Unix()

	//标注多次操作处理
	if len(set) > 0 { //set中为本次标注保存的数据(ck_bidopentime:1;ck_buyer:2;buyer:"XXX")
		for s, sv := range set {
			//特殊字段处理
			//if s == "ck_pclistag" && (*data)["ck_pclistag"] != nil {
			//	delete(set, s)
			//	continue
			//}
			//区分是标记字段，还是普通字段(标记字段:ck_buyer;普通字段:buyer)
			if strings.HasPrefix(s, preKey) {
				status := qu.IntAll(sv)                //标注字段状态
				field := s[3:]                         //去除前缀，被标注字段
				checkedStatus := qu.IntAll((*data)[s]) //数据库被标注状态
				rename := strings.Replace(s, "ck_", "re_", -1)
				reField[rename] = status
				if status == 1 && (checkedStatus == 2 || checkedStatus == 1) { //数据库已有标记,且页面标注正确,此次该字段不做标注保存
					delete(set, field)
					delete(set, s)
					continue
				} else {
					if set[field] == nil && status != 2 { //表示该字段没有要修改的值，此时标记为1或2(亦或是ck_wodrisext;ck_pkgisext;ck_pclisext)
						modifyFields[field] = set[s] //目前会有pclisext、pclistag、pkgisext、wodrisext
						continue
					} else { //此时有修改的值和标记2
						modifyFields[field] = status //记录标注的字段和状态
						if checkedStatus != 0 {      //已有标注信息的状态
							if status == 2 && checkedStatus == 2 { //字段已被修改且本次也为修改
								errdata, _ := util.MgoM.FindById(util.Config.Totablel, _id, map[string]interface{}{"userinfo": 0})
								if errdata != nil && len(*errdata) != 0 {
									delete(errset, field) //errdata中有该字段的错误记录，不再更新errdata
									//对比errdata错误值:相等变为1，errdata删除;不等变为2，标注表修改该字段
									errText := fmt.Sprint((*errdata)[field])
									text := fmt.Sprint(set[field])
									if errText == text {
										set[s] = 1           //更新标注表字段状态
										unerrset[field] = "" //删除errdata表字段
									}
								}
							}
						}
					}
				}
			}
		}
	}
	remap["field"] = reField
	set["review"] = remap
	set["re_data"] = 1
	//存储原错误信息
	//1、errdata操作
	errUpdata := map[string]interface{}{} //errdata所有要更新内容
	for f, _ := range errset {
		errset[f] = (*data)[f]
	}
	if len(modifyFields) > 0 { //有标注字段，记录标注信息
		errset["updatetime"] = time.Now().Unix()
		errset["modifyuser"] = loginuser
		userInfo["fields"] = modifyFields
		userInfo["updatetime"] = time.Now().Unix()
		userInfo["modifyuser"] = loginuser
		errUpdata["$push"] = map[string]interface{}{
			"userinfo": map[string]interface{}{
				"$each":     []interface{}{userInfo},
				"$position": 0,
			},
		}
	}
	if len(errset) > 0 {
		errUpdata["$set"] = errset
	}
	if len(unerrset) > 0 {
		errUpdata["$unset"] = unerrset
	}
	if len(errUpdata) > 0 {
		util.MgoM.Update(util.Config.Totablel, `{"_id":"`+_id+`"}`, errUpdata, true, false)
	}
	//更新表操作
	//更新正确信息
	update := map[string]interface{}{}
	if len(set) > 0 {
		if len(set) > 1 {
			set["updatetime"] = time.Now().Unix()
			//set["modifyuser"] = loginuser
		}
		update["$set"] = set
	}
	if len(unset) > 0 {
		update["$unset"] = unset
	}
	if len(update) > 0 {
		b = util.MgoM.Update(coll, `{"_id":"`+_id+`"}`, update, false, false)
		if b {
			if coll != util.Config.Fromtable && isSaveMarked { //util.coll此时的标注表  util.Config.Fromtable默认标注表
				data, _ := util.MgoM.FindById(coll, _id, nil)
				if data != nil && len(*data) > 0 {
					update := map[string]interface{}{"$set": *data}
					if len(unset) > 0 {
						update["$unset"] = unset
					}
					if !util.MgoM.Update(util.Config.Fromtable, map[string]interface{}{"_id": (*data)["_id"]}, update, true, false) {
						qu.Debug("同步marked失败，", _id)
					}
				}
			}
		}
		i.ServeJson(b)
	} else {
		i.ServeJson(true)
	}

	return nil
}

func (i *Front) ReviewNext() {
	defer qu.Catch()
	id := i.GetString("id")
	coll := i.GetSession("coll").(string)
	q := bson.M{"_id": bson.M{"$gt": mgo.StringTOBsonId(id)}}
	info, b := util.MgoM.Find(coll, q, bson.M{"_id": 1}, bson.M{"_id": 1}, false, 0, 1)
	if b && len(*info) > 0 {
		nxid := mgo.BsonIdToSId((*info)[0]["_id"])
		i.ServeJson(bson.M{"id": nxid, "exists": true})
	} else {
		i.ServeJson(bson.M{"exists": false})
	}
}

func (i *Front) ReviewStat() {
	defer qu.Catch()
	sess := util.MgoM.GetMgoConn()
	defer util.MgoM.DestoryMongoConn(sess)
	coll := i.GetSession("coll").(string)
	qu.Debug(coll)
	result := sess.DB(util.MgoM.DbName).C(coll).Find(nil).Iter()
	count, cmark, rmark, rgmark := 0, 0, 0, 0 // 总数， 标注数量，	审核数据量，	审核数据完全正确的数据量

	cmaps := make(map[string]int)         // 标注字段整体准确率
	umaps := make(map[string]interface{}) // 按人员 字段准确率
	for k, i2 := range util.Config.Fields {
		if i2 {
			cmaps[k] = 0
		}
	}
	for tmp := make(map[string]interface{}); result.Next(&tmp); {
		count++
		user := qu.ObjToString(tmp["modifyuser"])
		if qu.IntAll(tmp["ck_data"]) >= 1 {
			cmark++
			var up map[string]int
			if umaps[user] == nil {
				up = make(map[string]int)
			} else {
				up = umaps[user].(map[string]int)
			}
			up["ck_count"] += 1
			umaps[user] = up
		}
		if qu.IntAll(tmp["re_data"]) > 0 {
			rmark++
			if reField, ok := tmp["review"].(map[string]interface{}); ok {
				remap, _ := reField["field"].(map[string]interface{})
				flag := true // 数据整体准确率
				// 按人员统计字段准备率
				var up map[string]int
				if umaps[user] == nil {
					up = make(map[string]int)
				} else {
					up = umaps[user].(map[string]int)
				}
				for k1 := range cmaps {
					k2 := "re_" + k1
					if qu.IntAll(remap[k2]) == 1 {
						cmaps[k1] += 1 // 字段整体正确率
						up[k1] += 1
					} else {
						flag = false
					}
				}
				up["re_count"] += 1
				umaps[user] = up
				if flag {
					rgmark++
					up["re_rg_count"] += 1
				}
			}
		}
	}

	//qu.Debug("count:", count, "，cmark:", cmark, "，rmark:", rmark, "，rgmark:", rgmark)
	//qu.Debug(cmaps)
	//qu.Debug(umaps)
	if rmark == 0 {
		i.ServeJson(map[string]interface{}{"rep": false, "msg": "无质检审核数据！"})
		return
	}
	save := bson.M{"total_count": count, "cmake_count": cmark, "rmark_count": rmark, "rgmark_count": rgmark}
	save["result"] = cmaps
	save["user_result"] = umaps
	save["re_user"] = i.GetSession("loginuser").(string)
	id := util.MgoM.Save("review_result", save)
	if id != "" {
		// 前台页面数据
		dataSource := make(map[string]interface{})
		var userSelect []string
		var dataSelect []map[string]interface{}
		userSelect = append(userSelect, "全部")
		tmp := make(map[string]interface{})
		tmp["name"] = "全部"
		tmp["num1"] = cmark
		tmp["num2"] = rmark
		tmp["num3"] = rgmark
		tmp["num4"] = CountPr(rgmark, rmark)
		dataSelect = append(dataSelect, tmp)
		for k, v := range util.Config.Fields {
			if v {
				tmp1 := make(map[string]interface{})
				tmp1["name"] = k
				tmp1["num1"] = cmark
				tmp1["num2"] = rmark
				tmp1["num3"] = cmaps[k]
				tmp1["num4"] = CountPr(cmaps[k], rmark)
				dataSelect = append(dataSelect, tmp1)
			}
		}
		dataSource["全部"] = dataSelect
		for k1, v1 := range umaps {
			if v2, o := v1.(map[string]int); o {
				userSelect = append(userSelect, k1)
				var dataSelect1 []map[string]interface{}
				tmp2 := make(map[string]interface{})
				tmp2["name"] = "全部"
				tmp2["num1"] = v2["ck_count"]
				tmp2["num2"] = v2["re_count"]
				tmp2["num3"] = v2["re_rg_count"]
				tmp2["num4"] = CountPr(v2["re_rg_count"], v2["re_count"])
				dataSelect1 = append(dataSelect1, tmp2)
				for k, v := range util.Config.Fields {
					if v {
						tmp1 := make(map[string]interface{})
						tmp1["name"] = k
						tmp1["num1"] = v2["ck_count"]
						tmp1["num2"] = v2["re_count"]
						tmp1["num3"] = v2[k]
						tmp1["num4"] = CountPr(v2[k], v2["re_count"])
						dataSelect1 = append(dataSelect1, tmp1)
					}
				}
				dataSource[k1] = dataSelect1
			}
		}
		//var dataSource [][]string
		//var tmp []string
		//tmp  = append(tmp, "全部")
		//for k, v := range util.Config.Fields{
		//	if v {
		//		str := fmt.Sprintf("抽查字段标注数据量：%d,</br>抽查字段标注正确数量：%d,</br>占比%s", rmark, cmaps[k], CountPr(cmaps[k], rmark))
		//		tmp = append(tmp, str)
		//	}
		//}
		//str := fmt.Sprintf("抽查标注数据量：%d,</br>抽查数据标注正确数量：%d,</br>占比%s", rmark, rgmark, CountPr(rgmark, rmark))
		//tmp = append(tmp, str)
		//dataSource = append(dataSource, tmp)
		//for k1, v1 := range umaps{
		//	if v2, o := v1.(map[string]int); o {
		//		var tmp2 []string
		//		tmp2 = append(tmp2, k1)
		//		for k, v := range util.Config.Fields{
		//			if v {
		//				str := fmt.Sprintf("抽查字段标注数据量：%d,</br>抽查字段标注正确数量：%d,</br>占比%s", v2["count"], v2[k], CountPr(v2[k], v2["count"]))
		//				tmp2 = append(tmp2, str)
		//			}
		//		}
		//		str := fmt.Sprintf("抽查标注数据量：%d,</br>抽查数据标注正确数量：%d,</br>占比%s", v2["count"], v2["rgmark_count"], CountPr(v2["rgmark_count"], v2["count"]))
		//		tmp2 = append(tmp2, str)
		//		dataSource = append(dataSource, tmp2)
		//	}
		//}
		i.ServeJson(map[string]interface{}{"rep": true, "data": map[string]interface{}{"tableData": dataSource, "userSelect": userSelect}})
	} else {
		i.ServeJson(map[string]interface{}{"rep": false, "msg": "统计失败！"})
	}
}

func (i *Front) Logout() {
	i.DelSession("user")
	err := i.Redirect("/")
	if err != nil {
		return
	}
}