Merge branch 'dev1.0' of http://192.168.3.207:8080/data_processing/data_validation into dev1.0

* 'dev1.0' of http://192.168.3.207:8080/data_processing/data_validation:
  xg

# Conflicts:
#	src/front/front.go

src/front/front.go

@@ -1,14 +1,10 @@
-// front
 package front
 
 import (
 	"math"
 
 	//"container/list"
-	"encoding/json"
 	"fmt"
-	"go.mongodb.org/mongo-driver/bson"
-	"io/ioutil"
 	qu "qfw/util"
 	"qfw/util/redis"
 	"sync/atomic"
@@ -19,617 +15,8 @@ import (
 	"sync"
 	"time"
 	"util"
-
-	"github.com/go-xweb/xweb"
-	_ "github.com/influxdata/influxdb-client"
-	"github.com/tealeg/xlsx"
-	es "gopkg.in/olivere/elastic.v1"
 )
 
-type Front struct {
-	*xweb.Action
-	// elist         xweb.Mapper `xweb:"/elist"`
-	login         xweb.Mapper `xweb:"/"`
-	saveUser      xweb.Mapper `xweb:"/center/saveuser"`
-	listInfo      xweb.Mapper `xweb:"/center/list"`
-	checkId       xweb.Mapper `xweb:"/center/checkid"`
-	detail        xweb.Mapper `xweb:"/center/detail/(.*).html"`
-	biaozhu       xweb.Mapper `xweb:"/center/biaozhu"`
-	importByExcel xweb.Mapper `xweb:"/center/importbyexcel"`
-	importByEs    xweb.Mapper `xweb:"/center/importbyes"`
-	getEsCount    xweb.Mapper `xweb:"/center/getescount"`
-	finishCheck   xweb.Mapper `xweb:"/center/finishcheck"`
-	errCheck      xweb.Mapper `xweb:"/center/errcheck"`
-	syncMarked    xweb.Mapper `xweb:"/center/syncmarked"`
-	markedInit    xweb.Mapper `xweb:"/center/markedinit"`
-
-	review		  xweb.Mapper `xweb:"/center/review"`		// 质检
-	reviewList	  xweb.Mapper `xweb:"/center/reviewlist"`	// 质检数据列表
-	reviewDetail  xweb.Mapper `xweb:"/center/reviewdetail/(.*).html"`
-	reviewSave    xweb.Mapper `xweb:"/center/reviewsave"`
-	reviewStat	  xweb.Mapper `xweb:"/center/review/result"`
-}
-
-var preKey = "ck_"
-var preErr = "err_"
-var SE = qu.SimpleEncrypt{Key: "topJYBX2019"}
-var CheckLock = &sync.Mutex{}
-var LABELER, AUDITOR, ADMIN = 1, 2, 3 //标注人员，审核人员，管理员
-
-/*
-	status
-		-1:初始未验证
-		1:正确
-		2:新增
-		3:修改
-		4:删除
-*/
-
-//列表
-func (i *Front) ListInfo() error {
-	//loginuser := i.GetSession("loginuser").(string)
-	spidercode := i.GetString("spidercode")
-	coll := i.GetString("coll")
-	field := i.GetString("field")
-	stype := i.GetString("type")
-	min := i.GetString("minval") //min max不用int类型接收，以免有默认值0
-	max := i.GetString("maxval")
-	hasno, _ := i.GetBool("hasno")
-	notag, _ := i.GetBool("notag")
-	pagenum, _ := i.GetInteger("pagenum")
-	labeler := i.GetString("labeler")
-	if pagenum == 0 { //页码
-		pagenum = 1
-	}
-	qu.Debug("coll:", coll, "stype:", stype, "field:", field, "hasno:", hasno, "notag:", notag, "max:", max, "min:", min, "pagenum:", pagenum, "labeler:", labeler)
-	query := map[string]interface{}{}
-	if spidercode != "" { //爬虫代码
-		query["spidercode"] = spidercode
-	}
-	if stype != "-1" && stype != "" { //类型
-		if stype == util.SPECIALTYPE {
-			query["subtype"] = map[string]interface{}{
-				"$exists": false,
-			}
-		} else {
-			subtype := strings.Split(stype, "-")[1]
-			query["subtype"] = subtype
-		}
-	}
-	fieldScreen := false
-	if field != "-1" && field != "" { //字段
-		fieldScreen = true
-		queryfield := map[string]interface{}{
-			"$exists": !hasno,
-		}
-		if field == "budget" || field == "bidamount" { //金额区间
-			numMap := map[string]interface{}{}
-			if min != "" {
-				minint := qu.IntAll(min)
-				numMap["$gte"] = minint
-			}
-			if max != "" {
-				maxint := qu.IntAll(max)
-				numMap["$lte"] = maxint
-			}
-			if len(numMap) > 0 { //给定了区间，不再判断字段存在
-				queryfield = numMap
-			}
-		}
-		query[field] = queryfield
-	}
-	if fieldScreen && notag {
-		query[preKey+field] = map[string]interface{}{
-			"$exists": false,
-		}
-	} else if !fieldScreen && notag {
-		query["ck_data"] = 0
-	}
-
-	//标注人员
-	if labeler != "" {
-		query["modifyuser"] = labeler
-	}
-	qu.Debug("query:", coll, query)
-	if coll == "" {
-		coll = util.Config.Fromtable
-	}
-	i.SetSession("coll", coll)   //session中存入查询表
-	i.SetSession("query", query) //session中存入查询条件
-	listData := getListInfo(coll, query, pagenum)
-	i.T["list"] = listData
-	i.T["type"] = stype
-	i.T["field"] = field
-	i.T["spidercode"] = spidercode
-	i.T["coll"] = coll
-	i.T["topsubtype"] = util.TopSubStypeArr
-	i.T["allfield"] = util.AllFieldArr
-	i.T["fromtable"] = util.Config.Fromtable
-	i.T["dbname"] = util.Config.Dbname
-	i.T["max"] = max
-	i.T["min"] = min
-	i.T["hasno"] = hasno
-	i.T["notag"] = notag
-	checkedNum, allNum := GetCheckedAndAllDataInfo(query, coll) //已标和总数信息
-	i.T["checkednum"] = checkedNum
-	i.T["allnum"] = allNum
-	i.T["pagenum"] = pagenum
-	role := qu.IntAll(i.GetSession("role"))
-	i.T["role"] = role
-	i.T["labeler"] = labeler
-	if role >= AUDITOR {
-		i.T["labelers"] = GetLabeler(coll)
-	}
-	return i.Render("list.html", &i.T)
-}
-
-//判断id是否正在被标记
-func (i *Front) CheckId() {
-	defer qu.Catch()
-	CheckLock.Lock()
-	defer CheckLock.Unlock()
-	msg := ""
-	id := i.GetString("id")
-	coll := i.GetSession("coll").(string)
-	qu.Debug(coll, id)
-	if coll != "" {
-		exists, err := redis.Exists("extcheck", coll+"_"+id)
-		if err == nil {
-			if exists {
-				id, exists = GetNoCheckedId(id, coll)
-			}
-		} else {
-			msg = "校验数据出错！"
-		}
-		i.ServeJson(map[string]interface{}{"msg": msg, "id": id, "exists": exists})
-		return
-	}
-	i.ServeJson(map[string]interface{}{"msg": "数据校验表出错！"})
-}
-
-//标注
-func (i *Front) Biaozhu() error {
-	b := false
-	obj := []map[string]interface{}{}
-	//ispackage := i.GetString("ispackage")
-	key := i.GetString("key")
-	_id := i.GetString("_id")
-	stype, _ := i.GetInteger("stype")
-	err := json.Unmarshal([]byte(key), &obj)
-	if err != nil {
-		i.ServeJson(b)
-		return nil
-	}
-	//base := map[string]interface{}{}
-	set := map[string]interface{}{}    //更新、新增字段
-	unset := map[string]interface{}{}  //删除字段
-	errset := map[string]interface{}{} //记录修改的字段信息
-	isSaveMarked := false
-	if len(obj) == 1 { //单独保存某个一级
-		content, ok := obj[0]["content"].([]interface{})
-		if !ok || len(content) == 0 {
-			i.ServeJson(b)
-			return nil
-		}
-		title := qu.ObjToString(obj[0]["title"])
-		isext, _ := obj[0]["checkType"].(bool)
-		istag, _ := obj[0]["checkAllTag"].(bool)
-		status := qu.IntAll(obj[0]["status"])
-		switch title {
-		case "基本字段":
-			BzJBZD(content, set, unset, errset)
-		case "时间地点":
-			BzSJDD(content, set, unset, errset)
-		case "标的信息":
-			BzBDXX(content, set, unset, errset, isext, istag, status)
-		case "多包信息":
-			BzDBXX(content, set, unset, errset, isext, status)
-		case "中标候选人信息":
-			BzZBHXRXX(content, set, unset, errset, isext, status)
-		case "其余信息":
-			BzQYXX(content, set, unset, errset)
-		}
-	} else {
-		isSaveMarked = true
-		for j, val := range obj {
-			content, ok := val["content"].([]interface{})
-			status := qu.IntAll(val["status"])
-			if !ok {
-				continue
-				// i.ServeJson(b)
-				// return nil
-			}
-			isext, _ := val["checkType"].(bool)
-			istag, _ := val["checkAllTag"].(bool)
-			if j == 0 { //基本信息
-				BzJBZD(content, set, unset, errset)
-			} else if j == 1 { //时间地点
-				BzSJDD(content, set, unset, errset)
-			} else if j == 2 { //标的物
-				BzBDXX(content, set, unset, errset, isext, istag, status)
-			} else if j == 3 { //多包
-				BzDBXX(content, set, unset, errset, isext, status)
-			} else if j == 4 { //候选人
-				BzZBHXRXX(content, set, unset, errset, isext, status)
-			} else { //其余信息
-				BzQYXX(content, set, unset, errset)
-			}
-		}
-	}
-	qu.Debug("errset---", errset)
-	qu.Debug("set---", set)
-	qu.Debug("unset---", unset)
-	userInfo := map[string]interface{}{}     //记录本次标注日志
-	modifyFields := map[string]interface{}{} //本次标注所有字段
-	unerrset := map[string]interface{}{}     //记录errdata错误信息表删除
-
-	loginuser := i.GetSession("loginuser").(string)
-	coll := i.GetSession("coll").(string)
-	data, _ := util.MgoM.FindById(coll, _id, nil) //查询标注保存前的原始信息
-	modifyuser := qu.ObjToString((*data)["modifyuser"])
-	//标注多次操作处理
-	if len(set) > 0 { //set中为本次标注保存的数据(ck_bidopentime:1;ck_buyer:2;buyer:"XXX")
-		for s, sv := range set {
-			//特殊字段处理
-			if s == "ck_pclistag" && (*data)["ck_pclistag"] != nil {
-				delete(set, s)
-				continue
-			}
-			//区分是标记字段，还是普通字段(标记字段:ck_buyer;普通字段:buyer)
-			if strings.HasPrefix(s, preKey) {
-				status := qu.IntAll(sv)                                        //标注字段状态
-				field := s[3:]                                                 //去除前缀，被标注字段
-				checkedStatus := qu.IntAll((*data)[s])                         //数据库被标注状态
-				if status == 1 && (checkedStatus == 2 || checkedStatus == 1) { //数据库已有标记,且页面标注正确,此次该字段不做标注保存
-					qu.Debug("已标注字段field---", field)
-					delete(set, field)
-					delete(set, s)
-					continue
-				} else {
-					qu.Debug("未标注字段field---", field, status)
-					if set[field] == nil && status != 2 { //表示该字段没有要修改的值，此时标记为1或2(亦或是ck_wodrisext;ck_pkgisext;ck_pclisext)
-						modifyFields[field] = set[s] //目前会有pclisext、pclistag、pkgisext、wodrisext
-						continue
-					} else { //此时有修改的值和标记2
-						modifyFields[field] = status //记录标注的字段和状态
-						if checkedStatus != 0 {      //已有标注信息的状态
-							if status == 2 && checkedStatus == 2 { //字段已被修改且本次也为修改
-								errdata, _ := util.MgoM.FindById(util.Config.Totablel, _id, map[string]interface{}{"userinfo": 0})
-								//if field == "package" || field == "purchasinglist" || field == "winnerorder" {
-								//}
-								if errdata != nil && len(*errdata) != 0 {
-									qu.Debug(field, (*errdata)[field] != nil, fmt.Sprint((*errdata)[field]))
-									delete(errset, field) //errdata中有该字段的错误记录，不再更新errdata
-									//对比errdata错误值:相等变为1，errdata删除;不等变为2，标注表修改该字段
-									errText := fmt.Sprint((*errdata)[field])
-									text := fmt.Sprint(set[field])
-									qu.Debug("errText---", errText, text)
-									if errText == text {
-										set[s] = 1           //更新标注表字段状态
-										unerrset[field] = "" //删除errdata表字段
-										//delete(errset, field)
-									}
-								}
-							}
-						}
-					}
-				}
-			}
-		}
-	}
-	qu.Debug("errset---", errset)
-	qu.Debug("set---", set)
-	qu.Debug("unset---", unset)
-	qu.Debug("unerrset---", unerrset)
-	//1、errdata操作
-	//存储原错误信息
-	errUpdata := map[string]interface{}{} //errdata所有要更新内容
-	for f, v := range errset {
-		qu.Debug("err filed---", f, v)
-		//if f != "package" && f != "purchasinglist" && f != "winnerorder" {
-		errset[f] = (*data)[f]
-		//}
-	}
-	if len(modifyFields) > 0 { //有标注字段，记录标注信息
-		errset["updatetime"] = time.Now().Unix()
-		errset["modifyuser"] = loginuser
-		userInfo["fields"] = modifyFields
-		userInfo["updatetime"] = time.Now().Unix()
-		userInfo["modifyuser"] = loginuser
-		errUpdata["$push"] = map[string]interface{}{
-			"userinfo": map[string]interface{}{
-				"$each":     []interface{}{userInfo},
-				"$position": 0,
-			},
-		}
-	}
-	qu.Debug("set---", set)
-	qu.Debug("errset---", errset)
-	qu.Debug("unerrset---", unerrset)
-	qu.Debug("userInfo---", userInfo)
-	if len(errset) > 0 {
-		errUpdata["$set"] = errset
-	}
-	if len(unerrset) > 0 {
-		errUpdata["$unset"] = unerrset
-	}
-	if len(errUpdata) > 0 {
-		util.MgoM.Update(util.Config.Totablel, `{"_id":"`+_id+`"}`, errUpdata, true, false)
-	}
-	//更新表操作
-	//qu.Debug("set---", set)
-	//更新正确信息
-	update := map[string]interface{}{}
-	if len(set) > 0 {
-		set["ck_data"] = stype
-		set["updatetime"] = time.Now().Unix()
-		if modifyuser == "" {
-			set["modifyuser"] = modifyuser
-		}
-		update["$set"] = set
-	}
-	qu.Debug("unset---", unset)
-	if len(unset) > 0 {
-		update["$unset"] = unset
-	}
-	qu.Debug("update---", update)
-	if len(update) > 0 {
-		b = util.MgoM.Update(coll, `{"_id":"`+_id+`"}`, update, false, false)
-		if b {
-			if coll != util.Config.Fromtable && isSaveMarked { //util.coll此时的标注表  util.Config.Fromtable默认标注表
-				data, _ := util.MgoM.FindById(coll, _id, nil)
-				if data != nil && len(*data) > 0 {
-					update := map[string]interface{}{"$set": *data}
-					if len(unset) > 0 {
-						update["$unset"] = unset
-					}
-					if !util.MgoM.Update(util.Config.Fromtable, map[string]interface{}{"_id": (*data)["_id"]}, update, true, false) {
-						qu.Debug("同步marked失败，", _id)
-					}
-				}
-			}
-		}
-		i.ServeJson(b)
-	} else {
-		i.ServeJson(true)
-	}
-
-	return nil
-}
-
-//查询信息
-func (i *Front) Detail(id string) error {
-	coll, _ := i.GetSession("coll").(string)
-	//if coll != "" {
-	//	exists, _ := redis.Exists("extcheck", coll+"_"+id)
-	//	if exists {
-	//		tmpId, exists := GetNoCheckedId(id, coll)
-	//		if exists { //标注完成
-	//			return i.Render("finish.html", &i.T)
-	//		} else {
-	//			id = tmpId
-	//		}
-	//	}
-	//} else {
-	//	i.T["err"] = "数据查询表为空！"
-	//	return i.Render("err.html", &i.T)
-	//}
-	query, _ := i.GetSession("query").(map[string]interface{})
-	qu.Debug(coll, query)
-	rep := getDetail(id, coll)          //获取本条公告的信息
-	i.T["otherInfo"] = rep["otherInfo"] //展示关联公告信息
-	i.T["moreInfo"] = rep["moreInfo"]   //更多关联公告信息
-	//i.T["jyhref"] = util.JYHREFPRE + qu.CommonEncodeArticle("content", id) + ".html"
-	i.T["info"] = rep["info"]
-	i.T["common"] = rep["common"]
-	i.T["worder"] = rep["worder"]
-	i.T["packs"] = rep["packs"]
-	i.T["packskey"] = rep["packskey"]
-	//i.T["ck_pclisext"] = rep["ck_pclisext"]
-	i.T["ck_pclistag"] = rep["ck_pclistag"]
-	//i.T["ck_wodrisext"] = rep["ck_wodrisext"]
-	//i.T["ck_pkgisext"] = rep["ck_pkgisext"]
-	i.T["timeplace"] = rep["timeplace"]
-	i.T["purchasinglist"] = rep["purchasinglist"]
-	i.T["other"] = rep["other"]
-	i.T["PurchasinglistField"] = util.PurchasinglistField
-	i.T["WinnerorderField"] = util.WinnerorderField
-	i.T["PackageField"] = util.PackageField
-	i.T["topsubtype"] = util.TopSubStypeArr2
-	i.T[preKey+"purchasinglist"] = rep[preKey+"purchasinglist"]
-	i.T[preKey+"package"] = rep[preKey+"package"]
-	i.T[preKey+"winnerorder"] = rep[preKey+"winnerorder"]
-	i.T["worder_new"] = rep["worder_new"]
-	i.T["pcl_new"] = rep["pcl_new"]
-	i.T["pkg_new"] = rep["pkg_new"]
-	i.T["nextid"] = GetNextDataId(id, coll, query)              //下一条id
-	checkedNum, allNum := GetCheckedAndAllDataInfo(query, coll) //已标和总数信息
-	i.T["checkednum"] = checkedNum
-	i.T["allnum"] = allNum
-	i.T["fields"] = util.Config.Fields
-	//存入Redis
-	redis.Put("extcheck", coll+"_"+id, "", util.Config.RedisTimeout*60) //正在标注的数据存入redis避免多人同时标注(加上coll左前缀避免不同表相同id数据不能同时标注)
-	return i.Render("detail.html", &i.T)
-}
-
-//通过excel表格导入
-func (i *Front) ImportByExcel() {
-	defer qu.Catch()
-	//success := false
-	msg := ""
-	importNum := 0
-	successNum := int64(0)
-	coll := i.GetString("excelcoll")
-	if coll == "" {
-		i.ServeJson(map[string]interface{}{"msg": "表名错误"})
-		return
-	}
-	mf, _, err := i.GetFile("xlsx")
-	if err == nil {
-		binary, _ := ioutil.ReadAll(mf)
-		xls, _ := xlsx.OpenBinary(binary)
-		sheet := xls.Sheets[0]
-		rows := sheet.Rows
-		idcolnum := -1
-		ids := []string{}
-		cellFieldName := map[int]string{}             //记录客户需求字段所在的列
-		tmpMap := map[string]map[string]interface{}{} //excel表中需要保存字段集合
-		for rn, row := range rows {
-			if rn == 0 {
-				for j, cell := range row.Cells {
-					title := cell.Value
-					if fieldName := util.Config.CustomerField[title]; fieldName != "" { //客户需求字段
-						cellFieldName[j] = fieldName
-					}
-					if title == "唯一标识" || title == "信息标识" { //id所在列
-						idcolnum = j
-					}
-				}
-				if idcolnum == -1 {
-					break
-				}
-				continue
-			}
-			if len(row.Cells) < len(rows[0].Cells) {
-				break
-			}
-			tmp := map[string]interface{}{}
-			for j, f := range cellFieldName {
-				if val := row.Cells[j].Value; val != "" {
-					if f == "capital" { //注册资金（万元）
-						cf, _ := row.Cells[j].Float()
-						tmp[f] = cf
-					} else if f == "createtime" { //创建时间
-						ci, _ := row.Cells[j].Int64()
-						tmp[f] = ci
-					} else {
-						tmp[f] = val
-					}
-				}
-			}
-			id := row.Cells[idcolnum].String() //加密的id
-			if id == "" {
-				break
-			}
-			id = SE.DecodeString(id) //解密后id
-			tmpMap[id] = tmp
-			ids = append(ids, id)
-		}
-		importNum = len(ids) //excel表数据个数
-		if importNum > 0 {
-			_, msg, successNum = GetDataById1(coll, ids, "excel", tmpMap)
-			tmpMap = map[string]map[string]interface{}{}
-			ids = []string{}
-		}
-	}
-	msg = fmt.Sprintf("共查询%d条，导入成功%d条\n", importNum, successNum) + msg
-	i.ServeJson(map[string]interface{}{"msg": msg})
-}
-
-func (i *Front) GetEsCount() {
-	defer qu.Catch()
-	msg := ""
-	count := int64(0)
-	estext := i.GetString("estext") //es查询语句
-	esJson := map[string]interface{}{}
-	if json.Unmarshal([]byte(estext), &esJson) != nil || len(esJson) == 0 {
-		msg = "Es语句错误"
-	} else {
-		count = util.Es.Count(util.Index, util.Itype, estext)
-	}
-	i.ServeJson(map[string]interface{}{"count": count, "msg": msg})
-}
-
-//通过es语句导入
-func (i *Front) ImportByEs() {
-	defer qu.Catch()
-	//success := false
-	msg := ""
-	successNum := int64(0)
-	estext := i.GetString("estext") //es查询语句
-	coll := i.GetString("coll")     //导入表
-	if coll == "" {
-		i.ServeJson(map[string]interface{}{"msg": "表名错误"})
-		return
-	}
-	client := util.Es.GetEsConn()
-	defer util.Es.DestoryEsConn(client)
-	ch := make(chan bool, 5)
-	wg := &sync.WaitGroup{}
-	lock := &sync.Mutex{}
-	escount := util.Es.Count(util.Index, util.Itype, estext)
-	qu.Debug("查询总数:", escount)
-	if escount > 0 {
-		//查询条件类型转换
-		var q es.Query
-		tmpQuery := es.BoolQuery{
-			QueryStrings: estext,
-		}
-		q = tmpQuery
-		//游标查询，index不支持别名，只能写索引库的名称
-		res, err := client.Scroll(util.Index).Query(q).Size(200).Do() //查询一条获取游标
-		ids := []string{}                                             //id数据
-		if err == nil {
-			numDocs := 0
-			scrollId := res.ScrollId
-			for {
-				if scrollId == "" {
-					qu.Debug("ScrollId Is Error")
-					break
-				}
-				searchResult, err := client.Scroll(util.Index).Size(200).ScrollId(scrollId).Do() //查询
-				if err != nil {
-					if err.Error() == "EOS" { //迭代完毕
-						qu.Debug("Es Search Data Over：", err)
-					} else {
-						qu.Debug("Es Search Data Error：", err)
-					}
-					break
-				}
-				for _, hit := range searchResult.Hits.Hits {
-					//开始处理数据
-					wg.Add(1)
-					ch <- true
-					go func(tmpHit *es.SearchHit) {
-						defer func() {
-							<-ch
-							wg.Done()
-						}()
-						tmp := make(map[string]interface{})
-						if json.Unmarshal(*tmpHit.Source, &tmp) == nil {
-							id := qu.ObjToString(tmp["_id"])
-							tmp["id"] = id //记录数据原有id
-							lock.Lock()
-							ids = append(ids, id)
-							lock.Unlock()
-						}
-					}(hit)
-					numDocs += 1
-					if numDocs%500 == 0 {
-						qu.Debug("Current：", numDocs)
-					}
-				}
-				scrollId = searchResult.ScrollId
-			}
-			wg.Wait()
-			client.ClearScroll().ScrollId(scrollId).Do() //清理游标
-			//qu.Debug("Result Data Count：", numDocs)
-		} else {
-			qu.Debug("Es Search Data Error")
-		}
-		//判断数量
-		if int64(len(ids)) != escount {
-			msg = "查询数据和结果不一致"
-		} else { //入库
-			_, msg, successNum = GetDataById(coll, ids, "es", map[string]map[string]interface{}{})
-		}
-	} else {
-		msg = "无查询数据"
-	}
-	msg = fmt.Sprintf("共查询%d条，导入成功%d条\n", escount, successNum) + msg
-	i.ServeJson(map[string]interface{}{"msg": msg})
-}
-
 //查找未被标注的数据id
 func GetNoCheckedId(id, coll string) (string, bool) {
 	defer qu.Catch()
@@ -652,79 +39,6 @@ func GetNoCheckedId(id, coll string) (string, bool) {
 	return id, true
 }
 
-//同步数据
-func (i *Front) SyncMarked() {
-	syncColl := i.GetString("coll")
-	sess := util.MgoM.GetMgoConn()
-	defer util.MgoM.DestoryMongoConn(sess)
-	it := sess.DB(util.Config.Dbname).C(syncColl).Find(nil).Iter()
-	count, _ := sess.DB(util.Config.Dbname).C(syncColl).Find(nil).Count()
-	qu.Debug(syncColl, count)
-	n := 0
-	lock := &sync.Mutex{}
-	wg := &sync.WaitGroup{}
-	ch := make(chan bool, 3)
-	result := map[string]map[string]interface{}{}
-	idArr := []string{}
-	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
-		wg.Add(1)
-		ch <- true
-		go func(tmp map[string]interface{}) {
-			defer func() {
-				<-ch
-				wg.Done()
-			}()
-			id := qu.ObjToString(tmp["id"])
-			if id == "" {
-				return
-			}
-			tmpMap := map[string]interface{}{}
-			for _, f := range util.Config.CustomerField {
-				if val := tmp[f]; val != nil {
-					tmpMap[f] = val
-				}
-			}
-			lock.Lock()
-			idArr = append(idArr, id)
-			result[id] = tmpMap
-			lock.Unlock()
-		}(tmp)
-		tmp = map[string]interface{}{}
-	}
-	wg.Wait()
-	if count != int64(len(result)) {
-		i.ServeJson(map[string]interface{}{"msg": "同步失败", "flag": false})
-		return
-	}
-	//util.MgoM.C.Database(util.Config.Dbname).Collection(syncColl).Drop(util.MgoM.Ctx) //删除syncColl表(清空数据)
-	success, msg, successNum := GetDataById(syncColl, idArr, "syncoll", result)
-	result = map[string]map[string]interface{}{}
-	idArr = []string{}
-	msg = fmt.Sprintf("共查询%d条，同步成功%d条\n", count, successNum) + msg
-	i.ServeJson(map[string]interface{}{"msg": msg, "flag": success})
-}
-
-//将marked表中ck_data：2更新为0
-func (i *Front) MarkedInit() {
-	set := map[string]interface{}{
-		"$set": map[string]interface{}{
-			"ck_data": 0,
-		},
-	}
-	b := util.MgoM.Update(util.Config.Fromtable, `{"ck_data":2}`, set, false, true)
-	i.ServeJson(map[string]interface{}{"success": b})
-}
-
-//标注完成
-func (i *Front) FinishCheck() {
-	i.Render("finish.html")
-}
-
-//错误页面
-func (i *Front) ErrCheck() {
-	i.Render("err.html")
-}
-
 //查询列表数据
 func getListInfo(coll string, query map[string]interface{}, currentpage int) []map[string]interface{} {
 	start := (currentpage - 1) * 50
@@ -1759,7 +1073,7 @@ func GetDataById1(coll string, ids []string, stype string, tmp map[string]map[st
 				}
 				// 处理	package winner_all
 				if p, o1 := (*bidData)["package"].(map[string]interface{}); o1 {
-					for _, v := range p{
+					for _, v := range p {
 						v1 := v.(map[string]interface{})
 						t := make(map[string]interface{})
 						if v1["winner"] != nil {
@@ -1777,14 +1091,14 @@ func GetDataById1(coll string, ids []string, stype string, tmp map[string]map[st
 				(*bidData)["filetext"] = util.GetFileText(*bidData)
 				qu.Debug("es find project")
 				// es查询项目合并信息
-				esQ := `{"query":{"bool":{"must":[{"term":{"ids":"`+id+`"}}]}}}`
+				esQ := `{"query":{"bool":{"must":[{"term":{"ids":"` + id + `"}}]}}}`
 				info := util.Es.Get("projectset", "projectset", esQ)
 				if len(*info) == 1 {
 					ids := qu.ObjArrToStringArr((*info)[0]["ids"].([]interface{}))
 					if len(ids) > 0 {
 						var infolist []map[string]interface{}
-						for _, v := range ids{
-							if v == id {	// 当前公告
+						for _, v := range ids {
+							if v == id { // 当前公告
 								continue
 							}
 							if v < util.BIDDINGSTARTID {
@@ -1806,7 +1120,7 @@ func GetDataById1(coll string, ids []string, stype string, tmp map[string]map[st
 						}
 						(*bidData)["info"] = infolist
 					}
-				}else {
+				} else {
 					qu.Debug("projectset find err", esQ)
 				}
 
@@ -1893,79 +1207,6 @@ func GetCheckedAndAllDataInfo(query map[string]interface{}, coll string) (int, i
 	return checkedCount, allCount
 }
 
-//统计抽查
-func (i *Front) Tj() error {
-	comm := map[string]map[string]int{}
-	comm_win := map[string]map[string]int{}
-	pack := map[string]map[string]int{}
-	pack_win := map[string]map[string]int{}
-	list, _ := util.MgoM.Find(util.Config.Totablel, "{}", nil, nil, false, -1, -1)
-	for _, tmp := range *list {
-		for k, val := range tmp {
-			if len(k) > 3 && k[:3] == preKey {
-				comm = mapIntAdd(k, qu.ObjToString(val), comm)
-			}
-		}
-		if winnerorder, ok := tmp["winnerorder"].([]interface{}); ok {
-			for _, wd := range winnerorder {
-				if winner, ok := wd.(map[string]interface{}); ok {
-					for k, val := range winner {
-						if len(k) > 3 && k[:3] == preKey {
-							comm_win = mapIntAdd(k, qu.ObjToString(val), comm_win)
-						}
-					}
-				}
-			}
-		}
-		if ptmp, ok := tmp["package"].(map[string]interface{}); ok {
-			for _, pktmp := range ptmp {
-				if pkage, ok := pktmp.(map[string]interface{}); ok {
-					for k, val := range pkage {
-						if len(k) > 3 && k[:3] == preKey {
-							pack = mapIntAdd(k, qu.ObjToString(val), pack)
-						}
-						if k == "winnerorder" {
-							if wtmp, ok := val.([]interface{}); ok {
-								for _, winner := range wtmp {
-									if win, ok := winner.(map[string]interface{}); ok {
-										for wk, wval := range win {
-											if len(k) > 3 && k[:3] == preKey {
-												pack_win = mapIntAdd(wk, qu.ObjToString(wval), pack_win)
-											}
-										}
-									}
-								}
-							}
-						}
-					}
-				}
-			}
-		}
-	}
-	//data := map[string]interface{}{"total": len(list), "comm": comm, "pack": pack, "pack_win": pack_win}
-	//mongodb.Save("extcheck_tj", data)
-	i.T["comm"] = comm
-	i.T["pack"] = pack
-	i.T["comm_win"] = comm_win
-	i.T["pack_win"] = pack_win
-	i.T["total"] = len(*list)
-	return i.Render("tj.html", &i.T)
-}
-
-//标错列表
-func (i *Front) Elist() error {
-	attrname := i.GetString("attrname")
-	common := util.Config.Biaozhu["common"]
-	elist, _ := util.MgoM.Find(util.Config.Totablel, `{"ck_`+attrname+`":"0"}`, `{"_id":1}`, `{"_id":1}`, false, -1, -1)
-	for _, v := range *elist {
-		v["_id"] = mgo.BsonIdToSId(v["_id"])
-	}
-	i.T["elist"] = *elist
-	i.T["attrname"] = attrname
-	i.T["common"] = common
-	return i.Render("elist.html", &i.T)
-}
-
 //查询表中已标数据的标注人
 func GetLabeler(coll string) (labeler []string) {
 	defer qu.Catch()
@@ -1992,152 +1233,3 @@ func GetLabeler(coll string) (labeler []string) {
 	qu.Debug("labeler:", labeler)
 	return
 }
-
-
-func (i *Front) Review() error {
-	defer qu.Catch()
-	var labeler []map[string]interface{}
-	sess := util.MgoM.GetMgoConn()
-	defer util.MgoM.DestoryMongoConn(sess)
-	sess.DB(util.MgoM.DbName).C(util.Config.Fromtable).Pipe([]map[string]interface{}{
-		//查询条件
-		{
-			"$match": bson.M{
-				"ck_data": bson.M{"$gte": 1},
-			},
-		},
-		// 按照modifyuser分组，并统计
-		{
-			"$group": bson.M{
-				"_id": "$modifyuser",
-				"count": bson.M{"$sum": 1},
-			},
-		},
-		// 排序
-		{
-			"$sort": bson.M{"count": -1},
-		},
-	}).All(&labeler)
-
-	qu.Debug(labeler)
-	i.T["users"] = labeler
-	return i.Render("review.html", &i.T)
-}
-
-func (i *Front) ReviewList() {
-	defer qu.Catch()
-	if i.Method() == "POST" {
-		user := i.GetString("user")
-		q := make(map[string]interface{})
-		q["ck_data"] = bson.M{"$gte": 1}
-		if user != "0" && user != "-1"  {
-			q["modifyuser"] = user
-		}
-		qu.Debug(q, util.Config.Fromtable)
-		datas, b := util.MgoM.Find(util.Config.Fromtable, q, nil, nil, false, -1, -1)
-		if b && len(*datas) > 0 {
-			i.ServeJson(map[string]interface{}{
-				"rep":     b,
-				"data":    *datas,
-			})
-		} else {
-			i.ServeJson(map[string]interface{}{
-				"rep": false,
-				"msg": "未查询到数据",
-			})
-		}
-	}
-}
-
-func (i *Front) ReviewDetail(id string) error {
-	defer qu.Catch()
-	coll, _ := i.GetSession("coll").(string)
-	query, _ := i.GetSession("query").(map[string]interface{})
-	qu.Debug(coll, query)
-	rep := getDetail(id, coll)          //获取本条公告的信息
-	i.T["otherInfo"] = rep["otherInfo"] //展示关联公告信息
-	i.T["moreInfo"] = rep["moreInfo"]   //更多关联公告信息
-	i.T["info"] = rep["info"]
-	i.T["common"] = rep["common"]
-	i.T["worder"] = rep["worder"]
-	i.T["packs"] = rep["packs"]
-	i.T["packskey"] = rep["packskey"]
-	i.T["ck_pclisext"] = rep["ck_pclisext"]
-	i.T["ck_wodrisext"] = rep["ck_wodrisext"]
-	i.T["ck_pkgisext"] = rep["ck_pkgisext"]
-	i.T["timeplace"] = rep["timeplace"]
-	i.T["purchasinglist"] = rep["purchasinglist"]
-	i.T["other"] = rep["other"]
-	i.T["PurchasinglistField"] = util.PurchasinglistField
-	i.T["WinnerorderField"] = util.WinnerorderField
-	i.T["PackageField"] = util.PackageField
-	i.T["topsubtype"] = util.TopSubStypeArr2
-	i.T[preKey+"purchasinglist"] = rep[preKey+"purchasinglist"]
-	i.T[preKey+"package"] = rep[preKey+"package"]
-	i.T[preKey+"winnerorder"] = rep[preKey+"winnerorder"]
-	i.T["worder_new"] = rep["worder_new"]
-	i.T["pcl_new"] = rep["pcl_new"]
-	i.T["pkg_new"] = rep["pkg_new"]
-	i.T["nextid"] = GetNextDataId(id, coll, query)              //下一条id
-	return i.Render("re_detail.html", &i.T)
-}
-
-func (i *Front) ReviewSave() {
-	defer qu.Catch()
-	//_id := i.GetString("_id")
-
-}
-
-func (i *Front) ReviewStat() {
-	defer qu.Catch()
-	sess := util.MgoM.GetMgoConn()
-	defer util.MgoM.DestoryMongoConn(sess)
-	result := sess.DB(util.MgoM.DbName).C(util.Config.Fromtable).Find(nil).Iter()
-	count, cmark, rmark, rgmark := 0, 0, 0, 0		// 总数， 标注数量，	审核数据，	审核数据完全正确的数据量
-
-	cmaps := make(map[string]int)			// 标注字段整体准确率
-	umaps := make(map[string]interface{})	// 按人员 字段准确率
-	//for _, i2 := range util.Config.Field {
-	//	cmaps[i2] = 0
-	//}
-	for tmp := make(map[string]interface{}); result.Next(&tmp); {
-		count ++
-		if qu.IntAll(tmp["ck_data"]) >= 1 {
-			cmark ++
-		}
-		if qu.IntAll(tmp["review"]) > 0 {
-			rmark ++
-			if reField, ok := tmp["re_field"].(map[string]interface{}); ok {
-				if user, o1 := tmp["modifyuser"].(string); o1 {
-
-					// 每个人员标注的质检数量
-					if umaps["count"] != nil {
-						umaps["count"] = qu.IntAll(umaps["count"]) + 1
-					}else {
-						umaps["count"] = 1
-					}
-					// 按人员统计每个字段
-					var ump map[string]int
-					if umaps[user] != nil {
-						ump, _ = umaps[user].(map[string]int)
-					}else {
-						ump = make(map[string]int)
-					}
-
-					flag := true			// 数据整体准确率
-					for k, v := range reField{
-						if v == 1 {
-							cmaps[k] ++
-							ump[k] ++
-						}else {
-							flag = false
-						}
-					}
-					if flag {
-						rgmark ++
-					}
-				}
-			}
-		}
-	}
-}

src/front/server.go

@@ -1,5 +1,925 @@
+// front
 package front
 
-func getOtherInfo() {
+import (
+	"encoding/json"
+	"io/ioutil"
 
+	//"container/list"
+	"fmt"
+	qu "qfw/util"
+	"qfw/util/redis"
+
+	mgo "mongodb"
+	"strings"
+	"sync"
+	"time"
+	"util"
+
+	"github.com/go-xweb/xweb"
+	"go.mongodb.org/mongo-driver/bson"
+
+	"github.com/tealeg/xlsx"
+	es "gopkg.in/olivere/elastic.v1"
+)
+
+type Front struct {
+	*xweb.Action
+	// elist         xweb.Mapper `xweb:"/elist"`
+	login         xweb.Mapper `xweb:"/"`
+	saveUser      xweb.Mapper `xweb:"/center/saveuser"`
+	listInfo      xweb.Mapper `xweb:"/center/list"`
+	checkId       xweb.Mapper `xweb:"/center/checkid"`
+	detail        xweb.Mapper `xweb:"/center/detail/(.*).html"`
+	biaozhu       xweb.Mapper `xweb:"/center/biaozhu"`
+	importByExcel xweb.Mapper `xweb:"/center/importbyexcel"`
+	importByEs    xweb.Mapper `xweb:"/center/importbyes"`
+	getEsCount    xweb.Mapper `xweb:"/center/getescount"`
+	finishCheck   xweb.Mapper `xweb:"/center/finishcheck"`
+	errCheck      xweb.Mapper `xweb:"/center/errcheck"`
+	syncMarked    xweb.Mapper `xweb:"/center/syncmarked"`
+	markedInit    xweb.Mapper `xweb:"/center/markedinit"`
+
+	review		  xweb.Mapper `xweb:"/center/review"`		// 质检
+	reviewList	  xweb.Mapper `xweb:"/center/reviewlist"`	// 质检数据列表
+	reviewDetail  xweb.Mapper `xweb:"/center/reviewdetail/(.*).html"`
+	reviewSave    xweb.Mapper `xweb:"/center/reviewsave"`
+	reviewStat	  xweb.Mapper `xweb:"/center/review/result"`
+}
+
+var preKey = "ck_"
+var preErr = "err_"
+var SE = qu.SimpleEncrypt{Key: "topJYBX2019"}
+var CheckLock = &sync.Mutex{}
+var LABELER, AUDITOR, ADMIN = 1, 2, 3 //标注人员，审核人员，管理员
+
+/*
+	status
+		-1:初始未验证
+		1:正确
+		2:新增
+		3:修改
+		4:删除
+*/
+
+//列表
+func (i *Front) ListInfo() error {
+	//loginuser := i.GetSession("loginuser").(string)
+	spidercode := i.GetString("spidercode")
+	coll := i.GetString("coll")
+	field := i.GetString("field")
+	stype := i.GetString("type")
+	min := i.GetString("minval") //min max不用int类型接收，以免有默认值0
+	max := i.GetString("maxval")
+	hasno, _ := i.GetBool("hasno")
+	notag, _ := i.GetBool("notag")
+	pagenum, _ := i.GetInteger("pagenum")
+	labeler := i.GetString("labeler")
+	if pagenum == 0 { //页码
+		pagenum = 1
+	}
+	qu.Debug("coll:", coll, "stype:", stype, "field:", field, "hasno:", hasno, "notag:", notag, "max:", max, "min:", min, "pagenum:", pagenum, "labeler:", labeler)
+	query := map[string]interface{}{}
+	if spidercode != "" { //爬虫代码
+		query["spidercode"] = spidercode
+	}
+	if stype != "-1" && stype != "" { //类型
+		if stype == util.SPECIALTYPE {
+			query["subtype"] = map[string]interface{}{
+				"$exists": false,
+			}
+		} else {
+			subtype := strings.Split(stype, "-")[1]
+			query["subtype"] = subtype
+		}
+	}
+	fieldScreen := false
+	if field != "-1" && field != "" { //字段
+		fieldScreen = true
+		queryfield := map[string]interface{}{
+			"$exists": !hasno,
+		}
+		if field == "budget" || field == "bidamount" { //金额区间
+			numMap := map[string]interface{}{}
+			if min != "" {
+				minint := qu.IntAll(min)
+				numMap["$gte"] = minint
+			}
+			if max != "" {
+				maxint := qu.IntAll(max)
+				numMap["$lte"] = maxint
+			}
+			if len(numMap) > 0 { //给定了区间，不再判断字段存在
+				queryfield = numMap
+			}
+		}
+		query[field] = queryfield
+	}
+	if fieldScreen && notag {
+		query[preKey+field] = map[string]interface{}{
+			"$exists": false,
+		}
+	} else if !fieldScreen && notag {
+		query["ck_data"] = 0
+	}
+
+	//标注人员
+	if labeler != "" {
+		query["modifyuser"] = labeler
+	}
+	qu.Debug("query:", coll, query)
+	if coll == "" {
+		coll = util.Config.Fromtable
+	}
+	i.SetSession("coll", coll)   //session中存入查询表
+	i.SetSession("query", query) //session中存入查询条件
+	listData := getListInfo(coll, query, pagenum)
+	i.T["list"] = listData
+	i.T["type"] = stype
+	i.T["field"] = field
+	i.T["spidercode"] = spidercode
+	i.T["coll"] = coll
+	i.T["topsubtype"] = util.TopSubStypeArr
+	i.T["allfield"] = util.AllFieldArr
+	i.T["fromtable"] = util.Config.Fromtable
+	i.T["dbname"] = util.Config.Dbname
+	i.T["max"] = max
+	i.T["min"] = min
+	i.T["hasno"] = hasno
+	i.T["notag"] = notag
+	checkedNum, allNum := GetCheckedAndAllDataInfo(query, coll) //已标和总数信息
+	i.T["checkednum"] = checkedNum
+	i.T["allnum"] = allNum
+	i.T["pagenum"] = pagenum
+	role := qu.IntAll(i.GetSession("role"))
+	i.T["role"] = role
+	i.T["labeler"] = labeler
+	if role >= AUDITOR {
+		i.T["labelers"] = GetLabeler(coll)
+	}
+	return i.Render("list.html", &i.T)
+}
+
+//判断id是否正在被标记
+func (i *Front) CheckId() {
+	defer qu.Catch()
+	CheckLock.Lock()
+	defer CheckLock.Unlock()
+	msg := ""
+	id := i.GetString("id")
+	coll := i.GetSession("coll").(string)
+	qu.Debug(coll, id)
+	if coll != "" {
+		exists, err := redis.Exists("extcheck", coll+"_"+id)
+		if err == nil {
+			if exists {
+				id, exists = GetNoCheckedId(id, coll)
+			}
+		} else {
+			msg = "校验数据出错！"
+		}
+		i.ServeJson(map[string]interface{}{"msg": msg, "id": id, "exists": exists})
+		return
+	}
+	i.ServeJson(map[string]interface{}{"msg": "数据校验表出错！"})
+}
+
+//标注
+func (i *Front) Biaozhu() error {
+	b := false
+	obj := []map[string]interface{}{}
+	//ispackage := i.GetString("ispackage")
+	key := i.GetString("key")
+	_id := i.GetString("_id")
+	stype, _ := i.GetInteger("stype")
+	err := json.Unmarshal([]byte(key), &obj)
+	if err != nil {
+		i.ServeJson(b)
+		return nil
+	}
+	//base := map[string]interface{}{}
+	set := map[string]interface{}{}    //更新、新增字段
+	unset := map[string]interface{}{}  //删除字段
+	errset := map[string]interface{}{} //记录修改的字段信息
+	isSaveMarked := false
+	if len(obj) == 1 { //单独保存某个一级
+		content, ok := obj[0]["content"].([]interface{})
+		if !ok || len(content) == 0 {
+			i.ServeJson(b)
+			return nil
+		}
+		title := qu.ObjToString(obj[0]["title"])
+		isext, _ := obj[0]["checkType"].(bool)
+		istag, _ := obj[0]["checkAllTag"].(bool)
+		status := qu.IntAll(obj[0]["status"])
+		switch title {
+		case "基本字段":
+			BzJBZD(content, set, unset, errset)
+		case "时间地点":
+			BzSJDD(content, set, unset, errset)
+		case "标的信息":
+			BzBDXX(content, set, unset, errset, isext, istag, status)
+		case "多包信息":
+			BzDBXX(content, set, unset, errset, isext, status)
+		case "中标候选人信息":
+			BzZBHXRXX(content, set, unset, errset, isext, status)
+		case "其余信息":
+			BzQYXX(content, set, unset, errset)
+		}
+	} else {
+		isSaveMarked = true
+		for j, val := range obj {
+			content, ok := val["content"].([]interface{})
+			status := qu.IntAll(val["status"])
+			if !ok {
+				continue
+				// i.ServeJson(b)
+				// return nil
+			}
+			isext, _ := val["checkType"].(bool)
+			istag, _ := val["checkAllTag"].(bool)
+			if j == 0 { //基本信息
+				BzJBZD(content, set, unset, errset)
+			} else if j == 1 { //时间地点
+				BzSJDD(content, set, unset, errset)
+			} else if j == 2 { //标的物
+				BzBDXX(content, set, unset, errset, isext, istag, status)
+			} else if j == 3 { //多包
+				BzDBXX(content, set, unset, errset, isext, status)
+			} else if j == 4 { //候选人
+				BzZBHXRXX(content, set, unset, errset, isext, status)
+			} else { //其余信息
+				BzQYXX(content, set, unset, errset)
+			}
+		}
+	}
+	qu.Debug("errset---", errset)
+	qu.Debug("set---", set)
+	qu.Debug("unset---", unset)
+	userInfo := map[string]interface{}{}     //记录本次标注日志
+	modifyFields := map[string]interface{}{} //本次标注所有字段
+	unerrset := map[string]interface{}{}     //记录errdata错误信息表删除
+
+	loginuser := i.GetSession("loginuser").(string)
+	coll := i.GetSession("coll").(string)
+	data, _ := util.MgoM.FindById(coll, _id, nil) //查询标注保存前的原始信息
+	modifyuser := qu.ObjToString((*data)["modifyuser"])
+	//标注多次操作处理
+	if len(set) > 0 { //set中为本次标注保存的数据(ck_bidopentime:1;ck_buyer:2;buyer:"XXX")
+		for s, sv := range set {
+			//特殊字段处理
+			if s == "ck_pclistag" && (*data)["ck_pclistag"] != nil {
+				delete(set, s)
+				continue
+			}
+			//区分是标记字段，还是普通字段(标记字段:ck_buyer;普通字段:buyer)
+			if strings.HasPrefix(s, preKey) {
+				status := qu.IntAll(sv)                                        //标注字段状态
+				field := s[3:]                                                 //去除前缀，被标注字段
+				checkedStatus := qu.IntAll((*data)[s])                         //数据库被标注状态
+				if status == 1 && (checkedStatus == 2 || checkedStatus == 1) { //数据库已有标记,且页面标注正确,此次该字段不做标注保存
+					qu.Debug("已标注字段field---", field)
+					delete(set, field)
+					delete(set, s)
+					continue
+				} else {
+					qu.Debug("未标注字段field---", field, status)
+					if set[field] == nil && status != 2 { //表示该字段没有要修改的值，此时标记为1或2(亦或是ck_wodrisext;ck_pkgisext;ck_pclisext)
+						modifyFields[field] = set[s] //目前会有pclisext、pclistag、pkgisext、wodrisext
+						continue
+					} else { //此时有修改的值和标记2
+						modifyFields[field] = status //记录标注的字段和状态
+						if checkedStatus != 0 {      //已有标注信息的状态
+							if status == 2 && checkedStatus == 2 { //字段已被修改且本次也为修改
+								errdata, _ := util.MgoM.FindById(util.Config.Totablel, _id, map[string]interface{}{"userinfo": 0})
+								//if field == "package" || field == "purchasinglist" || field == "winnerorder" {
+								//}
+								if errdata != nil && len(*errdata) != 0 {
+									qu.Debug(field, (*errdata)[field] != nil, fmt.Sprint((*errdata)[field]))
+									delete(errset, field) //errdata中有该字段的错误记录，不再更新errdata
+									//对比errdata错误值:相等变为1，errdata删除;不等变为2，标注表修改该字段
+									errText := fmt.Sprint((*errdata)[field])
+									text := fmt.Sprint(set[field])
+									qu.Debug("errText---", errText, text)
+									if errText == text {
+										set[s] = 1           //更新标注表字段状态
+										unerrset[field] = "" //删除errdata表字段
+										//delete(errset, field)
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	qu.Debug("errset---", errset)
+	qu.Debug("set---", set)
+	qu.Debug("unset---", unset)
+	qu.Debug("unerrset---", unerrset)
+	//1、errdata操作
+	//存储原错误信息
+	errUpdata := map[string]interface{}{} //errdata所有要更新内容
+	for f, v := range errset {
+		qu.Debug("err filed---", f, v)
+		//if f != "package" && f != "purchasinglist" && f != "winnerorder" {
+		errset[f] = (*data)[f]
+		//}
+	}
+	if len(modifyFields) > 0 { //有标注字段，记录标注信息
+		errset["updatetime"] = time.Now().Unix()
+		errset["modifyuser"] = loginuser
+		userInfo["fields"] = modifyFields
+		userInfo["updatetime"] = time.Now().Unix()
+		userInfo["modifyuser"] = loginuser
+		errUpdata["$push"] = map[string]interface{}{
+			"userinfo": map[string]interface{}{
+				"$each":     []interface{}{userInfo},
+				"$position": 0,
+			},
+		}
+	}
+	qu.Debug("set---", set)
+	qu.Debug("errset---", errset)
+	qu.Debug("unerrset---", unerrset)
+	qu.Debug("userInfo---", userInfo)
+	if len(errset) > 0 {
+		errUpdata["$set"] = errset
+	}
+	if len(unerrset) > 0 {
+		errUpdata["$unset"] = unerrset
+	}
+	if len(errUpdata) > 0 {
+		util.MgoM.Update(util.Config.Totablel, `{"_id":"`+_id+`"}`, errUpdata, true, false)
+	}
+	//更新表操作
+	//qu.Debug("set---", set)
+	//更新正确信息
+	update := map[string]interface{}{}
+	if len(set) > 0 {
+		set["ck_data"] = stype
+		set["updatetime"] = time.Now().Unix()
+		if modifyuser == "" {
+			set["modifyuser"] = modifyuser
+		}
+		update["$set"] = set
+	}
+	qu.Debug("unset---", unset)
+	if len(unset) > 0 {
+		update["$unset"] = unset
+	}
+	qu.Debug("update---", update)
+	if len(update) > 0 {
+		b = util.MgoM.Update(coll, `{"_id":"`+_id+`"}`, update, false, false)
+		if b {
+			if coll != util.Config.Fromtable && isSaveMarked { //util.coll此时的标注表  util.Config.Fromtable默认标注表
+				data, _ := util.MgoM.FindById(coll, _id, nil)
+				if data != nil && len(*data) > 0 {
+					update := map[string]interface{}{"$set": *data}
+					if len(unset) > 0 {
+						update["$unset"] = unset
+					}
+					if !util.MgoM.Update(util.Config.Fromtable, map[string]interface{}{"_id": (*data)["_id"]}, update, true, false) {
+						qu.Debug("同步marked失败，", _id)
+					}
+				}
+			}
+		}
+		i.ServeJson(b)
+	} else {
+		i.ServeJson(true)
+	}
+
+	return nil
+}
+
+//查询信息
+func (i *Front) Detail(id string) error {
+	coll, _ := i.GetSession("coll").(string)
+	//if coll != "" {
+	//	exists, _ := redis.Exists("extcheck", coll+"_"+id)
+	//	if exists {
+	//		tmpId, exists := GetNoCheckedId(id, coll)
+	//		if exists { //标注完成
+	//			return i.Render("finish.html", &i.T)
+	//		} else {
+	//			id = tmpId
+	//		}
+	//	}
+	//} else {
+	//	i.T["err"] = "数据查询表为空！"
+	//	return i.Render("err.html", &i.T)
+	//}
+	query, _ := i.GetSession("query").(map[string]interface{})
+	qu.Debug(coll, query)
+	rep := getDetail(id, coll)          //获取本条公告的信息
+	i.T["otherInfo"] = rep["otherInfo"] //展示关联公告信息
+	i.T["moreInfo"] = rep["moreInfo"]   //更多关联公告信息
+	//i.T["jyhref"] = util.JYHREFPRE + qu.CommonEncodeArticle("content", id) + ".html"
+	i.T["info"] = rep["info"]
+	i.T["common"] = rep["common"]
+	i.T["worder"] = rep["worder"]
+	i.T["packs"] = rep["packs"]
+	i.T["packskey"] = rep["packskey"]
+	//i.T["ck_pclisext"] = rep["ck_pclisext"]
+	i.T["ck_pclistag"] = rep["ck_pclistag"]
+	//i.T["ck_wodrisext"] = rep["ck_wodrisext"]
+	//i.T["ck_pkgisext"] = rep["ck_pkgisext"]
+	i.T["timeplace"] = rep["timeplace"]
+	i.T["purchasinglist"] = rep["purchasinglist"]
+	i.T["other"] = rep["other"]
+	i.T["PurchasinglistField"] = util.PurchasinglistField
+	i.T["WinnerorderField"] = util.WinnerorderField
+	i.T["PackageField"] = util.PackageField
+	i.T["topsubtype"] = util.TopSubStypeArr2
+	i.T[preKey+"purchasinglist"] = rep[preKey+"purchasinglist"]
+	i.T[preKey+"package"] = rep[preKey+"package"]
+	i.T[preKey+"winnerorder"] = rep[preKey+"winnerorder"]
+	i.T["worder_new"] = rep["worder_new"]
+	i.T["pcl_new"] = rep["pcl_new"]
+	i.T["pkg_new"] = rep["pkg_new"]
+	i.T["nextid"] = GetNextDataId(id, coll, query)              //下一条id
+	checkedNum, allNum := GetCheckedAndAllDataInfo(query, coll) //已标和总数信息
+	i.T["checkednum"] = checkedNum
+	i.T["allnum"] = allNum
+	i.T["fields"] = util.Config.Fields
+	//存入Redis
+	redis.Put("extcheck", coll+"_"+id, "", util.Config.RedisTimeout*60) //正在标注的数据存入redis避免多人同时标注(加上coll左前缀避免不同表相同id数据不能同时标注)
+	return i.Render("detail.html", &i.T)
+}
+
+//通过excel表格导入
+func (i *Front) ImportByExcel() {
+	defer qu.Catch()
+	//success := false
+	msg := ""
+	importNum := 0
+	successNum := int64(0)
+	coll := i.GetString("excelcoll")
+	if coll == "" {
+		i.ServeJson(map[string]interface{}{"msg": "表名错误"})
+		return
+	}
+	mf, _, err := i.GetFile("xlsx")
+	if err == nil {
+		binary, _ := ioutil.ReadAll(mf)
+		xls, _ := xlsx.OpenBinary(binary)
+		sheet := xls.Sheets[0]
+		rows := sheet.Rows
+		idcolnum := -1
+		ids := []string{}
+		cellFieldName := map[int]string{}             //记录客户需求字段所在的列
+		tmpMap := map[string]map[string]interface{}{} //excel表中需要保存字段集合
+		for rn, row := range rows {
+			if rn == 0 {
+				for j, cell := range row.Cells {
+					title := cell.Value
+					if fieldName := util.Config.CustomerField[title]; fieldName != "" { //客户需求字段
+						cellFieldName[j] = fieldName
+					}
+					if title == "唯一标识" || title == "信息标识" { //id所在列
+						idcolnum = j
+					}
+				}
+				if idcolnum == -1 {
+					break
+				}
+				continue
+			}
+			if len(row.Cells) < len(rows[0].Cells) {
+				break
+			}
+			tmp := map[string]interface{}{}
+			for j, f := range cellFieldName {
+				if val := row.Cells[j].Value; val != "" {
+					if f == "capital" { //注册资金（万元）
+						cf, _ := row.Cells[j].Float()
+						tmp[f] = cf
+					} else if f == "createtime" { //创建时间
+						ci, _ := row.Cells[j].Int64()
+						tmp[f] = ci
+					} else {
+						tmp[f] = val
+					}
+				}
+			}
+			id := row.Cells[idcolnum].String() //加密的id
+			if id == "" {
+				break
+			}
+			id = SE.DecodeString(id) //解密后id
+			tmpMap[id] = tmp
+			ids = append(ids, id)
+		}
+		importNum = len(ids) //excel表数据个数
+		if importNum > 0 {
+			_, msg, successNum = GetDataById1(coll, ids, "excel", tmpMap)
+			tmpMap = map[string]map[string]interface{}{}
+			ids = []string{}
+		}
+	}
+	msg = fmt.Sprintf("共查询%d条，导入成功%d条\n", importNum, successNum) + msg
+	i.ServeJson(map[string]interface{}{"msg": msg})
+}
+
+func (i *Front) GetEsCount() {
+	defer qu.Catch()
+	msg := ""
+	count := int64(0)
+	estext := i.GetString("estext") //es查询语句
+	esJson := map[string]interface{}{}
+	if json.Unmarshal([]byte(estext), &esJson) != nil || len(esJson) == 0 {
+		msg = "Es语句错误"
+	} else {
+		count = util.Es.Count(util.Index, util.Itype, estext)
+	}
+	i.ServeJson(map[string]interface{}{"count": count, "msg": msg})
+}
+
+//通过es语句导入
+func (i *Front) ImportByEs() {
+	defer qu.Catch()
+	//success := false
+	msg := ""
+	successNum := int64(0)
+	estext := i.GetString("estext") //es查询语句
+	coll := i.GetString("coll")     //导入表
+	if coll == "" {
+		i.ServeJson(map[string]interface{}{"msg": "表名错误"})
+		return
+	}
+	client := util.Es.GetEsConn()
+	defer util.Es.DestoryEsConn(client)
+	ch := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
+	lock := &sync.Mutex{}
+	escount := util.Es.Count(util.Index, util.Itype, estext)
+	qu.Debug("查询总数:", escount)
+	if escount > 0 {
+		//查询条件类型转换
+		var q es.Query
+		tmpQuery := es.BoolQuery{
+			QueryStrings: estext,
+		}
+		q = tmpQuery
+		//游标查询，index不支持别名，只能写索引库的名称
+		res, err := client.Scroll(util.Index).Query(q).Size(200).Do() //查询一条获取游标
+		ids := []string{}                                             //id数据
+		if err == nil {
+			numDocs := 0
+			scrollId := res.ScrollId
+			for {
+				if scrollId == "" {
+					qu.Debug("ScrollId Is Error")
+					break
+				}
+				searchResult, err := client.Scroll(util.Index).Size(200).ScrollId(scrollId).Do() //查询
+				if err != nil {
+					if err.Error() == "EOS" { //迭代完毕
+						qu.Debug("Es Search Data Over：", err)
+					} else {
+						qu.Debug("Es Search Data Error：", err)
+					}
+					break
+				}
+				for _, hit := range searchResult.Hits.Hits {
+					//开始处理数据
+					wg.Add(1)
+					ch <- true
+					go func(tmpHit *es.SearchHit) {
+						defer func() {
+							<-ch
+							wg.Done()
+						}()
+						tmp := make(map[string]interface{})
+						if json.Unmarshal(*tmpHit.Source, &tmp) == nil {
+							id := qu.ObjToString(tmp["_id"])
+							tmp["id"] = id //记录数据原有id
+							lock.Lock()
+							ids = append(ids, id)
+							lock.Unlock()
+						}
+					}(hit)
+					numDocs += 1
+					if numDocs%500 == 0 {
+						qu.Debug("Current：", numDocs)
+					}
+				}
+				scrollId = searchResult.ScrollId
+			}
+			wg.Wait()
+			client.ClearScroll().ScrollId(scrollId).Do() //清理游标
+			//qu.Debug("Result Data Count：", numDocs)
+		} else {
+			qu.Debug("Es Search Data Error")
+		}
+		//判断数量
+		if int64(len(ids)) != escount {
+			msg = "查询数据和结果不一致"
+		} else { //入库
+			_, msg, successNum = GetDataById(coll, ids, "es", map[string]map[string]interface{}{})
+		}
+	} else {
+		msg = "无查询数据"
+	}
+	msg = fmt.Sprintf("共查询%d条，导入成功%d条\n", escount, successNum) + msg
+	i.ServeJson(map[string]interface{}{"msg": msg})
+}
+
+//同步数据
+func (i *Front) SyncMarked() {
+	syncColl := i.GetString("coll")
+	sess := util.MgoM.GetMgoConn()
+	defer util.MgoM.DestoryMongoConn(sess)
+	it := sess.DB(util.Config.Dbname).C(syncColl).Find(nil).Iter()
+	count, _ := sess.DB(util.Config.Dbname).C(syncColl).Find(nil).Count()
+	qu.Debug(syncColl, count)
+	n := 0
+	lock := &sync.Mutex{}
+	wg := &sync.WaitGroup{}
+	ch := make(chan bool, 3)
+	result := map[string]map[string]interface{}{}
+	idArr := []string{}
+	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
+		wg.Add(1)
+		ch <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			id := qu.ObjToString(tmp["id"])
+			if id == "" {
+				return
+			}
+			tmpMap := map[string]interface{}{}
+			for _, f := range util.Config.CustomerField {
+				if val := tmp[f]; val != nil {
+					tmpMap[f] = val
+				}
+			}
+			lock.Lock()
+			idArr = append(idArr, id)
+			result[id] = tmpMap
+			lock.Unlock()
+		}(tmp)
+		tmp = map[string]interface{}{}
+	}
+	wg.Wait()
+	if count != int64(len(result)) {
+		i.ServeJson(map[string]interface{}{"msg": "同步失败", "flag": false})
+		return
+	}
+	//util.MgoM.C.Database(util.Config.Dbname).Collection(syncColl).Drop(util.MgoM.Ctx) //删除syncColl表(清空数据)
+	success, msg, successNum := GetDataById(syncColl, idArr, "syncoll", result)
+	result = map[string]map[string]interface{}{}
+	idArr = []string{}
+	msg = fmt.Sprintf("共查询%d条，同步成功%d条\n", count, successNum) + msg
+	i.ServeJson(map[string]interface{}{"msg": msg, "flag": success})
 }
+
+//将marked表中ck_data：2更新为0
+func (i *Front) MarkedInit() {
+	set := map[string]interface{}{
+		"$set": map[string]interface{}{
+			"ck_data": 0,
+		},
+	}
+	b := util.MgoM.Update(util.Config.Fromtable, `{"ck_data":2}`, set, false, true)
+	i.ServeJson(map[string]interface{}{"success": b})
+}
+
+//标注完成
+func (i *Front) FinishCheck() {
+	i.Render("finish.html")
+}
+
+//错误页面
+func (i *Front) ErrCheck() {
+	i.Render("err.html")
+}
+
+//统计抽查
+func (i *Front) Tj() error {
+	comm := map[string]map[string]int{}
+	comm_win := map[string]map[string]int{}
+	pack := map[string]map[string]int{}
+	pack_win := map[string]map[string]int{}
+	list, _ := util.MgoM.Find(util.Config.Totablel, "{}", nil, nil, false, -1, -1)
+	for _, tmp := range *list {
+		for k, val := range tmp {
+			if len(k) > 3 && k[:3] == preKey {
+				comm = mapIntAdd(k, qu.ObjToString(val), comm)
+			}
+		}
+		if winnerorder, ok := tmp["winnerorder"].([]interface{}); ok {
+			for _, wd := range winnerorder {
+				if winner, ok := wd.(map[string]interface{}); ok {
+					for k, val := range winner {
+						if len(k) > 3 && k[:3] == preKey {
+							comm_win = mapIntAdd(k, qu.ObjToString(val), comm_win)
+						}
+					}
+				}
+			}
+		}
+		if ptmp, ok := tmp["package"].(map[string]interface{}); ok {
+			for _, pktmp := range ptmp {
+				if pkage, ok := pktmp.(map[string]interface{}); ok {
+					for k, val := range pkage {
+						if len(k) > 3 && k[:3] == preKey {
+							pack = mapIntAdd(k, qu.ObjToString(val), pack)
+						}
+						if k == "winnerorder" {
+							if wtmp, ok := val.([]interface{}); ok {
+								for _, winner := range wtmp {
+									if win, ok := winner.(map[string]interface{}); ok {
+										for wk, wval := range win {
+											if len(k) > 3 && k[:3] == preKey {
+												pack_win = mapIntAdd(wk, qu.ObjToString(wval), pack_win)
+											}
+										}
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	//data := map[string]interface{}{"total": len(list), "comm": comm, "pack": pack, "pack_win": pack_win}
+	//mongodb.Save("extcheck_tj", data)
+	i.T["comm"] = comm
+	i.T["pack"] = pack
+	i.T["comm_win"] = comm_win
+	i.T["pack_win"] = pack_win
+	i.T["total"] = len(*list)
+	return i.Render("tj.html", &i.T)
+}
+
+//标错列表
+func (i *Front) Elist() error {
+	attrname := i.GetString("attrname")
+	common := util.Config.Biaozhu["common"]
+	elist, _ := util.MgoM.Find(util.Config.Totablel, `{"ck_`+attrname+`":"0"}`, `{"_id":1}`, `{"_id":1}`, false, -1, -1)
+	for _, v := range *elist {
+		v["_id"] = mgo.BsonIdToSId(v["_id"])
+	}
+	i.T["elist"] = *elist
+	i.T["attrname"] = attrname
+	i.T["common"] = common
+	return i.Render("elist.html", &i.T)
+}
+
+func (i *Front) Review() error {
+	defer qu.Catch()
+	var labeler []map[string]interface{}
+	sess := util.MgoM.GetMgoConn()
+	defer util.MgoM.DestoryMongoConn(sess)
+	sess.DB(util.MgoM.DbName).C(util.Config.Fromtable).Pipe([]map[string]interface{}{
+		//查询条件
+		{
+			"$match": bson.M{
+				"ck_data": bson.M{"$gte": 1},
+			},
+		},
+		// 按照modifyuser分组，并统计
+		{
+			"$group": bson.M{
+				"_id": "$modifyuser",
+				"count": bson.M{"$sum": 1},
+			},
+		},
+		// 排序
+		{
+			"$sort": bson.M{"count": -1},
+		},
+	}).All(&labeler)
+
+	qu.Debug(labeler)
+	i.T["users"] = labeler
+	return i.Render("review.html", &i.T)
+}
+
+func (i *Front) ReviewList() {
+	defer qu.Catch()
+	if i.Method() == "POST" {
+		user := i.GetString("user")
+		q := make(map[string]interface{})
+		q["ck_data"] = bson.M{"$gte": 1}
+		if user != "0" && user != "-1"  {
+			q["modifyuser"] = user
+		}
+		qu.Debug(q, util.Config.Fromtable)
+		datas, b := util.MgoM.Find(util.Config.Fromtable, q, nil, nil, false, -1, -1)
+		qu.Debug(*datas)
+		if b && len(*datas) > 0 {
+			i.ServeJson(map[string]interface{}{
+				"rep":  b,
+				"data": *datas,
+			})
+		} else {
+			i.ServeJson(map[string]interface{}{
+				"rep": false,
+				"msg": "未查询到数据",
+			})
+		}
+	}
+}
+
+
+func (i *Front) ReviewDetail(id string) error {
+	defer qu.Catch()
+	coll, _ := i.GetSession("coll").(string)
+	query, _ := i.GetSession("query").(map[string]interface{})
+	qu.Debug(coll, query)
+	rep := getDetail(id, coll)          //获取本条公告的信息
+	i.T["otherInfo"] = rep["otherInfo"] //展示关联公告信息
+	i.T["moreInfo"] = rep["moreInfo"]   //更多关联公告信息
+	i.T["info"] = rep["info"]
+	i.T["common"] = rep["common"]
+	i.T["worder"] = rep["worder"]
+	i.T["packs"] = rep["packs"]
+	i.T["packskey"] = rep["packskey"]
+	i.T["ck_pclisext"] = rep["ck_pclisext"]
+	i.T["ck_wodrisext"] = rep["ck_wodrisext"]
+	i.T["ck_pkgisext"] = rep["ck_pkgisext"]
+	i.T["timeplace"] = rep["timeplace"]
+	i.T["purchasinglist"] = rep["purchasinglist"]
+	i.T["other"] = rep["other"]
+	i.T["PurchasinglistField"] = util.PurchasinglistField
+	i.T["WinnerorderField"] = util.WinnerorderField
+	i.T["PackageField"] = util.PackageField
+	i.T["topsubtype"] = util.TopSubStypeArr2
+	i.T[preKey+"purchasinglist"] = rep[preKey+"purchasinglist"]
+	i.T[preKey+"package"] = rep[preKey+"package"]
+	i.T[preKey+"winnerorder"] = rep[preKey+"winnerorder"]
+	i.T["worder_new"] = rep["worder_new"]
+	i.T["pcl_new"] = rep["pcl_new"]
+	i.T["pkg_new"] = rep["pkg_new"]
+	i.T["nextid"] = GetNextDataId(id, coll, query)              //下一条id
+	return i.Render("re_detail.html", &i.T)
+}
+
+
+func (i *Front) ReviewSave() {
+	defer qu.Catch()
+	//_id := i.GetString("_id")
+
+}
+
+func (i *Front) ReviewStat() {
+	defer qu.Catch()
+	sess := util.MgoM.GetMgoConn()
+	defer util.MgoM.DestoryMongoConn(sess)
+	result := sess.DB(util.MgoM.DbName).C(util.Config.Fromtable).Find(nil).Iter()
+	count, cmark, rmark, rgmark := 0, 0, 0, 0		// 总数， 标注数量，	审核数据，	审核数据完全正确的数据量
+
+	cmaps := make(map[string]int)			// 标注字段整体准确率
+	umaps := make(map[string]interface{})	// 按人员 字段准确率
+	//for _, i2 := range util.Config.Field {
+	//	cmaps[i2] = 0
+	//}
+	for tmp := make(map[string]interface{}); result.Next(&tmp); {
+		count ++
+		if qu.IntAll(tmp["ck_data"]) >= 1 {
+			cmark ++
+		}
+		if qu.IntAll(tmp["review"]) > 0 {
+			rmark ++
+			if reField, ok := tmp["re_field"].(map[string]interface{}); ok {
+				if user, o1 := tmp["modifyuser"].(string); o1 {
+
+					// 每个人员标注的质检数量
+					if umaps["count"] != nil {
+						umaps["count"] = qu.IntAll(umaps["count"]) + 1
+					}else {
+						umaps["count"] = 1
+					}
+					// 按人员统计每个字段
+					var ump map[string]int
+					if umaps[user] != nil {
+						ump, _ = umaps[user].(map[string]int)
+					}else {
+						ump = make(map[string]int)
+					}
+
+					flag := true			// 数据整体准确率
+					for k, v := range reField{
+						if v == 1 {
+							cmaps[k] ++
+							ump[k] ++
+						}else {
+							flag = false
+						}
+					}
+					if flag {
+						rgmark ++
+					}
+				}
+			}
+		}
+	}
+}

src/main.go

@@ -49,7 +49,7 @@ func init() {
 }
 
 func main() {
-	var SE = qu.SimpleEncrypt{Key: "topJYBX2019"}
-	qu.Debug(SE.EncodeString("6138fff01a75b8f4466dd76f"))
+	// var SE = qu.SimpleEncrypt{Key: "topJYBX2019"}
+	// qu.Debug(SE.EncodeString("6138fff01a75b8f4466dd76f"))
 	xweb.Run(":" + util.Config.Port)
 }