no message

bidding_data/config.json

@@ -0,0 +1,17 @@
+{
+  "mgodb": "192.168.3.207:27092",
+  "dbsize": 12,
+  "dbname": "wjh",
+  "dbcoll": "oprd_bidding",
+  "qyxyColl": "oprd_qyxy",
+  "uname": "SJZY_RWMIX_Other",
+  "upwd": "SJZY@M34I6x7D9ata",
+  "tagFile": "/Users/jianghan/Desktop/20210123标签规则.xlsx",
+  "operators": "移动,中移在线,中移系统,中移,铁通,联通,联合网络通信,电信,天翼视讯,天翼电信,广电网络,中国广电,湖北广电",
+  "checkDb": {
+    "addr": "192.168.3.166:27082",
+    "dbname": "guangdongyidong",
+    "dbsize": 5,
+    "dbcoll": "20220127Gdyd_5_13"
+  }
+}

bidding_data/main.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"mongodb"
+	"qfw/util"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	Sysconfig          map[string]interface{}
+	Mgo, Mgo1          *mongodb.MongodbSim
+	Dbname, Dbcoll     string
+	collSave, qyxyColl string
+
+	savePool chan map[string]interface{}
+	saveSp   chan bool
+
+	filePath string
+	tagArr   []string
+
+	operators    []string
+	TagMatchRule = map[string][]TagMatching{}
+)
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	collSave = util.ObjToString(Sysconfig["dbcoll"])
+	qyxyColl = util.ObjToString(Sysconfig["qyxyColl"])
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: Sysconfig["mgodb"].(string),
+		Size:        util.IntAllDef(Sysconfig["dbsize"], 5),
+		DbName:      Sysconfig["dbname"].(string),
+		//UserName: 	 Sysconfig["uname"].(string),
+		//Password: 	 Sysconfig["upwd"].(string),
+	}
+	Mgo.InitPool()
+	checkDb, _ := Sysconfig["checkDb"].(map[string]interface{})
+	Dbname = checkDb["dbname"].(string)
+	Dbcoll = checkDb["dbcoll"].(string)
+	Mgo1 = &mongodb.MongodbSim{
+		MongodbAddr: checkDb["addr"].(string),
+		Size:        util.IntAll(checkDb["dbsize"]),
+		DbName:      checkDb["dbname"].(string),
+	}
+	Mgo1.InitPool()
+
+	filePath = util.ObjToString(Sysconfig["tagFile"])
+	savePool = make(chan map[string]interface{}, 5000)
+	saveSp = make(chan bool, 5)
+
+	operators = strings.Split(util.ObjToString(Sysconfig["operators"]), ",")
+	initExcel(filePath)
+	util.Debug(len(TagMatchRule))
+}
+
+func main() {
+	go saveMethod()
+
+	sess := Mgo1.GetMgoConn()
+	defer Mgo1.DestoryMongoConn(sess)
+
+	ch := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+
+	//q := map[string]interface{}{"id": "61bc59f606a9d911e5ba6119"}
+	query := sess.DB(Dbname).C(Dbcoll).Find(nil).Select(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(&tmp); count++ {
+		if count%500 == 0 {
+			util.Debug("current ---", count)
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			taskinfo(tmp)
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	util.Debug("over ---", count)
+
+	c := make(chan bool, 1)
+	<-c
+}
+
+func saveMethod() {
+	arru := make([]map[string]interface{}, 200)
+	indexu := 0
+	for {
+		select {
+		case v := <-savePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					Mgo.SaveBulk(collSave, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, 200)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					Mgo.SaveBulk(collSave, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, 200)
+				indexu = 0
+			}
+		}
+	}
+}

bidding_data/pareExcel.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"github.com/tealeg/xlsx"
+	"regexp"
+	"strings"
+)
+
+type TagMatching struct {
+	matchKey      string
+	matchKeyWay   []string // title、detail、projectname、filetext
+	matchKeyReg   []*RegexpInfo
+	addKey        string
+	addKeyWay     []string
+	addKeyReg     []*RegexpInfo
+	excludeKey    string   //排除词
+	excludeKeyWay []string //排除词匹配方式
+	excludeKeyReg []*RegexpInfo
+}
+
+type RegexpInfo struct {
+	keyStr string
+	regs   *regexp.Regexp
+}
+
+var MatchWayMap = map[string]string{
+	"标题匹配":   "title",
+	"全文匹配":   "detail",
+	"附件匹配":   "filetext",
+	"项目名称匹配": "projectname",
+}
+
+func initExcel(path string) {
+	xlFile, err := xlsx.OpenFile(path)
+	if err != nil {
+		return
+	}
+
+	for _, sheet := range xlFile.Sheet {
+		tagArr = append(tagArr, sheet.Name)
+		var rules []TagMatching
+		for rowIndex, row := range sheet.Rows {
+			if rowIndex == 0 {
+				continue
+			}
+			tag := TagMatching{}
+			for cellIndex, cell := range row.Cells {
+				if cell.String() != "" {
+					if cellIndex == 0 {
+						tag.matchKey = cell.String()
+						tag.matchKeyReg = initRegex(cell.String())
+					} else if cellIndex == 1 {
+						tag.matchKeyWay = strings.Split(cell.String(), ",")
+					} else if cellIndex == 2 {
+						tag.addKey = cell.String()
+						tag.addKeyReg = initRegex(cell.String())
+					} else if cellIndex == 3 {
+						tag.addKeyWay = strings.Split(cell.String(), ",")
+					} else if cellIndex == 4 {
+						tag.excludeKey = cell.String()
+						tag.excludeKeyReg = initRegex(cell.String())
+					} else if cellIndex == 5 {
+						tag.excludeKeyWay = strings.Split(cell.String(), ",")
+					}
+				}
+			}
+			rules = append(rules, tag)
+		}
+		TagMatchRule[sheet.Name] = rules
+	}
+}
+
+func initRegex(key string) []*RegexpInfo {
+	var infos []*RegexpInfo
+	if strings.Contains(key, ",") {
+		for _, s := range strings.Split(key, ",") {
+			if strings.Contains(s, "&&") {
+				info := &RegexpInfo{
+					keyStr: s,
+					regs:   nil,
+				}
+				infos = append(infos, info)
+			} else {
+				info := &RegexpInfo{
+					keyStr: s,
+					regs:   regexp.MustCompile(".*(?i)" + s + ".*"),
+				}
+				infos = append(infos, info)
+			}
+
+		}
+	} else {
+		info := &RegexpInfo{
+			keyStr: key,
+			regs:   regexp.MustCompile(".*(?i)" + key + ".*"),
+		}
+		infos = append(infos, info)
+	}
+	return infos
+}

bidding_data/task.go

@@ -0,0 +1,202 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	"mongodb"
+	"qfw/util"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+var numReg = regexp.MustCompile("[0-9]+")
+
+func taskinfo(info map[string]interface{}) {
+	id := mongodb.BsonIdToSId(info["_id"])
+	//info, ok := tmp["v_baseinfo"].(map[string]interface{})
+	//if !ok {
+	//	util.Debug("data err ---", id)
+	//	return
+	//}
+	// 数据标签
+	//var mtkey []string
+	var tag []string
+	for key, v := range TagMatchRule {
+	M:
+		for _, matching := range v {
+			if matching.excludeKey != "" {
+				// 匹配排除词
+				for _, way := range matching.excludeKeyWay {
+					field := MatchWayMap[way]
+					if val := util.ObjToString(info[field]); val != "" {
+						for _, e1 := range matching.excludeKeyReg {
+							if e1.regs.MatchString(val) {
+								break M
+							}
+						}
+					}
+				}
+			}
+
+			if matching.matchKey != "" {
+				// 匹配关键词
+				for _, way := range matching.matchKeyWay {
+					field := MatchWayMap[way]
+					if val := util.ObjToString(info[field]); val != "" {
+						for _, r1 := range matching.matchKeyReg {
+							if r1.regs.MatchString(val) {
+								if matching.addKey != "" {
+									// 匹配附加词
+									for _, w1 := range matching.addKeyWay {
+										field = MatchWayMap[w1]
+										if v1 := util.ObjToString(info[field]); v1 != "" {
+											for _, r2 := range matching.addKeyReg {
+												if r2.regs != nil && r2.regs.MatchString(v1) {
+													//mtkey = append(mtkey, matching.matchKey)
+													tag = append(tag, key) // 标签
+													break M
+												} else {
+													// && 特殊处理
+													if strings.Contains(r2.keyStr, "&&") {
+														flag := true
+														for _, s := range strings.Split(r2.keyStr, "&&") {
+															if !strings.Contains(v1, s) {
+																flag = false
+																break
+															}
+														}
+														if flag {
+															//mtkey = append(mtkey, matching.matchKey)
+															tag = append(tag, key) // 标签
+															break M
+														}
+													}
+												}
+											}
+										}
+									}
+								} else {
+									//mtkey = append(mtkey, matching.matchKey)
+									tag = append(tag, key) // 标签
+									break M
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	//info["mtkey"] = strings.Join(mtkey, "-")
+	if len(tag) > 0 {
+		info["tag_rule"] = strings.Join(tag, ",")
+	} else {
+		info["tag_rule"] = "IT"
+	}
+	/**
+	项目结束日期 project_completedate(中标、成交、合同)
+	1、优先采用project_completedate
+	2、通过项目周期(projectperiod)、工期时长(project_duration)、工期单位(project_timeunit)、项目开始时间(project_startdate/signaturedate)(发布时间/合同签订时间)
+	*/
+	completeData := util.Int64All(info["project_completedate"])
+	subtype := util.ObjToString(info["subtype"])
+	if completeData == 0 {
+		if subtype == "中标" || subtype == "成交" || subtype == "合同" {
+			var dur int
+			if util.IntAll(info["project_duration"]) != 0 {
+				dur = util.IntAll(info["project_duration"])
+			} else {
+				period := periodMethod(info["projectperiod"])
+				if period > 0 {
+					dur = period
+				}
+			}
+			if dur > 0 {
+				timeunit := util.ObjToString(info["project_timeunit"])
+				startdate := util.Int64All(info["project_startdate"])
+				if startdate > 0 {
+					completeData = YcEndTime(startdate, dur, timeunit)
+				} else if startdate = util.Int64All(info["signaturedate"]); startdate > 0 {
+					completeData = YcEndTime(startdate, dur, timeunit)
+				} else {
+					startdate = util.Int64All(info["publishtime"])
+					completeData = YcEndTime(startdate, dur, timeunit)
+				}
+			}
+
+			if completeData != 0 {
+				info["project_completedate"] = completeData
+			}
+		}
+	}
+	// 企业标签（采购单位 1 /中标单位 2 ）
+	if info["buyer"] != nil {
+		qyxy, _ := Mgo.FindOne(qyxyColl, map[string]interface{}{"company_name": util.ObjToString(info["buyer"])})
+		if len(*qyxy) > 0 {
+			if (*qyxy)["bid_type"] == nil {
+				Mgo.UpdateById(qyxyColl, (*qyxy)["_id"], bson.M{"$set": bson.M{"bid_type": "1"}})
+			} else {
+				stype := strings.Split(util.ObjToString((*qyxy)["bid_type"]), ",")
+				if !strings.Contains(util.ObjToString((*qyxy)["bid_type"]), "1") {
+					stype = append(stype, "1")
+					sort.Strings(stype)
+					Mgo.UpdateById(qyxyColl, (*qyxy)["_id"], bson.M{"$set": bson.M{"bid_type": strings.Join(stype, ",")}})
+				}
+			}
+		}
+	}
+	if info["s_winner"] != nil {
+		winner := util.ObjToString(info["s_winner"])
+		for _, s := range strings.Split(winner, ",") {
+			qyxy, _ := Mgo.FindOne(qyxyColl, map[string]interface{}{"company_name": s})
+			if len(*qyxy) > 0 {
+				if (*qyxy)["bid_type"] == nil {
+					Mgo.UpdateById(qyxyColl, (*qyxy)["_id"], bson.M{"$set": bson.M{"bid_type": "2"}})
+				} else {
+					stype := strings.Split(util.ObjToString((*qyxy)["bid_type"]), ",")
+					if !strings.Contains(util.ObjToString((*qyxy)["bid_type"]), "2") {
+						stype = append(stype, "2")
+						Mgo.UpdateById(qyxyColl, (*qyxy)["_id"], bson.M{"$set": bson.M{"bid_type": strings.Join(stype, ",")}})
+					}
+				}
+			}
+		}
+		//for _, op := range operators {
+		//	if strings.Contains(winner, op) {
+		//		info["isOperators"] = true
+		//		break
+		//	}
+		//}
+	}
+	info["_id"] = mongodb.StringTOBsonId(id)
+	info["updatetime"] = time.Now().Unix()
+	savePool <- info
+}
+
+func periodMethod(str interface{}) int {
+	if util.ObjToString(str) != "" {
+		arr := numReg.FindStringSubmatch(util.ObjToString(str))
+		if len(arr) > 0 {
+			return util.IntAll(arr[0])
+		}
+	}
+	return 0
+}
+
+func YcEndTime(starttime int64, num int, unit string) int64 {
+	yuceendtime := int64(0)
+	if unit == "日历天" || unit == "天" || unit == "日" {
+		yuceendtime = starttime + int64(num*86400)
+	} else if unit == "周" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num*7).Unix()
+	} else if unit == "月" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, num, 0).Unix()
+	} else if unit == "年" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(num, 0, 0).Unix()
+	} else if unit == "工作日" {
+		n := num / 7 * 2
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num+n).Unix()
+	}
+	return yuceendtime
+}

es/biddingindex.go

@@ -0,0 +1,306 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	"log"
+	"mongodb"
+	qutil "qfw/util"
+	elastic "qfw/util/elastic"
+	"reflect"
+	"strings"
+	"unicode/utf8"
+
+	//elastic "qfw/util/elastic_v5"
+	"regexp"
+	//	"strings"
+	"sync"
+)
+
+var (
+	BulkSizeBack = 400
+	ESLEN        = 32766
+)
+
+var reg_letter = regexp.MustCompile("[a-z]*")
+
+func biddingTask() {
+	defer qutil.Catch()
+	//bidding库
+	session := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(session)
+
+	db := qutil.ObjToString(bidding["db"])
+	coll := qutil.ObjToString(bidding["collect"])
+	//q := map[string]interface{}{"updatetime": map[string]interface{}{"$gt": 1643262000}}
+	q := map[string]interface{}{"_id": mongodb.StringTOBsonId("5db7c324a5cb26b9b78b0f09")}
+	count, _ := session.DB(db).C(coll).Find(&q).Count()
+	index := qutil.ObjToString(bidding["index"])
+	stype := qutil.ObjToString(bidding["type"])
+	//线程池
+	UpdatesLock := sync.Mutex{}
+	qutil.Debug("查询语句:", nil, "同步总数:", count, "elastic库:")
+	//查询招标数据
+	query := session.DB(db).C(coll).Find(&q).Select(bson.M{
+		"projectinfo.attachment": 0,
+		"contenthtml":            0,
+		"publishdept":            0,
+	}).Sort("_id").Iter()
+	//查询抽取结果
+	n := 0
+	//更新数组
+	arrEs := []map[string]interface{}{}
+	thread := 10
+	espool := make(chan bool, 5)
+	var mpool = make(chan bool, thread)
+	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+		if n%2000 == 0 {
+			log.Println("current:", n, tmp["_id"])
+		}
+		if qutil.ObjToString(tmp["useable"]) == "0" {
+			continue
+		}
+		mpool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-mpool
+			}()
+			subscopeclass, _ := tmp["subscopeclass"].([]interface{}) //subscopeclass
+			if subscopeclass != nil {
+				m1 := map[string]bool{}
+				newclass := []string{}
+				for _, sc := range subscopeclass {
+					sclass, _ := sc.(string)
+					if !m1[sclass] {
+						m1[sclass] = true
+						newclass = append(newclass, sclass)
+					}
+				}
+				tmp["s_subscopeclass"] = strings.Join(newclass, ",")
+				tmp["subscopeclass"] = newclass
+			}
+			topscopeclass, _ := tmp["topscopeclass"].([]interface{}) //topscopeclass
+			if topscopeclass != nil {
+				m2 := map[string]bool{}
+				newclass := []string{}
+				for _, tc := range topscopeclass {
+					tclass, _ := tc.(string)
+					tclass = reg_letter.ReplaceAllString(tclass, "") // 去除字母
+					if !m2[tclass] {
+						m2[tclass] = true
+						newclass = append(newclass, tclass)
+					}
+				}
+				tmp["s_topscopeclass"] = strings.Join(newclass, ",")
+			}
+			//对projectscope字段的索引处理
+			ps, _ := tmp["projectscope"].(string)
+			if len(ps) > ESLEN {
+				tmp["projectscope"] = string(([]rune(ps))[:4000])
+			}
+			//对标的物为空处理
+			filetext := getFileText(tmp)
+			filetextS := filetext
+			if len([]rune(filetextS)) > 10 { //attach_text
+				tmp["filetext"] = filetext
+			}
+			if purchasing, ok := tmp["purchasing"].(string); ok && purchasing == "" {
+				delete(tmp, "purchasing")
+			}
+			if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok && len(purchasinglist) == 0 {
+				delete(tmp, "purchasinglist")
+			}
+			//数据为空处理
+			for _, f := range []string{"bidstatus", "city", "district", "channel"} {
+				if fVal, ok := tmp[f].(string); ok && fVal == "" {
+					delete(tmp, f)
+				}
+			}
+			UpdatesLock.Lock()
+			newTmp := map[string]interface{}{}
+			for field, ftype := range biddingIndexFieldsMap {
+				if tmp[field] != nil { //
+					if field == "projectinfo" {
+						mp, _ := tmp[field].(map[string]interface{})
+						if mp != nil {
+							newmap := map[string]interface{}{}
+							for k, ktype := range projectinfoFieldsMap {
+								mpv := mp[k]
+								if mpv != nil && reflect.TypeOf(mpv).String() == ktype {
+									newmap[k] = mp[k]
+								}
+							}
+							if len(newmap) > 0 {
+								newTmp[field] = newmap
+							}
+						}
+					} else if field == "purchasinglist" { //标的物处理
+						purchasinglist_new := []map[string]interface{}{}
+						if pcl, _ := tmp[field].([]interface{}); len(pcl) > 0 {
+							for _, ls := range pcl {
+								lsm_new := make(map[string]interface{})
+								lsm := ls.(map[string]interface{})
+								for pf, pftype := range purchasinglistFieldsMap {
+									lsmv := lsm[pf]
+									if lsmv != nil && reflect.TypeOf(lsmv).String() == pftype {
+										lsm_new[pf] = lsm[pf]
+									}
+								}
+								if lsm_new != nil && len(lsm_new) > 0 {
+									purchasinglist_new = append(purchasinglist_new, lsm_new)
+								}
+							}
+						}
+						if len(purchasinglist_new) > 0 {
+							newTmp[field] = purchasinglist_new
+						}
+					} else if field == "winnerorder" { //中标候选
+						winnerorder_new := []map[string]interface{}{}
+						if winnerorder, _ := tmp[field].([]interface{}); len(winnerorder) > 0 {
+							for _, win := range winnerorder {
+								winMap_new := make(map[string]interface{})
+								winMap := win.(map[string]interface{})
+								for wf, wftype := range winnerorderlistFieldsMap {
+									wfv := winMap[wf]
+									if wfv != nil && reflect.TypeOf(wfv).String() == wftype {
+										if wf == "sort" && qutil.Int64All(wfv) > 100 {
+											continue
+										}
+										winMap_new[wf] = winMap[wf]
+									}
+								}
+								if winMap_new != nil && len(winMap_new) > 0 {
+									winnerorder_new = append(winnerorder_new, winMap_new)
+								}
+							}
+						}
+						if len(winnerorder_new) > 0 {
+							newTmp[field] = winnerorder_new
+						}
+					} else if field == "qualifies" {
+						//项目资质
+						qs := []string{}
+						if q, _ := tmp[field].([]interface{}); len(q) > 0 {
+							for _, v := range q {
+								v1 := v.(map[string]interface{})
+								qs = append(qs, qutil.ObjToString(v1["key"]))
+							}
+						}
+						if len(qs) > 0 {
+							newTmp[field] = strings.Join(qs, ",")
+						}
+					} else if field == "review_experts" {
+						// 评审专家
+						if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
+							arr1 := qutil.ObjArrToStringArr(arr)
+							newTmp[field] = strings.Join(arr1, ",")
+						}
+					} else if field == "entidlist" {
+						newTmp[field] = tmp[field]
+					} else if field == "bidopentime" {
+						if tmp[field] != nil && tmp["bidendtime"] == nil {
+							newTmp["bidendtime"] = tmp[field]
+						}
+						if tmp[field] == nil && tmp["bidendtime"] != nil {
+							newTmp[field] = tmp["bidendtime"]
+						}
+					} else if field == "detail" { //过滤
+						detail, _ := tmp[field].(string)
+						if len([]rune(detail)) > detailLength {
+							detail = detail[:detailLength]
+						}
+						if strings.Contains(detail, qutil.ObjToString(tmp["title"])) {
+							newTmp[field] = FilterDetail(detail)
+						} else {
+							newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
+						}
+					} else if field == "_id" || field == "topscopeclass" { //不做处理
+						newTmp[field] = tmp[field]
+					} else if field == "publishtime" || field == "comeintime" {
+						//字段类型不正确，特别处理
+						if tmp[field] != nil && qutil.Int64All(tmp[field]) > 0 {
+							newTmp[field] = qutil.Int64All(tmp[field])
+						}
+					} else if field == "s" {
+						newTmp[field] = tmp[field]
+					} else { //其它字段判断数据类型，不正确舍弃
+						if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {
+							continue
+						} else {
+							if fieldval != "" {
+								newTmp[field] = fieldval
+							}
+						}
+					}
+				}
+			}
+			arrEs = append(arrEs, newTmp)
+			if len(arrEs) >= BulkSizeBack {
+				tmps := arrEs
+				espool <- true
+				go func(tmps []map[string]interface{}) {
+					defer func() {
+						<-espool
+					}()
+					elastic.BulkSave(index, stype, &tmps, true)
+				}(tmps)
+				arrEs = []map[string]interface{}{}
+			}
+			UpdatesLock.Unlock()
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	for i := 0; i < thread; i++ {
+		mpool <- true
+	}
+	UpdatesLock.Lock()
+	if len(arrEs) > 0 {
+		tmps := arrEs
+		elastic.BulkSave(index, stype, &tmps, true)
+	}
+	UpdatesLock.Unlock()
+	log.Println("create biddingback index...over", n)
+}
+
+var filterReg = regexp.MustCompile("<[^>]+>")
+var filterSpace = regexp.MustCompile("<[^>]*?>|[\\s\u3000\u2003\u00a0]")
+
+func FilterDetail(text string) string {
+	return filterReg.ReplaceAllString(text, "")
+}
+
+func FilterDetailSpace(text string) string {
+	return filterSpace.ReplaceAllString(text, "")
+}
+
+// 正则判断是否包含
+func checkContains(s, sub string) bool {
+	reg := regexp.MustCompile(`(?i)(^|([\s\t\n]+))(` + sub + `)($|([\s\t\n]+))`)
+	return reg.MatchString(s)
+}
+
+func getFileText(tmp map[string]interface{}) (filetext string) {
+	if attchMap, ok := tmp["attach_text"].(map[string]interface{}); attchMap != nil && ok {
+		for _, tmpData1 := range attchMap {
+			if tmpData2, ok := tmpData1.(map[string]interface{}); tmpData2 != nil && ok {
+				for _, result := range tmpData2 {
+					if resultMap, ok := result.(map[string]interface{}); resultMap != nil && ok {
+						if attach_url := qutil.ObjToString(resultMap["attach_url"]); attach_url != "" {
+							bs := OssGetObject(attach_url) //oss读数据
+							if utf8.RuneCountInString(filetext+bs) < fileLength {
+								filetext += bs + "\n"
+							} else {
+								if utf8.RuneCountInString(bs) > fileLength {
+									filetext = bs[0:fileLength]
+								} else {
+									filetext = bs
+								}
+								break
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	return
+}

es/config.json

@@ -0,0 +1,62 @@
+{
+  "uname": "root",
+  "upwd": "root",
+  "mongodb": {
+    "addr": "192.168.3.207:27092",
+    "pool": 10,
+    "db": "wjh"
+  },
+  "bidding": {
+    "db": "wjh",
+    "collect": "oprd_bidding",
+    "index": "oprd_bidding_v1",
+    "type": "bidding",
+    "extractdb": "wjh",
+    "indexfields": [
+      "buyerzipcode", "winnertel", "winnerperson", "contractcode", "winneraddr", "agencyaddr", "buyeraddr", "signaturedate", "projectperiod", "projectaddr", "agencytel", "agencyperson",
+      "buyerperson", "agency", "projectscope", "projectcode", "bidopentime", "supervisorrate", "buyertel", "bidamount", "winner", "buyer", "budget", "projectname", "bidstatus", "buyerclass",
+      "topscopeclass", "s_topscopeclass", "s_subscopeclass", "area", "city", "district", "s_winner", "_id", "title", "detail", "site", "comeintime", "href", "infoformat", "publishtime",
+      "s_sha", "spidercode", "subtype", "toptype", "projectinfo", "purchasing", "purchasinglist", "filetext", "channel", "winnerorder", "project_scale", "project_duration", "project_timeunit",
+      "project_startdate", "project_completedate", "payway", "contract_guarantee", "bid_guarantee", "qualifies", "funds", "review_experts", "bidmethod", "bidendtime", "bidopenaddress",
+      "docamount", "agencyrate", "agencyfee", "getdocmethod", "tag_rule", "bidway", "entidlist", "buyertagname"
+    ],
+    "indexfieldsmap": {
+      "buyerzipcode": "string", "winnertel": "string", "winnerperson": "string", "contractcode": "string", "winneraddr": "string", "agencyaddr": "string", "buyeraddr": "string", "signaturedate": "int64",
+      "projectperiod": "string", "projectaddr": "string", "agencytel": "string", "agencyperson": "string", "buyerperson": "string", "agency": "string", "projectscope": "string", "projectcode": "string",
+      "bidopentime": "int64", "supervisorrate": "float64", "buyertel": "string", "bidamount": "float64", "winner": "string", "buyer": "string", "budget": "float64", "projectname": "string", "bidstatus": "string",
+      "buyerclass": "string", "topscopeclass": "", "s_topscopeclass": "string", "s_subscopeclass": "string", "area": "string", "city": "string", "district": "string", "s_winner": "string", "_id": "", "title": "string",
+      "detail": "string", "site": "string", "comeintime": "int64", "href": "string", "infoformat": "int32", "publishtime": "int64", "s_sha": "string", "spidercode": "string", "subtype": "string", "toptype": "string",
+      "projectinfo": "", "purchasing": "string", "purchasinglist": "", "filetext": "string", "channel": "string", "winnerorder": "", "project_scale": "string", "project_duration": "int32", "project_timeunit": "string",
+      "project_startdate": "int64", "project_completedate": "int64", "payway": "string", "contract_guarantee": "bool", "bid_guarantee": "bool", "qualifies": "", "funds": "string",
+      "review_experts": "", "bidmethod": "string", "bidendtime": "int64", "bidopenaddress": "string", "docamount": "float64", "agencyrate": "float64", "agencyfee": "float64", "bidway": "string",
+      "getdocmethod": "string", "tag_rule": "string", "entidlist": "[]string", "buyertagname": "string"
+    },
+    "fields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,s_topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass,dataging,winnerorder,project_scale,project_duration,project_timeunit,project_startdate,project_completedate, payway,contract_guarantee,bid_guarantee,qualifies,funds,review_experts,bidmethod,bidendtime,bidopenaddress,docamount,bidway,agencyrate,agencyfee,getdocmethod",
+    "projectinfo": "approvecode,approvecontent,approvestatus,approvetime,approvedept,approvenumber,projecttype,approvecity",
+    "projectinfomap": {
+      "approvecode": "string", "approvecontent": "string", "approvestatus": "string", "approvetime": "string", "approvedept": "string", "approvenumber": "string", "projecttype": "string", "approvecity": "string"
+    },
+    "purchasinglist": "itemname,brandname,model,unitname,number,unitprice,totalprice",
+    "purchasinglistmap": {
+      "itemname": "string", "brandname": "string", "model": "string", "unitname": "string", "number": "float64", "unitprice": "float64", "totalprice": "float64"
+    },
+    "winnerorder": "sort,sortstr,entname",
+    "winnerordermap": {
+      "sort": "int", "sortstr": "string", "entname": "string"
+    },
+    "multiIndex": ""
+  },
+  "filelength": 50000,
+  "detaillength": 50000,
+  "project": {
+    "db": "wjh",
+    "collect": "oprd_projectset",
+    "index": "oprd_projectset_v1",
+    "type": "projectset"
+  },
+  "elastic": {
+    "addr": "http://192.168.3.206:9800",
+    "pool": 12,
+    "node": "4q7v7e6mQ5aeCwjUgM6HcA"
+  }
+}

es/main.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"log"
+	"mongodb"
+	_ "net/http/pprof"
+	"qfw/util"
+	elastic "qfw/util/elastic"
+	"strings"
+)
+
+var (
+	Sysconfig                map[string]interface{} //配置文件
+	mgo                      *mongodb.MongodbSim    //mongodb操作对象
+	biddingIndexFields       []string
+	biddingIndexFieldsMap    = map[string]string{}
+	projectinfoFields        []string
+	projectinfoFieldsMap     = map[string]string{}
+	multiIndex               []string
+	purchasinglistFields     []string
+	winnerorderlistFields    []string
+	purchasinglistFieldsMap  = map[string]string{}
+	winnerorderlistFieldsMap = map[string]string{}
+	BulkSize                 = 400
+	detailLength             = 50000
+	fileLength               = 50000
+	savesize                 = 500
+
+	bidding, project map[string]interface{}
+)
+var UpdataMgoCache = make(chan []map[string]interface{}, 1000)
+var SP = make(chan bool, 5)
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	detailLength = util.IntAllDef(Sysconfig["detaillength"], 50000)
+	fileLength = util.IntAllDef(Sysconfig["filelength"], 50000)
+	bidding, _ = Sysconfig["bidding"].(map[string]interface{})
+	project, _ = Sysconfig["project"].(map[string]interface{})
+	mconf, _ := Sysconfig["mongodb"].(map[string]interface{})
+
+	mgo = &mongodb.MongodbSim{ //mongodb为binding连接
+		MongodbAddr: mconf["addr"].(string),
+		Size:        util.IntAllDef(mconf["pool"], 5),
+		DbName:      mconf["db"].(string),
+		//UserName:	 Sysconfig["uname"].(string),
+		//Password:    Sysconfig["upwd"].(string),
+	}
+	mgo.InitPool()
+
+	//初始化es
+	econf := Sysconfig["elastic"].(map[string]interface{})
+	elastic.InitElasticSize(econf["addr"].(string), util.IntAllDef(econf["pool"], 5))
+
+	//
+	if bidding["indexfields"] != nil {
+		biddingIndexFields = util.ObjArrToStringArr(bidding["indexfields"].([]interface{}))
+	}
+	if bidding["projectinfo"] != nil {
+		pf := util.ObjToString(bidding["projectinfo"])
+		if pf != "" {
+			projectinfoFields = strings.Split(pf, ",")
+		}
+	}
+	if bidding["purchasinglist"] != nil {
+		pcl := util.ObjToString(bidding["purchasinglist"])
+		if pcl != "" {
+			purchasinglistFields = strings.Split(pcl, ",")
+		}
+	}
+	if bidding["winnerorder"] != nil {
+		winnerorder := util.ObjToString(bidding["winnerorder"])
+		if winnerorder != "" {
+			winnerorderlistFields = strings.Split(winnerorder, ",")
+		}
+	}
+	//
+	if bidding["indexfieldsmap"] != nil {
+		for k, v := range bidding["indexfieldsmap"].(map[string]interface{}) {
+			biddingIndexFieldsMap[k] = util.ObjToString(v)
+		}
+		log.Println(biddingIndexFieldsMap)
+	}
+	if bidding["projectinfomap"] != nil {
+		for k, v := range bidding["projectinfomap"].(map[string]interface{}) {
+			projectinfoFieldsMap[k] = util.ObjToString(v)
+		}
+		log.Println(projectinfoFieldsMap)
+	}
+	if bidding["purchasinglistmap"] != nil {
+		for k, v := range bidding["purchasinglistmap"].(map[string]interface{}) {
+			purchasinglistFieldsMap[k] = util.ObjToString(v)
+		}
+		log.Println(purchasinglistFieldsMap)
+	}
+	if bidding["winnerordermap"] != nil {
+		for k, v := range bidding["winnerordermap"].(map[string]interface{}) {
+			winnerorderlistFieldsMap[k] = util.ObjToString(v)
+		}
+		log.Println(winnerorderlistFieldsMap)
+	}
+	log.Println(projectinfoFields)
+	log.Println(purchasinglistFields)
+
+	//初始化oss
+	InitOss()
+
+}
+
+func main() {
+
+	//biddingTask()
+	projectTask()
+	ch := make(chan bool, 1)
+	<-ch
+}

es/ossclient.go

@@ -0,0 +1,52 @@
+// ossclient
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"qfw/util"
+
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+)
+
+var (
+	ossEndpoint        = "https://oss-cn-beijing.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
+	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
+	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
+	ossBucketName      = "topjy"
+	ossclient          *oss.Client
+)
+
+func InitOss() {
+	client, err := oss.New(ossEndpoint, ossAccessKeyId, ossAccessKeySecret)
+	if err != nil {
+		fmt.Println("Error:", err)
+		os.Exit(-1)
+	}
+	ossclient = client
+}
+
+func OssGetObject(objectName string) string {
+	util.Catch()
+	// 获取存储空间。
+	bucket, err := ossclient.Bucket(ossBucketName)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+
+	// 下载文件到流。
+	body, err := bucket.GetObject(objectName)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+	defer body.Close()
+	data, err := ioutil.ReadAll(body)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+	return string(data)
+}

es/projectindex.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"log"
+	"math"
+	"qfw/util"
+	"qfw/util/elastic"
+	"strconv"
+)
+
+func projectTask() {
+	defer util.Catch()
+	session := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(session)
+	c, _ := project["collect"].(string)
+	db, _ := project["db"].(string)
+	index, _ := project["index"].(string)
+	itype, _ := project["type"].(string)
+	count, _ := session.DB(db).C(c).Find(nil).Count()
+	savepool := make(chan bool, 5)
+	log.Println(db, c, "查询语句:", nil, "同步总数:", count, "elastic库:", index)
+	query := session.DB(db).C(c).Find(nil).Iter()
+	arr := make([]map[string]interface{}, savesize)
+	var n int
+	i := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); i++ {
+		if n%5000 == 0 {
+			log.Println("current---------", n)
+		}
+		pp := map[string]map[string]interface{}{}
+		if packages, ok := tmp["package"].(map[string]interface{}); ok {
+			for _, pks := range packages {
+				if pk, ok := pks.([]interface{}); ok {
+					for _, v := range pk {
+						if p, ok := v.(map[string]interface{}); ok {
+							winner := util.ObjToString(p["winner"])
+							bidamount := util.Float64All((p["bidamount"]))
+							if len(winner) > 4 && bidamount > 0 {
+								p := map[string]interface{}{
+									"winner":    winner,
+									"bidamount": bidamount,
+								}
+								pp[winner] = p
+							}
+						}
+					}
+				}
+			}
+		} else {
+			winner := util.ObjToString(tmp["winner"])
+			bidamount := util.Float64All(tmp["bidamount"])
+			if len(winner) > 4 && bidamount > 0 {
+				p := map[string]interface{}{
+					"winner":    winner,
+					"bidamount": bidamount,
+				}
+				pp[winner] = p
+			}
+		}
+
+		pk1 := []map[string]interface{}{}
+		for _, v := range pp {
+			pk1 = append(pk1, v)
+		}
+		if len(pk1) > 0 {
+			tmp["package1"] = pk1
+		}
+		budget := util.Float64All(tmp["budget"])
+		bidamount := util.Float64All(tmp["bidamount"])
+		if float64(budget) > 0 && float64(bidamount) > 0 {
+			rate := float64(1) - float64(bidamount)/float64(budget)
+			f, _ := strconv.ParseFloat(strconv.FormatFloat(rate, 'f', 4, 64), 64)
+			//不在0~0.6之间，不生成费率；只生成预算，中标金额舍弃，索引增加折扣率异常标识
+			if f < 0 || f > 0.6 {
+				delete(tmp, "bidamount")
+				tmp["prate_flag"] = 1
+			} else {
+				tmp["project_rate"] = f
+			}
+		}
+		if topscopeclass, ok := tmp["topscopeclass"].([]interface{}); ok {
+			tc := []string{}
+			m2 := map[string]bool{}
+			for _, v := range topscopeclass {
+				str := util.ObjToString(v)
+				str = reg_letter.ReplaceAllString(str, "") // 去除字母
+				if !m2[str] {
+					m2[str] = true
+					tc = append(tc, str)
+				}
+			}
+			tmp["topscopeclass"] = tc
+		}
+		//不生索引字段
+		delete(tmp, "package")
+		delete(tmp, "infofield")
+
+		bidopentime := util.Int64All(tmp["bidopentime"]) //开标日期
+		fzb_publishtime := int64(0)                      //记录第一个招标信息的publishtime
+		bidcycle_flag := false                           //判断是否已计算出标书表编制周期
+		list := tmp["list"].([]interface{})
+		for _, m := range list {
+			tmpM := m.(map[string]interface{})
+			//删除purchasing,review_experts
+			delete(tmpM, "purchasing")
+			delete(tmpM, "review_experts")
+			if bidamount, ok := tmpM["bidamount"].(string); ok && len(bidamount) > 0 { //bidamount为string类型，转成float
+				tmpB := util.Float64All(tmpM["bidamount"])
+				tmpM["bidamount"] = tmpB
+			}
+			//projectscope截断
+			listProjectscopeRune := []rune(util.ObjToString(tmpM["projectscope"]))
+			if len(listProjectscopeRune) > 1000 {
+				tmpM["projectscope"] = string(listProjectscopeRune[:1000])
+			}
+			//计算bidcycle标书表编制周期字段
+			if !bidcycle_flag && bidopentime > 0 { //bidopentime>0证明list中有bidopentime，无则不用计算bidcycle
+				if toptype := util.ObjToString(tmpM["toptype"]); toptype == "招标" {
+					zb_bidopentime := util.Int64All(tmpM["bidopentime"])
+					zb_publishtime := util.Int64All(tmpM["publishtime"])
+					if zb_publishtime > 0 {
+						if zb_bidopentime > 0 {
+							if tmpTime := zb_bidopentime - zb_publishtime; tmpTime > 0 {
+								f_day := float64(tmpTime) / float64(86400)
+								day := math.Ceil(f_day)
+								tmp["bidcycle"] = int(day)
+								bidcycle_flag = true
+							}
+						}
+						if fzb_publishtime == 0 { //仅赋值第一个招标信息的publishtime
+							fzb_publishtime = zb_publishtime
+						}
+					}
+				}
+			}
+		}
+		//计算bidcycle标书表编制周期字段
+		//list中招标信息中未能计算出bidcycle,用第一个招标信息的fzb_publishtime和外围bidopentime计算
+		if !bidcycle_flag && bidopentime > 0 && fzb_publishtime > 0 {
+			if tmpTime := bidopentime - fzb_publishtime; tmpTime > 0 {
+				f_day := float64(tmpTime) / float64(86400)
+				day := math.Ceil(f_day)
+				tmp["bidcycle"] = int(day)
+			}
+		}
+		//projectscope截断
+		projectscopeRune := []rune(util.ObjToString(tmp["projectscope"]))
+		if len(projectscopeRune) > 1000 {
+			tmp["projectscope"] = string(projectscopeRune[:1000])
+		}
+		tmp["s_projectname"] = tmp["projectname"]
+		arr[i] = tmp
+		n++
+		if i == savesize-1 {
+			savepool <- true
+			tmps := arr
+			go func(tmpn *[]map[string]interface{}) {
+				defer func() {
+					<-savepool
+				}()
+				elastic.BulkSave(index, itype, tmpn, true)
+			}(&tmps)
+			i = 0
+			arr = make([]map[string]interface{}, savesize)
+		}
+		if n%savesize == 1000 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+
+	if i > 0 {
+		//util.Debug(arr)
+		elastic.BulkSave(index, itype, &arr, true)
+	}
+	log.Println("create project index...over", n)
+}

export_excel/config.json

@@ -0,0 +1,28 @@
+{
+  "mongodb": "192.168.3.207:27092",
+  "dbname": "wjh",
+  "dbcoll": "oprd_projectset",
+  "dbsize": 5,
+  "fields": {
+    "buyer": "采购单位名称",
+    "s_subscopeclass": "合同分类",
+    "area": "省份",
+    "city": "城市",
+    "title": "公告标题",
+    "bidstatus": "项目状态",
+    "jyhref1": "项目状态剑鱼链接",
+    "bidtype": "招标类型",
+    "jyhref2": "招标公告剑鱼链接",
+    "tag_rule": "项目分类（ICT/综合布线/其他）",
+    "publishtime": "发布时间",
+    "project_duration": "工期时长",
+    "project_timeunit": "工期单位",
+    "projectname": "项目名称",
+    "budget": "预算金额（万元）",
+    "bidamount": "中标金额（万元）",
+    "agency": "招标代理机构",
+    "s_winner": "中标单位名称",
+    "winnerperson": "中标单位联系人",
+    "winnertel": "中标单位联系方式"
+  }
+}

export_excel/main.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"fmt"
+	"github.com/tealeg/xlsx"
+	"mongodb"
+	"qfw/util"
+	"sort"
+)
+
+var (
+	Sysconfig      map[string]interface{}
+	Mgo            *mongodb.MongodbSim
+	Dbname, DbColl string
+	Fields         map[string]interface{}
+	FieldsArr      []string
+
+	JYHref = "https://www.jianyu360.cn/article/content/"
+)
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: Sysconfig["mongodb"].(string),
+		Size:        util.IntAllDef(Sysconfig["dbsize"], 5),
+		DbName:      Sysconfig["dbname"].(string),
+	}
+	Mgo.InitPool()
+	Dbname = Sysconfig["dbname"].(string)
+	DbColl = Sysconfig["dbcoll"].(string)
+
+	Fields = Sysconfig["fields"].(map[string]interface{})
+	for k := range Fields {
+		FieldsArr = append(FieldsArr, k)
+	}
+	sort.Strings(FieldsArr)
+
+}
+
+func main() {
+
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	file := xlsx.NewFile()
+	sheet, err := file.AddSheet("sheet1")
+	if err != nil {
+		panic(err)
+	}
+	row := sheet.AddRow()
+	for _, v := range FieldsArr {
+		row.AddCell().SetValue(Fields[v])
+	}
+
+	//q := map[string]interface{}{"_id": mongodb.StringTOBsonId("594fef0e61a0721f156e9bc3")}
+	query := sess.DB(Dbname).C(DbColl).Find(nil).Select(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(&tmp); count++ {
+		if count%500 == 0 {
+			util.Debug("current ---", count)
+		}
+		row := sheet.AddRow()
+		for _, v := range FieldsArr {
+			if v == "jyhref1" {
+				if util.ObjToString(tmp["bidstatus"]) == "中标" || util.ObjToString(tmp["bidstatus"]) == "成交" ||
+					util.ObjToString(tmp["bidstatus"]) == "合同" {
+					list := util.ObjArrToStringArr(tmp["ids"].([]interface{}))
+					id := list[len(list)-1]
+					row.AddCell().SetValue(fmt.Sprintf(JYHref+"%s.html", util.CommonEncodeArticle("content", id)))
+				} else {
+					row.AddCell().SetValue("")
+				}
+			} else if v == "jyhref2" {
+				if util.ObjToString(tmp["bidtype"]) == "招标" {
+					list := util.ObjArrToStringArr(tmp["ids"].([]interface{}))
+					id := list[0]
+					row.AddCell().SetValue(fmt.Sprintf(JYHref+"%s.html", util.CommonEncodeArticle("content", id)))
+				} else {
+					row.AddCell().SetValue("")
+				}
+			} else {
+				row.AddCell().SetValue(tmp[v])
+			}
+		}
+	}
+	util.Debug("over ---", count)
+	fname := fmt.Sprintf("./项目导出%s.xlsx", util.NowFormat(util.DATEFORMAT))
+	err = file.Save(fname)
+	if err != nil {
+		panic(err)
+	}
+}

import_execl/config.json

@@ -0,0 +1,13 @@
+{
+  "mongodb": "192.168.3.207:27092",
+  "dbname": "wjh",
+  "dbcoll": "test",
+  "dbsize": 5,
+  "filepath": "/Users/jianghan/Desktop/20220111广东移动第一批10家单位数据V2.xlsx",
+  "update_field": {
+    "省份": "area"
+  },
+  "add_field": {
+    "采购单位标签": "buyer_new"
+  }
+}

import_execl/main.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"github.com/tealeg/xlsx"
+	"mongodb"
+	"qfw/util"
+	"time"
+)
+
+var (
+	Sysconfig map[string]interface{}
+	Mgo       *mongodb.MongodbSim
+	DbColl    string
+	FilePath  string
+	UpdateMap map[string]interface{}
+	AddMap    map[string]interface{}
+	keyIndex  map[int]interface{}
+
+	updatePool chan []map[string]interface{}
+	updateSp   chan bool
+	SE         = util.SimpleEncrypt{Key: "topJYBX2019"}
+)
+
+type RowData struct {
+	row   int
+	id    string
+	field map[string]interface{}
+}
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: Sysconfig["mongodb"].(string),
+		Size:        util.IntAllDef(Sysconfig["dbsize"], 5),
+		DbName:      Sysconfig["dbname"].(string),
+	}
+	Mgo.InitPool()
+	DbColl = Sysconfig["dbcoll"].(string)
+
+	FilePath = Sysconfig["filepath"].(string)
+	UpdateMap = Sysconfig["update_field"].(map[string]interface{})
+	AddMap = Sysconfig["add_field"].(map[string]interface{})
+	keyIndex = make(map[int]interface{}) // 记录需要保存的字段/表格列索引
+
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp = make(chan bool, 5)
+}
+
+func main() {
+	go updateMethod()
+
+	xlFile, err := xlsx.OpenFile(FilePath)
+	if err != nil {
+		return
+	}
+	sheet := xlFile.Sheets[0]
+	for rowIndex, row := range sheet.Rows {
+		fieldMap := make(map[string]interface{})
+		for cellIndex, cell := range row.Cells {
+			util.Debug(rowIndex, cellIndex, cell.String())
+			if rowIndex == 0 {
+				//根据列标题获取列索引
+				if cell.String() == "唯一标识" {
+					keyIndex[cellIndex] = "id"
+				} else {
+					for k, v := range AddMap {
+						if cell.String() == k {
+							keyIndex[cellIndex] = v
+						}
+					}
+					for k, v := range UpdateMap {
+						if cell.String() == k {
+							keyIndex[cellIndex] = v
+						}
+					}
+				}
+			} else {
+				util.Debug(keyIndex)
+				if field := util.ObjToString(keyIndex[cellIndex]); field != "" {
+					if field == "id" {
+						fieldMap[field] = SE.DecodeString(cell.String())
+					} else if field == "budget" || field == "bidamount" {
+						fieldMap[field], _ = cell.Float()
+					} else {
+						fieldMap[field] = cell.String()
+					}
+				}
+			}
+		}
+		if rowIndex != 0 && len(fieldMap) > 0 {
+			saveInfo := []map[string]interface{}{
+				{
+					"_id": mongodb.StringTOBsonId(util.ObjToString(fieldMap["id"])),
+				},
+				{"$set": fieldMap},
+			}
+			updatePool <- saveInfo
+		}
+	}
+
+	c := make(chan bool, 1)
+	<-c
+}
+
+func updateMethod() {
+	arru := make([][]map[string]interface{}, 200)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					Mgo.UpSertBulk(DbColl, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					Mgo.UpSertBulk(DbColl, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		}
+	}
+}

project_data/config.json

@@ -0,0 +1,15 @@
+{
+  "loadStart": 0,
+  "validdays":150,
+  "statusdays": 15,
+  "mongodbServers": "192.168.3.207:27092",
+  "mongodbPoolSize": 10,
+  "mongodbName": "wjh",
+  "hints":"publishtime_1",
+  "extractColl": "oprd_bidding",
+  "projectColl": "oprd_projectset",
+  "backupFlag": true,
+  "siteColl": "site",
+  "thread": 1,
+  "operators": "移动,中移在线,中移系统,中移,铁通,联通,联合网络通信,电信,天翼视讯,天翼电信,广电网络,中国广电,湖北广电"
+}

project_data/deepcopy.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"reflect"
+	"time"
+)
+
+// Interface for delegating copy process to type
+type Interface interface {
+	DeepCopy() interface{}
+}
+
+// Iface is an alias to Copy; this exists for backwards compatibility reasons.
+func Iface(iface interface{}) interface{} {
+	return Copy(iface)
+}
+
+// Copy creates a deep copy of whatever is passed to it and returns the copy
+// in an interface{}.  The returned value will need to be asserted to the
+// correct type.
+func Copy(src interface{}) interface{} {
+	if src == nil {
+		return nil
+	}
+
+	// Make the interface a reflect.Value
+	original := reflect.ValueOf(src)
+
+	// Make a copy of the same type as the original.
+	cpy := reflect.New(original.Type()).Elem()
+
+	// Recursively copy the original.
+	copyRecursive(original, cpy)
+
+	// Return the copy as an interface.
+	return cpy.Interface()
+}
+
+// copyRecursive does the actual copying of the interface. It currently has
+// limited support for what it can handle. Add as needed.
+func copyRecursive(original, cpy reflect.Value) {
+	// check for implement deepcopy.Interface
+	if original.CanInterface() {
+		if copier, ok := original.Interface().(Interface); ok {
+			cpy.Set(reflect.ValueOf(copier.DeepCopy()))
+			return
+		}
+	}
+
+	// handle according to original's Kind
+	switch original.Kind() {
+	case reflect.Ptr:
+		// Get the actual value being pointed to.
+		originalValue := original.Elem()
+
+		// if  it isn't valid, return.
+		if !originalValue.IsValid() {
+			return
+		}
+		cpy.Set(reflect.New(originalValue.Type()))
+		copyRecursive(originalValue, cpy.Elem())
+
+	case reflect.Interface:
+		// If this is a nil, don't do anything
+		if original.IsNil() {
+			return
+		}
+		// Get the value for the interface, not the pointer.
+		originalValue := original.Elem()
+
+		// Get the value by calling Elem().
+		copyValue := reflect.New(originalValue.Type()).Elem()
+		copyRecursive(originalValue, copyValue)
+		cpy.Set(copyValue)
+
+	case reflect.Struct:
+		t, ok := original.Interface().(time.Time)
+		if ok {
+			cpy.Set(reflect.ValueOf(t))
+			return
+		}
+		// Go through each field of the struct and copy it.
+		for i := 0; i < original.NumField(); i++ {
+			// The Type's StructField for a given field is checked to see if StructField.PkgPath
+			// is set to determine if the field is exported or not because CanSet() returns false
+			// for settable fields.  I'm not sure why.  -mohae
+			if original.Type().Field(i).PkgPath != "" {
+				continue
+			}
+			copyRecursive(original.Field(i), cpy.Field(i))
+		}
+
+	case reflect.Slice:
+		if original.IsNil() {
+			return
+		}
+		// Make a new slice and copy each element.
+		cpy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap()))
+		for i := 0; i < original.Len(); i++ {
+			copyRecursive(original.Index(i), cpy.Index(i))
+		}
+
+	case reflect.Map:
+		if original.IsNil() {
+			return
+		}
+		cpy.Set(reflect.MakeMap(original.Type()))
+		for _, key := range original.MapKeys() {
+			originalValue := original.MapIndex(key)
+			copyValue := reflect.New(originalValue.Type()).Elem()
+			copyRecursive(originalValue, copyValue)
+			copyKey := Copy(key.Interface())
+			cpy.SetMapIndex(reflect.ValueOf(copyKey), copyValue)
+		}
+
+	default:
+		cpy.Set(original)
+	}
+}

project_data/init.go

@@ -0,0 +1,462 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"math"
+	"mongodb"
+	"qfw/util"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+)
+
+var (
+	Sysconfig                map[string]interface{} //读取配置文件
+	MongoTool                *mongodb.MongodbSim    //mongodb连接
+	ExtractColl, ProjectColl string                 //抽取表、项目表、项目快照表、站点表
+	Thread                   int                    //配置项线程数
+
+	operators []string // 运营商
+)
+
+var (
+	//判断是日期
+	_datereg   = regexp.MustCompile("20[0-2][0-9][年-][0-9]{1,2}[月-][0-9]{1,2}[日-]([0-9]{1,2}时[0-9]{0,2})?")
+	_numreg1   = regexp.MustCompile("^[0-9-]{1,8}$")
+	_zimureg1  = regexp.MustCompile("^[a-zA-Z-]{1,7}$")
+	_nzreg     = regexp.MustCompile("^[0-9a-zA-Z-]+$")
+	_hanreg    = regexp.MustCompile(`^[\p{Han}:：【】\\[\\]（）()－-、]+$`)
+	replaceStr = regexp.MustCompile("(工程|采购|项目|[?！、【】（）—()－-]|栏标价|中标候选人|招标代理)")
+	//判断带有分包、等特定词的
+	pStr = regexp.MustCompile("(勘察|监理|施工|设计|验收|标段|分包|子包|[0-9A-Z]包|[一二三四五六七八九十0-9]批)")
+	//判断包含数值
+	nreg1 = regexp.MustCompile("[0-9]{2,}")
+	//判断包含字母
+	zreg1 = regexp.MustCompile("[a-zA-Z]{1,}")
+	//判断包含汉字
+	hreg1 = regexp.MustCompile(`[\p{Han}]+`)
+	//判断项目编号是在10以内的纯数字结构
+	numCheckPc = regexp.MustCompile("^[0-9-]{1,10}$")
+	//仅初始化使用
+	compareNoPass = map[string]bool{}
+	compareAB     = map[string]bool{}
+	compareAB2D   = map[string]bool{}
+	compareABD    = map[string]bool{}
+	compareAB2CD  = map[string]bool{}
+	compareABCD   = map[string]bool{}
+)
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	MongoTool = &mongodb.MongodbSim{
+		MongodbAddr: Sysconfig["mongodbServers"].(string),
+		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
+		DbName:      Sysconfig["mongodbName"].(string),
+		//UserName:    "root",
+		//Password:    "root",
+	}
+	MongoTool.InitPool()
+
+	ExtractColl = Sysconfig["extractColl"].(string)
+	ProjectColl = Sysconfig["projectColl"].(string)
+	Thread = util.IntAll(Sysconfig["thread"])
+
+	operators = strings.Split(util.ObjToString(Sysconfig["operators"]), ",")
+
+	//加载项目数据
+	//---不能通过
+	vm := []string{"C", "D"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				compareNoPass[key] = true
+				//fmt.Println(key)
+			}
+		}
+	}
+	//fmt.Println("-------------------")
+
+	//三个元素一致 [AB][AB][AB]，分值最高
+	vm = []string{"A", "B"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				compareAB[key] = true
+				//fmt.Println(key)
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareAB))
+	//---至少两个一致，其他可能不存在
+	//[AB][AB][ABD]
+	//[AB][ABD][AB]
+	vm = []string{"A", "B"}
+	vm2 := []string{"A", "B", "D"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 3; k++ {
+				key := vm[i] + vm[j] + vm2[k]
+				if !compareAB[key] {
+					compareAB2D[key] = true
+					//fmt.Println(key)
+
+				}
+			}
+		}
+	}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 3; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm2[j] + vm[k]
+				if !compareAB[key] {
+					compareAB2D[key] = true
+					//fmt.Println(key)
+
+				}
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareAB2D))
+	//---至少一个一致，其他可能不存在
+	//[ABD][ABD][ABD] //已经删除DDD
+	vm = []string{"A", "B", "D"}
+	for i := 0; i < 3; i++ {
+		for j := 0; j < 3; j++ {
+			for k := 0; k < 3; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] {
+					compareABD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareABD))
+
+	//[AB][ABCD][AB]
+	//[AB][AB][ABCD]
+	vm = []string{"A", "B"}
+	vm2 = []string{"A", "B", "C", "D"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 4; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm2[j] + vm[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
+					compareAB2CD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 4; k++ {
+				key := vm[i] + vm[j] + vm2[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
+					compareAB2CD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareAB2CD))
+	//[ABECD][ABECD][ABECD]  //已经删除[CD][CD][CD]   //这个要重点讨论
+	vm = []string{"A", "B", "C", "D"}
+	for i := 0; i < 4; i++ {
+		for j := 0; j < 4; j++ {
+			for k := 0; k < 4; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareABD[key] && !compareNoPass[key] && !compareAB2CD[key] {
+					compareABCD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+}
+
+//项目合并对象
+type ProjectTask struct {
+	InitMinTime int64 //最小时间，小于0的处理一次
+	name        string
+	thread      int //线程数
+	//查找锁
+	findLock sync.Mutex
+	wg       sync.WaitGroup
+	//map锁
+	AllIdsMapLock sync.Mutex
+	//对应的id
+	AllIdsMap map[string]*ID
+	//采购单位、项目名称、项目编号
+	mapPb, mapPn, mapPc map[string]*Key
+
+	//bidtype、bidstatus 锁
+	mapBidLock sync.Mutex
+	//更新或新增通道
+	updatePool chan []map[string]interface{}
+	//savePool   chan map[string]interface{}
+	//saveSign, updateSign chan bool
+	//表名
+	coll string
+	//当前状态是全量还是增量
+	currentType string //当前是跑全量还是跑增量
+	//
+	clearContimes int
+	//当前时间
+	currentTime int64
+	//保存长度
+	saveSize   int
+	pici       int64
+	validTime  int64
+	statusTime int64
+	//结果时间的更新		最近两天的公告不再更新jgtime
+	jgTime int64
+	//	LockPool     chan *sync.Mutex
+	//	LockPoolLock sync.Mutex
+	//	m1, m23, m4  map[int]int
+	//	l1, l23, l4  map[int]*sync.Mutex
+	Brun bool
+}
+
+func CheckHanAndNum(str string) (b bool) {
+	return nreg1.MatchString(str) && hreg1.MatchString(str)
+}
+func CheckZimuAndNum(str string) (b bool) {
+	return zreg1.MatchString(str) && nreg1.MatchString(str)
+}
+
+type KeyMap struct {
+	Lock sync.Mutex
+	Map  map[string]*Key
+}
+
+type ID struct {
+	Id   string
+	Lock sync.Mutex
+	P    *ProjectInfo
+}
+type Key struct {
+	Arr  []string
+	Lock sync.Mutex
+}
+type IdAndLock struct {
+	Id   string
+	Lock sync.Mutex
+}
+
+func NewKeyMap() *KeyMap {
+	return &KeyMap{
+		Map:  map[string]*Key{},
+		Lock: sync.Mutex{},
+	}
+}
+
+//招标信息实体类
+type Info struct {
+	Id                  string                   `json:"_id"`
+	Href                string                   `json:"href"` //源地址
+	Publishtime         int64                    `json:"publishtime"`
+	Comeintime          int64                    `json:"comeintime"`
+	Title               string                   `json:"title"`
+	TopType             string                   `json:"toptype"`
+	SubType             string                   `json:"subtype"`
+	ProjectName         string                   `json:"projectname"`
+	ProjectCode         string                   `json:"projectcode"`
+	ProjectScope        string                   `json:"projectscope"`
+	ContractCode        string                   `json:"contractcode"`
+	Buyer               string                   `json:"buyer"`
+	Buyerperson         string                   `json:"buyerperson"`
+	Buyertel            string                   `json:"buyertel"`
+	Agency              string                   `json:"agency"`
+	Area                string                   `json:"area"`
+	City                string                   `json:"city"`
+	District            string                   `json:"district"`
+	Infoformat          int                      `json:"infoformat"`
+	ReviewExperts       []string                 `json:"review_experts"`
+	Purchasing          string                   `json:"purchasing"`
+	WinnerOrder         []map[string]interface{} `json:"winnerorder"`
+	ProjectScale        string                   `json:"project_scale"`
+	ProjectDuration     int                      `json:"project_duration"`
+	ProjectTimeUnit     string                   `json:"project_timeunit"`
+	ProjectStartDate    int64                    `json:"project_startdate"`
+	ProjectCompleteDate int64                    `json:"project_completedate"`
+	Payway              string                   `json:"payway"`
+	ContractGuarantee   bool                     `json:"contract_guarantee"`
+	BidGuarantee        bool                     `json:"bid_guarantee"`
+	Qualifies           []map[string]interface{} `json:"qualifies"`
+	EntIdList           []string                 `json:"entidlist"`
+	HasPackage          bool                     // `json:"haspackage"`
+	Package             map[string]interface{}   `json:"package"`
+	Topscopeclass       []string                 `json:"topscopeclass"`
+	Subscopeclass       []string                 `json:"subscopeclass"`
+	Buyerclass          string                   `json:"buyerclass"`
+	Bidopentime         int64                    `json:"bidopentime"`
+	Budget              float64                  `json:"budget"`
+	Bidamount           float64                  `json:"bidamount"`
+	TagRule             string                   `json:"tag_rule"`
+	Winners             []string
+	dealtype            int
+	PTC                 string //从标题中抽的项目编号
+	pnbval              int    //项目名称、编号、采购单位存在的个数
+	LenPC               int    //项目编号长度
+	LenPN               int    //项目名称长度
+	LenPTC              int    //标题抽的项目编号长度
+	//以下三个元素做对比，计算包含时候使用
+	PNBH  int //0初始，+包含，-被包含
+	PCBH  int
+	PTCBH int
+}
+
+//项目实体类
+type ProjectInfo struct {
+	Id                 primitive.ObjectID     `json:"_id"`
+	FirstTime          int64                  `json:"firsttime,omitempty"` //项目的最早时间
+	LastTime           int64                  `json:"lasttime,omitempty"`  //项目的最后时间
+	Ids                []string               `json:"ids,omitempty"`
+	Topscopeclass      []string               `json:"topscopeclass,omitempty"`
+	Subscopeclass      []string               `json:"subscopeclass,omitempty"` //子行业分类
+	Winners            []string               `json:"s_winner,omitempty"`      //中标人
+	ProjectName        string                 `json:"projectname,omitempty"`   //项目名称
+	ProjectCode        string                 `json:"projectcode,omitempty"`   //项目代码唯一（纯数字的权重低）
+	ContractCode       string                 `json:"contractcode,omitempty"`  //项目编号
+	Buyer              string                 `json:"buyer,omitempty"`         //采购单位唯一
+	MPN                []string               `json:"mpn,omitempty"`           //合并后多余的项目名称
+	MPC                []string               `json:"mpc,omitempty"`           //合并后多余的项目编号
+	Buyerperson        string                 `json:"buyerperson"`             //采购联系人
+	Buyertel           string                 `json:"buyertel"`                //采购联系人电话
+	Agency             string                 `json:"agency"`                  //代理机构
+	Area               string                 `json:"area"`                    //地区
+	City               string                 `json:"city"`                    //地市
+	District           string                 `json:"district"`                //区县
+	Bidstatus          string                 `json:"bidstatus"`               //
+	Bidtype            string                 `json:"bidtype"`                 //
+	ReviewExperts      []string               `json:"review_experts"`          // 项目评审专家
+	Purchasing         string                 `json:"purchasing"`              // 标的物
+	Package            map[string]interface{} `json:"package,omitempty"`       //分包的对比对象
+	Buyerclass         string                 `json:"buyerclass"`              //采购单位分类
+	Bidopentime        int64                  `json:"bidopentime,omitempty"`   //开标时间
+	Jgtime             int64                  `json:"jgtime"`                  //结果中标时间
+	Zbtime             int64                  `json:"zbtime"`                  //招标时间
+	Bidamount          float64                `json:"bidamount,omitempty"`     //中标金额
+	Budget             float64                `json:"budget,omitempty"`        //预算
+	Winnerorder        []string               `json:"winnerorder"`             //中标候选人
+	ProjectScale       string                 `json:"project_scale"`           //项目规模
+	ProjectDuration    int                    `json:"project_duration"`        //工期时长
+	ProjectTimeunit    string                 `json:"project_timeunit"`        //工期时长单位
+	ProjectStartDate   int64                  `json:"project_startdate"`       //开工日期
+	ProjctCompleteDate int64                  `json:"projct_completedate"`     //竣工日期
+	Payway             string                 `json:"payway"`                  //付款方式
+	ContractGuarantee  bool                   `json:"contract_guarantee"`      //履约保证金 是否支持包含
+	BidGuarantee       bool                   `json:"bid_guarantee"`           //投标保证金 是否支持包含
+	Qualifies          string                 `json:"qualifies"`               //资质条件
+	TagRule            string                 `json:"tag_rule"`                //数据标签
+	IsOperators        bool                   `json:"isOperators"`             //是否是运营商
+	EntIdList          []string               `json:"entidlist"`               //企业id
+	score              int
+	comStr             string
+	resVal, pjVal      int
+	InfoFiled          map[string]InfoField `json:"infofield"`    //逻辑处理需要的info字段
+	Budgettag          int                  `json:"budgettag"`    //预算是否有效标记
+	Bidamounttag       int                  `json:"bidamounttag"` //中标金额是否有效标记
+}
+
+//存储部分招标信息字段，业务逻辑处理需要
+type InfoField struct {
+	Budget       float64 `json:"budget"`
+	Bidamount    float64 `json:"bidamount"`
+	ContractCode string  `json:"contractcode"`
+	ProjectName  string  `json:"projectname"`
+	ProjectCode  string  `json:"projectcode"`
+	Bidstatus    string  `json:"bidstatus"`
+}
+
+//二分字符串查找
+func BinarySearch(s []string, k string) int {
+	sort.Strings(s)
+	lo, hi := 0, len(s)-1
+	for lo <= hi {
+		m := (lo + hi) >> 1
+		if s[m] < k {
+			lo = m + 1
+		} else if s[m] > k {
+			hi = m - 1
+		} else {
+			return m
+		}
+	}
+	return -1
+}
+
+func Duplicate(a interface{}) (ret []interface{}) {
+	va := reflect.ValueOf(a)
+	for i := 0; i < va.Len(); i++ {
+		if i > 0 && reflect.DeepEqual(va.Index(i-1).Interface(), va.Index(i).Interface()) {
+			continue
+		}
+		ret = append(ret, va.Index(i).Interface())
+	}
+	return ret
+}
+
+//计算文本相似度
+func CosineSimilar(srcWords1, dstWords1 string) float64 {
+	srcWords, dstWords := strings.Split(srcWords1, ""), strings.Split(dstWords1, "")
+	// get all words
+	allWordsMap := make(map[string]int, 0)
+	for _, word := range srcWords {
+		if _, found := allWordsMap[word]; !found {
+			allWordsMap[word] = 1
+		} else {
+			allWordsMap[word] += 1
+		}
+	}
+	for _, word := range dstWords {
+		if _, found := allWordsMap[word]; !found {
+			allWordsMap[word] = 1
+		} else {
+			allWordsMap[word] += 1
+		}
+	}
+
+	// stable the sort
+	allWordsSlice := make([]string, 0)
+	for word, _ := range allWordsMap {
+		allWordsSlice = append(allWordsSlice, word)
+	}
+
+	// assemble vector
+	srcVector := make([]int, len(allWordsSlice))
+	dstVector := make([]int, len(allWordsSlice))
+	for _, word := range srcWords {
+		if index := BinarySearch(allWordsSlice, word); index != -1 {
+			srcVector[index] += 1
+		}
+	}
+	for _, word := range dstWords {
+		if index := BinarySearch(allWordsSlice, word); index != -1 {
+			dstVector[index] += 1
+		}
+	}
+
+	// calc cos
+	numerator := float64(0)
+	srcSq := 0
+	dstSq := 0
+	for i, srcCount := range srcVector {
+		dstCount := dstVector[i]
+		numerator += float64(srcCount * dstCount)
+		srcSq += srcCount * srcCount
+		dstSq += dstCount * dstCount
+	}
+	denominator := math.Sqrt(float64(srcSq * dstSq))
+
+	v1 := numerator / denominator
+	//	if v1 > 0.6 {
+	//		log.Println(v1, srcWords1, dstWords1)
+	//	}
+	return v1
+}

project_data/load_data.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"time"
+)
+
+//  初始加载数据，默认加载最近6个月的数据
+func (p *ProjectTask) loadData(starttime int64) {
+	log.Println("load project start..", starttime)
+	p.findLock.Lock()
+	defer p.findLock.Unlock()
+	p.AllIdsMapLock.Lock()
+	defer p.AllIdsMapLock.Unlock()
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+	q := map[string]interface{}{
+		"lasttime": map[string]interface{}{"$gte": starttime},
+	}
+	it := sess.DB(MongoTool.DbName).C(p.coll).Find(&q).Iter()
+	n := 0
+	pool := make(chan *ProjectInfo, 100)
+	over := make(chan bool)
+	go func() {
+		for {
+			select {
+			case tmp := <-pool:
+				n++
+				if n%5000 == 0 {
+					log.Println("current", n, "\n", tmp.Id, len(p.mapPn), len(p.mapPc), len(p.mapPb)) //, tmp.ProjectName, tmp.MPN, tmp.ProjectCode, tmp.MPC, tmp.Buyer)
+				}
+				if tmp != nil {
+					id := tmp.Id.Hex()
+					for _, v := range append([]string{tmp.ProjectName}, tmp.MPN...) {
+						if v != "" {
+							//v = pcReplace.ReplaceAllString(v, "")
+							if v != "" {
+								k := p.mapPn[v]
+								if k == nil {
+									k = &Key{Arr: []string{id}}
+									p.mapPn[v] = k
+								} else {
+									k.Arr = append(k.Arr, id)
+								}
+							}
+						}
+					}
+					for _, v := range append([]string{tmp.ProjectCode}, tmp.MPC...) {
+						if v != "" {
+							//v = pcReplace.ReplaceAllString(v, "")
+							if v != "" {
+								k := p.mapPc[v]
+								if k == nil {
+									k = &Key{Arr: []string{id}}
+									p.mapPc[v] = k
+								} else {
+									k.Arr = append(k.Arr, id)
+								}
+							}
+						}
+					}
+					if tmp.Buyer != "" && len([]rune(tmp.Buyer)) > 2 {
+						k := p.mapPb[tmp.Buyer]
+						if k == nil {
+							k = &Key{Arr: []string{id}}
+							p.mapPb[tmp.Buyer] = k
+						} else {
+							k.Arr = append(k.Arr, id)
+						}
+					}
+					p.AllIdsMap[id] = &ID{Id: id, P: tmp}
+				}
+			case <-over:
+				return
+			}
+		}
+	}()
+	for {
+		result := make(map[string]interface{})
+		if it.Next(&result) {
+			go func(res map[string]interface{}) {
+				bys, _ := json.Marshal(result)
+				var tmp *ProjectInfo
+				_ = json.Unmarshal(bys, &tmp)
+				saveFiled(p, result, tmp)
+				pool <- tmp
+			}(result)
+		} else {
+			break
+		}
+	}
+	time.Sleep(2 * time.Second)
+	over <- true
+	log.Println("load project over..", n)
+}
+
+func saveFiled(p *ProjectTask, res map[string]interface{}, tmp *ProjectInfo) {
+	tmpMap := make(map[string]InfoField)
+	infoMap := res["infofield"].(map[string]interface{})
+	for _, v := range infoMap {
+		var field InfoField
+		b, _ := json.Marshal(v)
+		_ = json.Unmarshal(b, &field)
+		tmpMap[tmp.Id.Hex()] = field
+	}
+	tmp.InfoFiled = tmpMap
+}

project_data/main.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"qfw/util"
+	"qfw/util/elastic"
+	"time"
+)
+
+var (
+	SingleClear = 0
+	Es          *elastic.Elastic
+	Index       string
+	Itype       string
+)
+
+func main() {
+	if Sysconfig["loadStart"] != nil {
+		loadStart := util.Int64All(Sysconfig["loadStart"])
+		if loadStart > -1 {
+			P_QL.loadData(loadStart)
+		}
+	}
+
+	P_QL.currentType = "project"
+	P_QL.pici = time.Now().Unix()
+	P_QL.taskQl()
+
+	c := make(chan bool, 1)
+	<-c
+}

project_data/merge_comparepnc.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"strings"
+)
+
+//对比项目名称、项目编号
+
+func comparePNC(info *Info, compareProject *ProjectInfo) (compareStr string, score int) {
+	if info.ProjectName != "" {
+		pns := []string{}
+		if compareProject.ProjectName != "" {
+			pns = append(pns, compareProject.ProjectName)
+		}
+		if len(compareProject.MPN) > 0 {
+			pns = append(pns, compareProject.MPN...)
+		}
+		ifind := 0
+		templen := 0
+		buyer := info.Buyer
+		if buyer == "" {
+			buyer = compareProject.Buyer
+		}
+		pn := info.ProjectName
+		if buyer != "" {
+			pn = strings.Replace(pn, buyer, "", -1)
+		}
+		for _, v := range pns {
+			if info.ProjectName == v {
+				ifind = 1
+				break
+			} else {
+				//if strings.Contains(info.ProjectName, v) || strings.Contains(v, info.ProjectName) ||
+				retv := CheckContain(info.ProjectName, v)
+				if retv == 1 {
+					ifind = 1
+					break
+				} else {
+					if buyer != "" {
+						v = strings.Replace(v, buyer, "", -1)
+					}
+					//v1 := CosineSimilar(pn, v)
+					if retv == 2 {
+						templen = len([]rune(v))
+						ifind = 2
+					} else if ifind == 0 {
+						ifind = 3
+					}
+				}
+			}
+		}
+		switch ifind {
+		case 0:
+			compareStr = "D"
+		case 1:
+			compareStr = "A"
+			score += 4
+			if len([]rune(info.ProjectName)) > 18 {
+				score += 2
+			}
+		case 2:
+			compareStr = "B"
+			score += 2
+			if templen > info.LenPN {
+				templen = info.LenPN
+			}
+			info.PNBH = templen
+			if templen > 12 {
+				score += 1
+			}
+		case 3:
+			compareStr = "C"
+		}
+	} else {
+		compareStr = "D"
+	}
+
+	/*
+				项目编号 - －()（） 要注意
+				init_text = ["号","(重)","(第二次)","（重）"]
+		all_clean_mark = ["[","(","【","（","〖","]",")","】","）","〗","-","〔","〕","《","［","］","｛","｝","{","—"," ","－","﹝","﹞","–"]
+	*/
+	for index, pc := range []string{info.ProjectCode, info.PTC} {
+		if pc != "" {
+			pcs := []string{}
+			if compareProject.ProjectCode != "" {
+				pcs = append(pcs, compareProject.ProjectCode)
+			}
+			if len(compareProject.MPC) > 0 {
+				pcs = append(pcs, compareProject.MPC...)
+			}
+			ifind := 0
+			templen := 0
+			for _, v := range pcs {
+				if pc == v {
+					ifind = 1
+					break
+				} else {
+					// math.Abs(float64(len([]rune(pc))-len([]rune(v)))) < 6
+					//if !_numreg1.MatchString(pc) && !_zimureg1.MatchString(pc) && !_numreg1.MatchString(v) && !_zimureg1.MatchString(v)
+					if strings.Contains(pc, v) || strings.Contains(v, pc) {
+						t1 := pc
+						t2 := v
+						if len(v) > len(pc) {
+							t1 = v
+							t2 = pc
+						}
+						t3 := strings.Replace(t1, t2, "", -1)
+						t3 = _datereg.ReplaceAllString(t3, "")
+						if t3 == "" {
+							ifind = 1
+							break
+						} else {
+							ifind = 2
+							templen = len([]rune(v))
+						}
+					} else if ifind == 0 {
+						ifind = 3
+					}
+				}
+			}
+			switch ifind {
+			case 0:
+				compareStr += "D"
+			case 1:
+				compareStr += "A"
+				score += 4
+				if len([]rune(pc)) > 18 {
+					score += 2
+				}
+			case 2:
+				compareStr += "B"
+				score += 2
+				if index == 0 {
+					if templen > info.LenPC {
+						templen = info.LenPC
+					}
+					info.PCBH = templen
+					if templen > 12 {
+						score += 1
+					}
+
+				} else {
+					if templen > info.LenPTC {
+						templen = info.LenPTC
+					}
+					info.PTCBH = templen
+					if templen > 12 {
+						score += 1
+					}
+				}
+
+			case 3:
+				compareStr += "C"
+			}
+
+		} else {
+			compareStr += "D"
+		}
+	}
+	return
+}
+
+func CheckContain(b1, b2 string) (res int) {
+	b1 = replaceStr.ReplaceAllString(b1, "")
+	b2 = replaceStr.ReplaceAllString(b2, "")
+
+	if b1 == b2 {
+		res = 1 //相等
+		return
+	}
+	bs1 := []rune(b1)
+	bs2 := []rune(b2)
+	tmp := ""
+	for i := 0; i < len(bs1); i++ {
+		for j := 0; j < len(bs2); j++ {
+			if bs1[i] == bs2[j] {
+				tmp += string(bs1[i])
+			} else if tmp != "" {
+				b1 = strings.Replace(b1, tmp, "", -1)
+				b2 = strings.Replace(b2, tmp, "", -1)
+				tmp = ""
+			}
+		}
+	}
+	if tmp != "" {
+		b1 = strings.Replace(b1, tmp, "", -1)
+		b2 = strings.Replace(b2, tmp, "", -1)
+	}
+	if b1 == b2 {
+		res = 1 //相等
+	} else if b1 == "" || b2 == "" {
+		res = 2 //包含
+	} else {
+		res = 3 //不相等
+	}
+	return
+}

project_data/merge_select.go

@@ -0,0 +1,510 @@
+package main
+
+//根据字符特征打分
+//3为最高分，pj为评级 A AD A  AA AA AB
+func Select(compareStr string, info *Info, compareInfo *ProjectInfo) (res, pj int) {
+	//没有可对比的项目名称、或项目编号 //评级
+	if compareNoPass[compareStr] {
+
+	} else {
+		switch compareStr {
+		case "AAA":
+			res = 3
+			pj = 3
+		case "AAB":
+			res = 3
+			pj = 3
+		case "ABA":
+			res = 3
+			pj = 3
+		case "ABB":
+			if info.LenPN > 10 || info.PCBH > 8 || info.PTCBH > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BAA":
+			if info.PNBH > 10 || info.LenPC > 8 || info.LenPTC > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BAB":
+			if info.PNBH > 10 || info.LenPTC > 8 || info.PTCBH > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BBA":
+			if info.PNBH > 10 || info.PCBH > 8 || info.LenPC > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BBB":
+			v := 0
+			if info.PNBH > 10 {
+				v++
+			}
+			if info.PCBH > 8 {
+				v++
+			}
+			if info.PTCBH > 8 {
+				v++
+			}
+			if v > 1 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 2
+		case "AAD":
+			if info.LenPC > 8 || info.LenPN > 12 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "ABD":
+			if info.LenPN > 10 && info.PCBH > 8 {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BAD":
+			if info.LenPC > 13 || (info.PNBH > 10 && info.LenPC > 8) {
+				res = 3
+				pj = 3
+			} else if info.PNBH > 10 || info.LenPC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BBD":
+			if info.PNBH > 12 && info.PCBH > 10 {
+				res = 3
+				pj = 1
+			} else if info.PNBH > 10 && info.PCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "ADA":
+			if info.LenPN > 12 || (info.LenPTC > 8 && !StrOrNum2.MatchString(info.PTC)) {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 2
+		case "ADB":
+			if info.LenPN > 10 && info.PTCBH > 8 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PTCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BDA":
+			if info.PNBH > 10 && info.LenPTC > 8 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.PNBH > 10 || info.LenPTC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BDB":
+			if info.PNBH > 12 && info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+			} else if info.PNBH > 10 && info.PTCBH > 8 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 2
+		case "ADD":
+			if info.LenPN > 18 {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "BDD":
+			if info.PNBH > 10 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "DAA":
+			if info.LenPTC > 8 || info.LenPC > 8 {
+				res = 3
+				pj = 2
+			} else {
+				res = 2
+				pj = 3
+			}
+		case "DAB":
+			if info.LenPC > 8 && info.PTCBH > 8 {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 || info.PTCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "DAD":
+			if info.LenPC > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "DBA":
+			if info.PCBH > 8 && info.LenPC > 8 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.PCBH > 8 || info.LenPC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "DBB":
+			if info.PCBH > 10 && info.PTCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 1
+			} else if info.PCBH > 8 && info.PTCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "DBD":
+			if info.PCBH > 12 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+				pj = 1
+			} else {
+				res = 1
+				pj = 1
+			}
+		case "DDA":
+			if info.LenPTC > 14 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 1
+			} else if info.LenPTC > 8 {
+				res = 2
+				pj = 1
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "DDB":
+			if info.PTCBH > 12 && !StrOrNum2.MatchString(info.PTC) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "ACA":
+			if info.LenPN > 10 && info.LenPTC > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.LenPTC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "ACB":
+			if info.LenPN > 10 && info.PTCBH > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PTCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BCA":
+			if (info.PNBH > 10 && info.LenPTC > 8) || info.LenPTC > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.PNBH > 10 || info.LenPTC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BCB":
+			if info.PNBH > 12 && info.PTCBH > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 1
+			} else if info.PNBH > 10 || info.PTCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "AAC":
+			if (info.LenPN > 10 && info.LenPC > 8) || info.LenPN > 14 || (info.LenPC > 10 && !StrOrNum2.MatchString(info.ProjectCode)) {
+				res = 3
+				pj = 3
+			} else {
+				res = 2
+				pj = 3
+			}
+		case "ABC":
+			if info.LenPN > 14 && info.PCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BAC":
+			if info.PNBH > 14 && info.LenPC > 8 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.PNBH > 10 || info.LenPC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BBC":
+			if info.PNBH > 14 && info.PCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 1
+			} else if info.PNBH > 10 || info.PCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "ACC":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPN > 16 {
+					res = 2
+					pj = 1
+				} else {
+					res = 1
+					pj = 2
+				}
+			}
+		case "ACD":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPN > 16 {
+
+					res = 2
+				} else {
+					res = 1
+
+				}
+				pj = 1
+			}
+
+		case "ADC":
+			if info.LenPN > 16 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "BCC":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PNBH > 12 {
+					res = 1
+				}
+				pj = 1
+			}
+		case "BCD":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PNBH > 8 {
+					res = 1
+				}
+				pj = 1
+			}
+		case "BDC":
+			if info.PNBH > 7 {
+				res = 1
+			}
+			pj = 1
+		case "CAA":
+			if info.LenPC > 12 || info.LenPTC > 12 {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 || info.LenPTC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "CAB":
+			if info.LenPC > 12 && info.PTCBH > 8 {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 12 || info.PTCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "CAC":
+			if info.LenPC > 9 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 1
+			}
+		case "CAD":
+			if info.LenPC > 9 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "CBA":
+			if info.LenPTC > 14 && info.PCBH > 12 {
+				res = 3
+				pj = 2
+			} else if info.LenPTC > 12 || info.PCBH > 10 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "CBB":
+			if info.PCBH > 13 && info.PTCBH > 13 {
+				res = 3
+				pj = 1
+			} else if info.PCBH > 9 || info.PTCBH > 9 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "CBC":
+			if info.PCBH > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else if info.PCBH > 5 {
+				res = 1
+			}
+			pj = 1
+		case "CBD":
+			if info.PCBH > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else if info.PCBH > 5 {
+				res = 1
+			}
+			pj = 1
+		case "CCA":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPTC > 12 && !StrOrNum2.MatchString(info.PTC) {
+					res = 2
+				} else if info.LenPTC > 5 {
+					res = 1
+				}
+				pj = 1
+			}
+		case "CCB":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
+					res = 1
+				}
+				pj = 1
+			}
+		case "CDA":
+			if info.LenPTC > 12 && !StrOrNum2.MatchString(info.PTC) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "CDB":
+			if info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
+				res = 1
+				pj = 1
+			}
+		case "DAC":
+			if info.LenPC > 13 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+			} else if info.LenPC > 8 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "DBC":
+			if info.PCBH > 8 {
+				res = 1
+			}
+			pj = 1
+		case "DCA":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPTC > 10 {
+					res = 2
+				} else {
+					res = 1
+				}
+				pj = 1
+			}
+		case "DCB":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PTCBH > 8 && !StrOrNum2.MatchString(info.PTC) {
+					res = 1
+				}
+				pj = 1
+			}
+		}
+
+	}
+	return
+}

project_data/project.go

@@ -0,0 +1,1485 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"math"
+	"mongodb"
+	qu "qfw/util"
+	"sort"
+	"strings"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+/**
+项目合并，对比，计算，合并，生成项目
+**/
+
+//从对应map中获取对比的项目id
+func (p *ProjectTask) getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
+	//	p.ConCurrentLock(n1, n2, n3, n4)
+	//	defer p.ConCurrentUnLock(n1, n2, n3, n4)
+	p.wg.Add(1)
+	//查找到id数组
+	res = []*Key{}
+	//是否查找到，并标识位置。-1代表值为空或已经存在。
+	bpn, bpc, bptc, bpb = -1, -1, -1, -1
+	if pn != "" {
+		ids := p.mapPn[pn]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			p.mapPn[pn] = ids
+			bpn = 0
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+
+	if pc != "" {
+		ids := p.mapPc[pc]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			p.mapPc[pc] = ids
+			bpc = len(res)
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+
+	if ptc != "" {
+		ids := p.mapPc[ptc]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			p.mapPc[ptc] = ids
+			bptc = len(res)
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+
+	if pb != "" {
+		ids := p.mapPb[pb]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			p.mapPb[pb] = ids
+			bpb = len(res)
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+	repeatId := map[string]bool{}
+	idArr = []string{} //项目id
+	IDArr = []*ID{}    //项目信息
+	for _, m := range res {
+		for _, id := range m.Arr {
+			if !repeatId[id] {
+				repeatId[id] = true
+				//_, _ = strconv.ParseInt(id[0:8], 16, 64)
+				p.AllIdsMapLock.Lock()
+				Id := p.AllIdsMap[id]
+				p.AllIdsMapLock.Unlock()
+				if Id != nil {
+					Id.Lock.Lock()
+					idArr = append(idArr, id)
+					IDArr = append(IDArr, Id)
+				}
+			}
+		}
+	}
+	return
+}
+
+func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{}) {
+	p.findLock.Lock()
+	defer p.findLock.Unlock()
+
+	//只有或没有采购单位的无法合并
+	//bpn, bpc, bptc, bpb 是否查找到，并标识位置。-1代表未查找到。
+	//pids 是项目id数组集合
+	//IDArr，是单个项目ID对象集合
+	bpn, bpc, bptc, bpb, pids, _, IDArr := p.getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer) //, info.LenPN, info.LenPC, info.LenPTC, len([]rune(info.Buyer)))
+	defer p.wg.Done()
+	//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock
+
+	for _, m := range pids {
+		defer m.Lock.Unlock()
+	}
+	for _, id := range IDArr {
+		defer id.Lock.Unlock()
+	}
+
+	bFindProject := false
+	findPid := ""
+	//获取完id，进行计算
+	//定义两组
+	comRes1 := []*ProjectInfo{} //优先级最高的对比结果数组
+	comRes2 := []*ProjectInfo{} //优化级其次
+	comRes3 := []*ProjectInfo{}
+	for _, v := range IDArr {
+		comStr := ""
+		compareProject := v.P
+		compareProject.score = 0
+		//问题出地LastTime!!!!!
+		diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime)))
+		if diffTime <= p.validTime {
+			//代理机构完全不相同，直接新建项目
+			if CheckContain(compareProject.Agency, info.Agency) == 3 {
+				continue
+			}
+			//地区(省、市、区)不同，直接新建项目
+			if ComparePlace(compareProject, info) {
+				continue
+			}
+
+			info.PNBH = 0
+			info.PCBH = 0
+			info.PTCBH = 0
+			compareStr, score := comparePNC(info, compareProject)
+
+			resVal, pjVal := Select(compareStr, info, compareProject)
+			//---------------------------------------
+			//log.Println(resVal, pjVal, compareProject)
+			if resVal > 0 {
+				compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount, score2 := p.compareBCTABB(info, compareProject, diffTime, score)
+
+				//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构
+				comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
+				compareProject.comStr = comStr
+				compareProject.pjVal = pjVal
+				compareProject.resVal = resVal
+				//log.Println(compareProject.comStr)
+				eqV := compareResult(resVal, pjVal, score2, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount)
+				if eqV == 1 {
+					comRes1 = append(comRes1, compareProject)
+				} else if eqV == 2 {
+					comRes2 = append(comRes2, compareProject)
+				} else if eqV == 3 {
+					comRes3 = append(comRes3, compareProject)
+				}
+			}
+		}
+	}
+	//--------------------------------对比完成-----------------------
+
+	//更新数组、更新项目
+	for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} {
+		if len(resN) > 0 {
+			if len(resN) > 1 {
+				sort.Slice(resN, func(i, j int) bool {
+					return resN[i].score > resN[j].score
+				})
+			}
+
+			ex := 0
+			resArr := []*ProjectInfo{}
+			for i, res := range resN {
+				choose, e := p.CompareStatus(resN[i], info)
+				if !choose {
+					ex = e
+					resArr = append(resArr, res)
+				}
+			}
+			if len(resArr) > 0 {
+				bFindProject = true
+				findPid = resArr[0].Id.Hex()
+				p.UpdateProject(tmp, info, resArr[0], kv+1, resArr[0].comStr, ex)
+				for k2, bv := range []int{bpn, bpc, bptc, bpb} {
+					if bv > -1 {
+						pids[bv].Arr = append(pids[bv].Arr, findPid)
+						if k2 == 0 {
+							if resArr[0].ProjectName == "" {
+								resArr[0].ProjectName = info.ProjectName
+							} else {
+								if resArr[0].MPN == nil {
+									resArr[0].MPN = []string{info.ProjectName}
+								} else {
+									resArr[0].MPN = append(resArr[0].MPN, info.ProjectName)
+								}
+							}
+
+						} else if k2 < 3 {
+							if resArr[0].ProjectCode == "" {
+								resArr[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
+							} else {
+								if resArr[0].MPC == nil {
+									resArr[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
+								} else {
+									resArr[0].MPC = append(resArr[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
+								}
+							}
+						} else {
+							if resArr[0].Buyer == "" {
+								resArr[0].Buyer = info.Buyer
+							}
+						}
+					}
+				}
+			} else {
+				bFindProject = false
+				findPid = ""
+			}
+			break
+		}
+	}
+
+	if !bFindProject {
+		//if !IsCreatePro(info) {
+		//	qu.Debug("舍弃数据---", info.Id)
+		//	return
+		//}
+		id, p1 := p.NewProject(tmp, info)
+		p.AllIdsMapLock.Lock()
+		p.AllIdsMap[id] = &ID{Id: id, P: p1}
+		p.AllIdsMapLock.Unlock()
+		for _, m := range pids {
+			m.Arr = append(m.Arr, id)
+		}
+	}
+}
+
+func (p *ProjectTask) compareBCTABB(info *Info, cp *ProjectInfo, diffTime int64, score int) (compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string, score2 int) {
+	compareBuyer = "D"
+	if len([]rune(info.Buyer)) > 3 && len([]rune(cp.Buyer)) > 3 {
+		v := CheckContain(info.Buyer, cp.Buyer)
+		if v == 1 {
+			compareBuyer = "A"
+			score += 3
+		} else {
+			//v1 := CosineSimilar(info.Buyer, cp.Buyer)
+			if v == 2 {
+				compareBuyer = "B"
+				score += 1
+			} else {
+				compareBuyer = "C"
+			}
+		}
+	}
+	//---------------------------------------
+
+	compareCity = ""
+	if info.Area != "全国" && info.Area != "" && info.Area == cp.Area {
+		compareCity += "A"
+		score += 2
+	} else if info.Area == "全国" || cp.Area == "全国" {
+		compareCity += "B"
+		score += 1
+	} else {
+		compareCity += "C"
+	}
+	if compareCity != "C" {
+		if info.City != "" && info.City == cp.City {
+			compareCity += "A"
+			score += 2
+		} else {
+			if info.Area == "全国" || cp.Area == "全国" {
+				compareCity += "B"
+			} else if info.City == compareCity {
+				compareCity += "B"
+			} else {
+				compareCity += "C"
+			}
+		}
+	} else {
+		compareCity += "C"
+	}
+	score2 = 0
+	if compareCity == "AA" {
+		if info.District != "" && info.District == cp.District {
+			score2 = 1
+		}
+	}
+
+	compareTime = "D"
+	if diffTime < 45*86400 {
+		compareTime = "A"
+		score += 2
+	} else if diffTime < 90*86400 {
+		compareTime = "B"
+		score += 1
+	}
+
+	compareAgency = "D"
+	if info.Agency != "" {
+		if info.Agency == cp.Agency {
+			compareAgency = "A"
+			score += 2
+			score2 += 1
+		} else if cp.Agency != "" {
+			if strings.Contains(info.Agency, cp.Agency) || strings.Contains(cp.Agency, info.Agency) {
+				compareAgency = "B"
+				score += 1
+				score2 += 1
+			} else {
+				compareAgency = "C"
+			}
+		}
+	}
+	compareBudget = "C"
+	if info.Budget > 0 && (info.Budget == cp.Budget || (cp.Bidamount > 0 && info.Budget > cp.Bidamount && (info.Budget-cp.Bidamount) < (0.15*info.Budget))) {
+		compareBudget = "A"
+		score += 1
+		score2 += 1
+	}
+	//	else if info.Budget == 0 && cp.Budget == 0 {
+	//		compareBudget = "B"
+	//	}
+	compareBidmount = "C"
+	if info.Bidamount > 0 && (info.Bidamount == cp.Bidamount || (cp.Budget > 0 && cp.Budget > info.Bidamount && (cp.Budget-info.Bidamount) < 0.15*cp.Budget)) {
+		compareBidmount = "A"
+		score += 1
+		score2 += 1
+	}
+	//	else if info.Bidamount == 0 && cp.Bidamount == 0 {
+	//		compareBidmount = "B"
+	//	}
+
+	cp.score = score
+	return
+}
+
+func compareResult(resVal, pjVal, score2 int, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string) int {
+	eqV := 0
+	switch resVal {
+	case 3:
+		if pjVal == 3 && comStr[3:] != "CCCDCCC" {
+			eqV = 1
+		} else if compareBuyer < "C" {
+			if pjVal > 1 {
+				eqV = 1
+			} else { //if (compareCity[1:1] != "C" || compareTime != "D") && score2 > 0
+				eqV = 2
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && (compareCity[1:1] != "C" || score2 > 0) {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+		} else {
+			if pjVal == 3 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if pjVal == 2 && compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			} else if compareCity == "AA" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+		}
+	case 2:
+		if compareBuyer < "C" {
+			if pjVal > 1 {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" || score2 > 0 {
+				eqV = 3
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+
+		} else {
+			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		}
+	case 1:
+		if compareBuyer < "C" {
+			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		} else {
+			if pjVal > 1 && compareTime == "A" && score2 > 0 && (compareBudget == "A" || compareBidmount == "A") && compareCity[1:1] != "C" {
+				eqV = 3
+			}
+		}
+	}
+	return eqV
+}
+
+//招标时间zbtime、中标时间jgtime、项目状态bidstatus、招标类型bidtype、最后发布时间lasttime、首次发布时间firsttime
+func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *ProjectInfo) {
+	//pId := primitive.NewObjectID() //NewObjectId()
+	pId := mongodb.StringTOBsonId(thisinfo.Id)
+	set := map[string]interface{}{}
+	set["_id"] = pId
+	for _, f := range FIELDS {
+		if tmp[f] != nil && tmp[f] != "" {
+			set[f] = tmp[f]
+		}
+	}
+	bidopentime := qu.Int64All(tmp["bidopentime"])
+	if bidopentime > 0 {
+		set["bidopentime"] = bidopentime
+	}
+	//异常标记
+	if thisinfo.TopType != "招标" && thisinfo.TopType != "拟建" && thisinfo.TopType != "预告" {
+		set["exception"] = 1
+	}
+	//projecthref保存
+	if jsonData, ok := tmp["jsondata"].(map[string]interface{}); ok {
+		if jsonData != nil && qu.ObjToString(jsonData["projecthref"]) != "" {
+			set["projecthref"] = jsonData["projecthref"]
+		}
+	}
+	//合同编号
+	if thisinfo.ContractCode != "" {
+		set["contractcode"] = thisinfo.ContractCode
+	}
+
+	bt := qu.ObjToString(tmp["toptype"])
+	bs := qu.ObjToString(tmp["subtype"])
+	p.mapBidLock.Lock()
+	if thisinfo.Infoformat == 2 || thisinfo.SubType == "拟建" {
+		set["bidstatus"] = "拟建"
+		bt = "拟建"
+	} else {
+		if bidtype[bs] != "" {
+			set["bidtype"] = bidtype[bs]
+		} else {
+			set["bidtype"] = "招标"
+		}
+		if bt == "招标" {
+			set["projectscope"] = qu.ObjToString(tmp["projectscope"])
+			set["bidstatus"] = bt
+		} else {
+			if bidstatus[bs] != "" {
+				set["bidstatus"] = thisinfo.SubType
+				bt = thisinfo.SubType
+			} else if bs == "" {
+				set["bidstatus"] = ""
+				bt = ""
+			} else {
+				set["bidstatus"] = "其它"
+				bt = "其它"
+			}
+		}
+	}
+	p.mapBidLock.Unlock()
+
+	pkg := PackageFormat(thisinfo, nil)
+	p1 := p.NewCachePinfo(pId, thisinfo, bs, bt, pkg)
+	set["tag_rule"] = p1.TagRule
+	set["buyertagname"] = tmp["buyertagname"]
+
+	now := time.Now().Unix()
+	set["createtime"] = now
+	set["sourceinfoid"] = thisinfo.Id
+	set["sourceinfourl"] = tmp["href"]
+	set["firsttime"] = tmp["publishtime"]
+	set["lasttime"] = tmp["publishtime"]
+	//增量用系统时间，全量(历史)入库时间
+	if p.currentType == "project" {
+		set["pici"] = p.pici
+	} else {
+		set["pici"] = tmp["comeintime"]
+	}
+	set["ids"] = []string{thisinfo.Id}
+	if thisinfo.TopType == "招标" {
+		//if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
+		set["zbtime"] = tmp["publishtime"]
+		p1.Zbtime = qu.Int64All(tmp["publishtime"])
+		//}
+	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
+		set["jgtime"] = tmp["publishtime"]
+		p1.Jgtime = thisinfo.Publishtime
+	}
+	if len(thisinfo.Subscopeclass) > 0 {
+		set["s_subscopeclass"] = strings.Join(thisinfo.Subscopeclass, ",")
+	}
+	if len(thisinfo.Winners) > 0 {
+		set["s_winner"] = strings.Join(thisinfo.Winners, ",")
+		p1.Winners = thisinfo.Winners
+		for _, op := range operators {
+			if BinarySearch(p1.Winners, op) > -1 {
+				set["isOperators"] = false
+			}
+		}
+		if tmp["winnertel"] != nil {
+			set["winnertel"] = tmp["winnertel"]
+		}
+		if tmp["winnerperson"] != nil {
+			set["winnerperson"] = tmp["winnerperson"]
+		}
+	}
+	if thisinfo.HasPackage {
+		set["multipackage"] = 1
+		set["package"] = pkg
+	} else {
+		set["multipackage"] = 0
+	}
+	//项目评审专家
+	if len(thisinfo.ReviewExperts) > 0 {
+		set["review_experts"] = thisinfo.ReviewExperts
+		p1.ReviewExperts = thisinfo.ReviewExperts
+	}
+	//标的物
+	if thisinfo.Purchasing != "" {
+		list := Duplicate(strings.Split(thisinfo.Purchasing, ",")) //标的物 去重 03/03
+		p := strings.Join(qu.ObjArrToStringArr(list), ",")
+		set["purchasing"] = p
+		p1.Purchasing = p
+	}
+	//中标候选人
+	if len(thisinfo.WinnerOrder) > 0 {
+		var list = []string{}
+		for _, v := range thisinfo.WinnerOrder {
+			if BinarySearch(list, qu.ObjToString(v["entname"])) == -1 {
+				list = append(list, qu.ObjToString(v["entname"]))
+			}
+		}
+		set["winnerorder"] = list
+		p1.Winnerorder = list
+	}
+	//项目规模
+	if len(thisinfo.ProjectScale) > 0 {
+		p1.ProjectScale = thisinfo.ProjectScale
+		set["project_scale"] = thisinfo.ProjectScale
+	}
+	//工期时长
+	if thisinfo.ProjectDuration > 0 {
+		p1.ProjectDuration = thisinfo.ProjectDuration
+		set["project_duration"] = thisinfo.ProjectDuration
+	}
+	// 工期单位
+	if thisinfo.ProjectDuration > 0 && len(thisinfo.ProjectTimeUnit) > 0 {
+		p1.ProjectTimeunit = thisinfo.ProjectTimeUnit
+		set["project_timeunit"] = thisinfo.ProjectTimeUnit
+	}
+	//开工日期
+	if thisinfo.ProjectStartDate > 0 {
+		p1.ProjectStartDate = thisinfo.ProjectStartDate
+		set["project_startdate"] = thisinfo.ProjectStartDate
+	}
+	//竣工日期
+	if thisinfo.ProjectCompleteDate > 0 {
+		p1.ProjctCompleteDate = thisinfo.ProjectCompleteDate
+		set["project_completedate"] = thisinfo.ProjectCompleteDate
+	}
+	//付款方式
+	if len(thisinfo.Payway) > 0 {
+		p1.Payway = thisinfo.Payway
+		set["payway"] = thisinfo.Payway
+	}
+	// 履约保证金
+	if tmp["contract_guarantee"] != nil {
+		p1.ContractGuarantee = thisinfo.ContractGuarantee
+		set["contract_guarantee"] = thisinfo.ContractGuarantee
+	}
+	// 投标保证金
+	if tmp["bid_guarantee"] != nil {
+		p1.BidGuarantee = thisinfo.BidGuarantee
+		set["bid_guarantee"] = thisinfo.BidGuarantee
+	}
+	// 资质条件
+	if len(thisinfo.Qualifies) > 0 {
+		var str []string
+		for _, v := range thisinfo.Qualifies {
+			if len(qu.ObjToString(v["key"])) > 0 {
+				if BinarySearch(str, qu.ObjToString(v["key"])) == -1 {
+					str = append(str, qu.ObjToString(v["key"]))
+				}
+			}
+		}
+		if len(str) > 0 {
+			p1.Qualifies = strings.Join(str, ",")
+			set["qualifies"] = strings.Join(str, ",")
+		}
+	}
+	if len(p1.EntIdList) > 0 {
+		set["entidlist"] = p1.EntIdList
+	}
+	p1.InfoFiled = make(map[string]InfoField)
+	infofield := InfoField{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    bs,
+	}
+	p1.InfoFiled[thisinfo.Id] = infofield
+	res := StructToMap(infofield)
+	set["infofield"] = map[string]interface{}{
+		thisinfo.Id: res,
+	}
+	if tmp["budget"] != nil && tmp["budget"] != "" {
+		set["budget"] = thisinfo.Budget
+		p1.Budgettag = 0
+		set["budgettag"] = 0
+	} else {
+		p1.Budgettag = 1
+		set["budgettag"] = 1
+	}
+	if tmp["bidamount"] != nil && tmp["bidamount"] != "" {
+		set["bidamount"] = thisinfo.Bidamount
+		p1.Bidamounttag = 0
+		set["bidamounttag"] = 0
+	} else {
+		p1.Bidamounttag = 1
+		set["bidamounttag"] = 1
+	}
+	if p1.Bidamount > 0 {
+		set["sortprice"] = p1.Bidamount
+	} else if p1.Budget > 0 {
+		set["sortprice"] = p1.Budget
+	}
+	push := p.PushListInfo(tmp, thisinfo.Id)
+	push["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	set["list"] = []bson.M{
+		push,
+	}
+	//p.savePool <- set
+	p.updatePool <- []map[string]interface{}{
+		{
+			"_id": pId,
+		},
+		{
+			"$set": set,
+		},
+	}
+	return pId.Hex(), &p1
+}
+
+//项目中list的信息
+func (p *ProjectTask) PushListInfo(tmp map[string]interface{}, infoid string) bson.M {
+	res := bson.M{
+		"infoid": infoid,
+	}
+	for _, k := range INFOFIELDS {
+		if tmp[k] != nil {
+			res[k] = tmp[k]
+		}
+	}
+	return res
+}
+
+//生成存放在内存中的对象
+func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidtype, bidstatus string, pkg map[string]interface{}) ProjectInfo {
+	p1 := ProjectInfo{
+		Id:            id,
+		Ids:           []string{thisinfo.Id},
+		ProjectName:   thisinfo.ProjectName,
+		ProjectCode:   thisinfo.ProjectCode,
+		ContractCode:  thisinfo.ContractCode,
+		Buyer:         thisinfo.Buyer,
+		Buyerclass:    thisinfo.Buyerclass,
+		Buyerperson:   thisinfo.Buyerperson,
+		Buyertel:      thisinfo.Buyertel,
+		Topscopeclass: thisinfo.Topscopeclass,
+		Subscopeclass: thisinfo.Subscopeclass,
+		Agency:        thisinfo.Agency,
+		Area:          thisinfo.Area,
+		City:          thisinfo.City,
+		District:      thisinfo.District,
+		MPN:           []string{},
+		MPC:           []string{},
+		FirstTime:     thisinfo.Publishtime,
+		LastTime:      thisinfo.Publishtime,
+		Budget:        thisinfo.Budget,
+		Package:       pkg,
+		Bidamount:     thisinfo.Bidamount,
+		Bidstatus:     bidstatus,
+		Bidtype:       bidtype,
+		Winners:       thisinfo.Winners,
+		TagRule:       thisinfo.TagRule,
+		EntIdList:     thisinfo.EntIdList,
+	}
+	if thisinfo.LenPTC > 5 {
+		p1.MPC = append(p1.MPC, thisinfo.PTC)
+	}
+	return p1
+}
+
+//更新项目	全量合并
+func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string, ex int) {
+	if p.currentType != "updateInfo" {
+		if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
+			log.Println("repeat", thisinfo.Id, ",pid", pInfo.Id)
+			return
+		}
+	}
+	set := map[string]interface{}{}
+	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
+	if len(pInfo.Ids) > 30 {
+		//异常标记
+		set["listtag"] = 1
+	}
+
+	//zbtime、lasttime、jgtime
+	pInfo.LastTime = thisinfo.Publishtime
+	set["lasttime"] = thisinfo.Publishtime
+	if thisinfo.TopType == "招标" {
+		if pInfo.Zbtime <= 0 {
+			set["zbtime"] = tmp["publishtime"]
+		}
+		if pInfo.Jgtime > 0 {
+			pInfo.Jgtime = int64(0)
+			set["jgtime"] = int64(0)
+		}
+	} else if thisinfo.TopType == "结果" {
+		if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+			if pInfo.Jgtime > 0 {
+				//jg1 := int64(math.Abs(float64(pInfo.Jgtime - thisinfo.Publishtime)))
+				//公告状态和项目状态同样都是中标或者成交，
+				if thisinfo.SubType == "成交" && pInfo.Bidstatus == "中标" {
+					if p.jgTime < thisinfo.Publishtime {
+						set["jgtime"] = tmp["publishtime"]
+						pInfo.Jgtime = thisinfo.Publishtime
+					}
+					//公告状态和项目状态同样是流标或者废标
+				} else if (thisinfo.SubType == "流标" || thisinfo.SubType == "废标") && (pInfo.Bidstatus == "流标" || pInfo.Bidstatus == "废标") {
+					if p.jgTime < thisinfo.Publishtime {
+						set["jgtime"] = tmp["publishtime"]
+						pInfo.Jgtime = thisinfo.Publishtime
+					}
+				}
+			} else {
+				set["jgtime"] = tmp["publishtime"]
+				pInfo.Jgtime = thisinfo.Publishtime
+			}
+		}
+	} else if thisinfo.SubType == "合同" {
+		if pInfo.Bidstatus == "中标" || pInfo.Bidstatus == "成交" || pInfo.Bidstatus == "" {
+			//中标、成交不更新jgtime
+		} else {
+			set["jgtime"] = tmp["publishtime"]
+			pInfo.Jgtime = thisinfo.Publishtime
+		}
+	}
+	if thisinfo.Bidopentime > pInfo.Bidopentime {
+		pInfo.Bidopentime = thisinfo.Bidopentime
+		set["bidopentime"] = pInfo.Bidopentime
+	}
+
+	bt := qu.ObjToString(tmp["toptype"])
+	bs := qu.ObjToString(tmp["subtype"])
+	p.mapBidLock.Lock()
+	if bt == "招标" {
+		//招标状态，更新projectscope
+		if tmp["projectscope"] != nil {
+			set["projectscope"] = qu.ObjToString(tmp["projectscope"])
+		}
+		set["bidstatus"] = bt
+		pInfo.Bidstatus = bt
+		if bidtype[bs] != "" {
+			set["bidtype"] = bidtype[bs]
+			pInfo.Bidtype = bidtype[bs]
+		} else {
+			set["bidtype"] = "招标"
+			pInfo.Bidtype = "招标"
+		}
+	} else {
+		if bidstatus[bs] != "" {
+			set["bidstatus"] = thisinfo.SubType
+			pInfo.Bidstatus = thisinfo.SubType
+		} else if thisinfo.Infoformat == 2 {
+			set["bidstatus"] = "拟建"
+			pInfo.Bidstatus = "拟建"
+		} else if bs == "" && bt == "结果" {
+			if pInfo.Bidstatus == "招标" {
+				set["bidstatus"] = ""
+				pInfo.Bidstatus = ""
+			}
+		} else {
+			set["bidstatus"] = "其它"
+			pInfo.Bidstatus = "其它"
+		}
+	}
+	p.mapBidLock.Unlock()
+
+	//异常标记
+	if ex > 0 {
+		set["exception"] = ex
+	}
+	//3\4\5--省、市、县
+	//if thisinfo.Area != "全国" {
+	//	if pInfo.Area == "全国" {
+	//		pInfo.Area = thisinfo.Area
+	//		set["area"] = thisinfo.Area
+	//	} else if pInfo.Area != thisinfo.Area {
+	//		//xt = false
+	//	}
+	//	if pInfo.City == "" && thisinfo.City != "" {
+	//		pInfo.City = thisinfo.City
+	//		set["city"] = thisinfo.City
+	//	} else if pInfo.City != thisinfo.City {
+	//		//xt = false
+	//	}
+	//	if thisinfo.District != "" && pInfo.District == "" {
+	//		pInfo.District = thisinfo.District
+	//		set["district"] = thisinfo.District
+	//	}
+	//}
+	//相同城市的公告才会合并到一起(全国列外)
+	if thisinfo.Area != "全国" {
+		pInfo.Area = thisinfo.Area
+		set["area"] = thisinfo.Area
+		pInfo.City = thisinfo.City
+		set["city"] = thisinfo.City
+		if thisinfo.District != "" {
+			pInfo.District = thisinfo.District
+			set["district"] = thisinfo.District
+		}
+	}
+	//6--项目名称
+	if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
+		pInfo.ProjectName = thisinfo.ProjectName
+		set["projectname"] = thisinfo.ProjectName
+	}
+	//7--项目编号
+	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+		pInfo.ProjectCode = thisinfo.ProjectCode
+		set["projectcode"] = thisinfo.ProjectCode
+	}
+	//7--采购单位
+	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
+		pInfo.Buyer = thisinfo.Buyer
+		set["buyer"] = thisinfo.Buyer
+
+		pInfo.Buyerclass = thisinfo.Buyerclass
+		set["buyerclass"] = thisinfo.Buyerclass
+	}
+	if pInfo.Buyer == "" {
+		set["buyerclass"] = ""
+	}
+	//采购单位联系人
+	if thisinfo.Buyerperson != "" {
+		pInfo.Buyerperson = thisinfo.Buyerperson
+		set["buyerperson"] = pInfo.Buyerperson
+	} else {
+		pInfo.Buyerperson = ""
+		set["buyerperson"] = ""
+	}
+	//采购单位電話
+	if thisinfo.Buyertel != "" {
+		pInfo.Buyertel = thisinfo.Buyertel
+		set["buyertel"] = pInfo.Buyertel
+	} else {
+		pInfo.Buyertel = ""
+		set["buyertel"] = ""
+	}
+	if thisinfo.ContractCode != "" {
+		if pInfo.ContractCode == "" {
+			set["contractcode"] = thisinfo.ContractCode
+		} else {
+			list := strings.Split(pInfo.ContractCode, ",")
+			if BinarySearch(list, thisinfo.ContractCode) == -1 {
+				list = append(list, thisinfo.ContractCode)
+				sort.Strings(list)
+			}
+			set["contractcode"] = strings.Join(list, ",")
+		}
+	}
+
+	//8--代理机构
+	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
+		pInfo.Agency = thisinfo.Agency
+		set["agency"] = thisinfo.Agency
+	}
+
+	if len(thisinfo.Topscopeclass) > 0 {
+		sort.Strings(pInfo.Topscopeclass)
+		for _, k := range thisinfo.Topscopeclass {
+			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
+				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
+				sort.Strings(pInfo.Topscopeclass)
+			}
+		}
+		set["topscopeclass"] = pInfo.Topscopeclass
+	}
+
+	//项目评审专家
+	if len(thisinfo.ReviewExperts) > 0 {
+		set["review_experts"] = thisinfo.ReviewExperts
+		pInfo.ReviewExperts = thisinfo.ReviewExperts
+	}
+	if thisinfo.Purchasing != "" {
+		if pInfo.Purchasing == "" {
+			list := Duplicate(strings.Split(thisinfo.Purchasing, ",")) //标的物 去重 03/03
+			p := strings.Join(qu.ObjArrToStringArr(list), ",")
+			pInfo.Purchasing = p
+			set["purchasing"] = p
+		} else {
+			list := strings.Split(pInfo.Purchasing, ",")
+			list = qu.ObjArrToStringArr(Duplicate(list))
+			for _, k := range list {
+				if BinarySearch(strings.Split(thisinfo.Purchasing, ","), k) == -1 {
+					list = append(list, k)
+					sort.Strings(list)
+				}
+			}
+			set["purchasing"] = strings.Join(list, ",")
+		}
+	}
+
+	//中标候选人
+	if len(thisinfo.WinnerOrder) > 0 {
+		var list = []string{}
+		for _, v := range thisinfo.WinnerOrder {
+			if BinarySearch(list, qu.ObjToString(v["entname"])) == -1 {
+				list = append(list, qu.ObjToString(v["entname"]))
+			}
+		}
+		set["winnerorder"] = list
+		pInfo.Winnerorder = list
+	}
+
+	if len(thisinfo.Subscopeclass) > 0 {
+		sort.Strings(pInfo.Subscopeclass)
+		for _, k := range thisinfo.Subscopeclass {
+			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
+				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
+				sort.Strings(pInfo.Subscopeclass)
+			}
+		}
+		set["subscopeclass"] = pInfo.Subscopeclass
+		set["s_subscopeclass"] = strings.Join(pInfo.Subscopeclass, ",")
+	}
+
+	if len(thisinfo.Winners) > 0 {
+		if len(pInfo.Winners) <= 0 {
+			set["winner"] = qu.ObjToString(tmp["winner"])
+		}
+		sort.Strings(pInfo.Winners)
+		for _, k := range thisinfo.Winners {
+			if !pInfo.IsOperators {
+				for _, op := range operators {
+					if strings.Contains(k, op) {
+						pInfo.IsOperators = true
+						set["isOperators"] = true
+						break
+					}
+				}
+			}
+			if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+				if BinarySearch(pInfo.Winners, k) != -1 {
+					deleteSlice(pInfo.Winners, k, "")
+				}
+			} else {
+				if BinarySearch(pInfo.Winners, k) == -1 {
+					pInfo.Winners = append(pInfo.Winners, k)
+				}
+			}
+		}
+		set["s_winner"] = strings.Join(pInfo.Winners, ",")
+		if tmp["winnertel"] != nil {
+			set["winnertel"] = tmp["winnertel"]
+		}
+		if tmp["winnerperson"] != nil {
+			set["winnerperson"] = tmp["winnerperson"]
+		}
+	}
+	// 公告标签tag_rule
+	if thisinfo.TagRule != "" {
+		if pInfo.TagRule != "" {
+			tags := strings.Split(pInfo.TagRule, ",")
+			for _, s := range strings.Split(thisinfo.TagRule, ",") {
+				if BinarySearch(tags, s) == -1 {
+					tags = append(tags, s)
+				}
+			}
+			set["tag_rule"] = strings.Join(tags, ",")
+		} else {
+			set["tag_rule"] = thisinfo.TagRule
+		}
+	}
+	if tmp["buyertagname"] != nil {
+		set["buyertagname"] = tmp["buyertagname"]
+	}
+	//项目规模
+	if len(thisinfo.ProjectScale) > 0 {
+		pInfo.ProjectScale = thisinfo.ProjectScale
+		set["project_scale"] = thisinfo.ProjectScale
+	}
+	//工期时长
+	if thisinfo.ProjectDuration > 0 {
+		pInfo.ProjectDuration = thisinfo.ProjectDuration
+		set["project_duration"] = thisinfo.ProjectDuration
+	}
+	// 工期单位
+	if thisinfo.ProjectDuration > 0 && len(thisinfo.ProjectTimeUnit) > 0 {
+		pInfo.ProjectTimeunit = thisinfo.ProjectTimeUnit
+		set["project_timeunit"] = thisinfo.ProjectTimeUnit
+	}
+	//开工日期
+	if thisinfo.ProjectStartDate > 0 {
+		if pInfo.ProjectStartDate > 0 {
+			if pInfo.ProjectStartDate < thisinfo.ProjectStartDate {
+				pInfo.ProjectStartDate = thisinfo.ProjectStartDate
+				set["project_startdate"] = thisinfo.ProjectStartDate
+			}
+		} else {
+			pInfo.ProjectStartDate = thisinfo.ProjectStartDate
+			set["project_startdate"] = thisinfo.ProjectStartDate
+		}
+	}
+	//竣工日期
+	//if thisinfo.ProjectCompleteDate > 0 {
+	//	if pInfo.ProjctCompleteDate > 0 {
+	//		if pInfo.ProjctCompleteDate < thisinfo.ProjectCompleteDate {
+	//			pInfo.ProjctCompleteDate = thisinfo.ProjectCompleteDate
+	//			set["project_completedate"] = thisinfo.ProjectCompleteDate
+	//		}
+	//	} else {
+	//		pInfo.ProjctCompleteDate = thisinfo.ProjectCompleteDate
+	//		set["project_completedate"] = thisinfo.ProjectCompleteDate
+	//	}
+	//}
+	// 付款方式
+	if len(thisinfo.Payway) > 0 {
+		pInfo.Payway = thisinfo.Payway
+		set["payway"] = thisinfo.Payway
+	}
+	// 履约保证金
+	if tmp["contract_guarantee"] != nil {
+		pInfo.ContractGuarantee = thisinfo.ContractGuarantee
+		set["contract_guarantee"] = thisinfo.ContractGuarantee
+	}
+	// 投标保证金
+	if tmp["bid_guarantee"] != nil {
+		pInfo.BidGuarantee = thisinfo.BidGuarantee
+		set["bid_guarantee"] = thisinfo.BidGuarantee
+	}
+	// 资质条件
+	if len(thisinfo.Qualifies) > 0 {
+		var str []string
+		if len(pInfo.Qualifies) > 0 {
+			str = append(str, strings.Split(pInfo.Qualifies, ",")...)
+		}
+		for _, v := range thisinfo.Qualifies {
+			if len(qu.ObjToString(v["key"])) > 0 {
+				if BinarySearch(str, qu.ObjToString(v["key"])) == -1 {
+					str = append(str, qu.ObjToString(v["key"]))
+				}
+			}
+		}
+		if len(str) > 0 {
+			pInfo.Qualifies = strings.Join(str, ",")
+			set["qualifies"] = strings.Join(str, ",")
+		}
+	}
+	if len(thisinfo.EntIdList) > 0 {
+		for _, v := range thisinfo.EntIdList {
+			if BinarySearch(pInfo.EntIdList, v) == -1 {
+				pInfo.EntIdList = append(pInfo.EntIdList, v)
+			}
+		}
+		set["entidlist"] = pInfo.EntIdList
+	}
+
+	if thisinfo.HasPackage { //多包处理
+		set["multipackage"] = 1
+		pkg := PackageFormat(thisinfo, pInfo)
+		pInfo.Package = pkg
+		set["package"] = pInfo.Package
+	}
+	//处理多包后，计算预算金额、中标金额
+	CountAmount(pInfo, thisinfo, tmp)
+	if pInfo.Budget >= 0 && pInfo.Budgettag != 1 {
+		set["budget"] = pInfo.Budget
+		set["budgettag"] = 0
+	}
+	if pInfo.Bidamount >= 0 && pInfo.Bidamounttag != 1 {
+		set["bidamount"] = pInfo.Bidamount
+		set["bidamounttag"] = 0
+	}
+	if pInfo.Bidamount >= pInfo.Budget {
+		set["sortprice"] = pInfo.Bidamount
+	} else if pInfo.Budget >= pInfo.Bidamount {
+		set["sortprice"] = pInfo.Budget
+	}
+
+	infofield := InfoField{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    bs,
+	}
+	copyMap := Copy(pInfo.InfoFiled).(map[string]InfoField)
+	copyMap[thisinfo.Id] = infofield
+	tmpMap := make(map[string]interface{})
+	for k, v := range copyMap {
+		tmpMap[k] = StructToMap(v)
+	}
+	tmpMap[thisinfo.Id] = StructToMap(infofield)
+	pInfo.InfoFiled = copyMap
+	set["infofield"] = tmpMap
+
+	set["mpn"] = pInfo.MPN
+	set["mpc"] = pInfo.MPC
+	if p.currentType == "project" {
+		set["pici"] = p.pici
+	} else {
+		set["pici"] = tmp["comeintime"]
+	}
+	update := map[string]interface{}{}
+	if len(set) > 0 {
+		update["$set"] = set
+	}
+	push := p.PushListInfo(tmp, thisinfo.Id)
+	push["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	push["compareStr"] = comStr
+	push["resVal"] = pInfo.resVal
+	push["pjVal"] = pInfo.pjVal
+	update["$push"] = map[string]interface{}{
+		"list": push,
+		"ids":  thisinfo.Id,
+	}
+	//clearMap := map[string]interface{}{}
+	//ClearData(clearMap, set)
+	//if len(clearMap) > 0 {
+	//	update["$unset"] = clearMap
+	//}
+	if len(update) > 0 {
+		updateInfo := []map[string]interface{}{
+			{
+				"_id": pInfo.Id,
+			},
+			update,
+		}
+		p.updatePool <- updateInfo
+	}
+}
+
+/**
+ *	更新项目时，项目状态的处理
+ *	返回是否新增项目，异常标记
+ *	1、新增项目时，招标信息的状态(toptype)不是招标、拟建、预告	异常：1
+ *	   异常1是在项目新建的时候才会产生
+ *	3、项目合并时，项目状态是”流标“/”废标“，招标信息状态不是”招标“		异常：2
+ *	4、项目合并时，项目状态是”合同“/”其它“，招标信息类型是”结果“		异常：3
+ */
+func (p *ProjectTask) CompareStatus(project *ProjectInfo, info *Info) (bool, int) {
+	if info.TopType == "拟建" || info.TopType == "预告" || info.TopType == "招标" {
+		if project.Bidstatus == "拟建" || project.Bidstatus == "预告" || project.Bidstatus == "招标" {
+			return false, 0
+		} else if project.Bidstatus == "废标" || project.Bidstatus == "流标" {
+			return false, 0
+		} else {
+			return true, 0
+		}
+	} else if info.TopType == "结果" {
+		if project.Bidstatus == "拟建" || project.Bidstatus == "预告" || project.Bidstatus == "招标" {
+			return false, 0
+		} else if project.Bidstatus == info.SubType {
+			//状态一样，根据发布时间判断是否合并
+			if (info.Publishtime - project.LastTime) > p.statusTime {
+				return true, 0
+			} else {
+				return false, 0
+			}
+		} else if project.Bidstatus == "成交" && info.SubType == "中标" {
+			return true, 0
+		} else if project.Bidstatus == "流标" || project.Bidstatus == "废标" {
+			return false, 2
+		} else if project.Bidstatus == "合同" || project.Bidstatus == "其它" {
+			return false, 3
+		} else {
+			return false, 0
+		}
+	} else {
+		return false, 0
+	}
+}
+
+/*
+ *	对比地区(省、市)，存在且不同，不能合并
+ *	返回是否新建项目
+ */
+func ComparePlace(project *ProjectInfo, info *Info) bool {
+	if info.Area == "全国" || info.Area == "" {
+		return false
+	}
+	if project.Area == "全国" || project.Area == "" {
+		return false
+	}
+	if info.Area == project.Area {
+		if info.City == "" || project.City == "" {
+			return false
+		} else if info.City == project.City {
+			return false
+		}
+	} else {
+		return true
+	}
+	return true
+}
+
+func packageEle(map1 map[string]interface{}, id string) map[string]interface{} {
+	p2 := map[string]interface{}{}
+	for _, k := range PackageEle {
+		if map1[k] != nil {
+			p2[k] = map1[k]
+		}
+		infoid := p2["infoid"]
+		if infoid == nil {
+			p2["infoid"] = id
+		}
+	}
+	return p2
+}
+
+func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
+	p1 := map[string]interface{}{}
+	if project != nil && project.Package != nil && len(project.Package) > 0 {
+		packageCopy := Copy(project.Package).(map[string]interface{})
+		p1 = packageCopy
+		for k, v := range info.Package {
+			if v1, ok := v.(map[string]interface{}); ok {
+				v2 := map[string]interface{}{}
+				v2 = packageEle(v1, info.Id)
+				if v2["bidstatus"] == nil {
+					v2["bidstatus"] = info.SubType
+				}
+				addFlag := false
+				for k1, v3 := range p1 {
+					if v4, ok := v3.([]map[string]interface{}); ok {
+						//if qu.ObjToString(v4[0]["origin"]) == qu.ObjToString(v2["origin"]) && qu.ObjToString(v4[0]["name"]) == qu.ObjToString(v2["name"]) {
+						if k1 == k {
+							v4 = append(v4, v2)
+							p1[k1] = v4
+							addFlag = true
+							break
+						}
+					}
+				}
+				if !addFlag {
+					p1[k] = []map[string]interface{}{v2}
+				}
+			}
+		}
+	} else {
+		for k, v := range info.Package {
+			v1, _ := v.(map[string]interface{})
+			p2 := map[string]interface{}{}
+			p2 = packageEle(v1, info.Id)
+			if p2["bidstatus"] == nil {
+				p2["bidstatus"] = info.SubType
+			}
+			p1[k] = []map[string]interface{}{p2}
+		}
+	}
+	return p1
+}
+
+// 计算预算(budget)、中标金额(bidamount)
+func CountAmount(project *ProjectInfo, info *Info, tmp map[string]interface{}) {
+	if info.HasPackage {
+		budget := 0.0
+		for _, v := range project.Package {
+			v1, _ := v.([]map[string]interface{})
+			for _, v2 := range v1 {
+				if v2["budget"] != nil {
+					b1 := qu.Float64All(v2["budget"])
+					if b1 > 0 {
+						budget = budget + b1
+						break
+					}
+				} else {
+					project.Budgettag = 1
+				}
+			}
+		}
+		if budget > 0 {
+			project.Budget = budget
+			project.Budgettag = 0
+		} else if budget == 0 && info.Budget > 0 {
+			project.Budget = info.Budget
+			project.Budgettag = 0
+		}
+	} else {
+		//招标没有多包
+		k := KeyPackage.FindStringSubmatch(info.ProjectName)
+		if len(k) > 0 {
+			//招标是单包
+			if len(project.Package) > 0 {
+				//项目有多包
+				flag := false
+				for _, v := range project.Package {
+					v1, _ := v.([]map[string]interface{})
+					if len(v1) > 0 && v1[0]["name"] == info.ProjectName {
+						flag = true
+					}
+				}
+				if !flag {
+					project.Budget = project.Budget + info.Budget
+					project.Budgettag = 0
+				}
+			} else {
+				//项目没有多包
+				if info.Budget > 0 {
+					project.Budget = project.Budget + info.Budget
+					project.Budgettag = 0
+				} else if info.Budget == 0 && tmp["budget"] != nil {
+					project.Budgettag = 0
+				}
+			}
+		} else {
+			if info.Budget > 0 && project.Budget < info.Budget {
+				project.Budget = info.Budget
+				project.Budgettag = 0
+			}
+		}
+	}
+	if info.SubType == "中标" || info.SubType == "成交" || info.SubType == "合同" {
+		if info.HasPackage {
+			bidamount := 0.0
+			for _, v := range project.Package {
+				v1, _ := v.([]map[string]interface{})
+				for _, v2 := range v1 {
+					b1 := qu.Float64All(v2["bidamount"])
+					if b1 > 0 {
+						bidamount = bidamount + b1
+						break
+					}
+				}
+			}
+			if bidamount > 0 {
+				project.Bidamount = bidamount
+				project.Bidamounttag = 0
+			} else if bidamount == 0 && info.Bidamount > 0 {
+				project.Bidamount = info.Bidamount
+				project.Bidamounttag = 0
+			}
+		} else {
+			//招标没有多包
+			k := KeyPackage.FindStringSubmatch(info.ProjectName)
+			if len(k) > 0 {
+				//招标是单包
+				if len(project.Package) > 0 {
+					//项目有多包
+					flag := false
+					for _, v := range project.Package {
+						v1, _ := v.([]map[string]interface{})
+						if len(v1) > 0 && v1[0]["name"] == info.ProjectName {
+							flag = true
+						}
+					}
+					if !flag {
+						project.Bidamount = project.Bidamount + info.Bidamount
+						project.Bidamounttag = 0
+					}
+				} else {
+					//项目没有多包
+					if info.Bidamount > 0 {
+						project.Bidamount = project.Bidamount + info.Bidamount
+						project.Bidamounttag = 0
+					} else if info.Bidamount == 0 && tmp["bidamount"] != nil {
+						project.Bidamounttag = 0
+					} else {
+						project.Bidamounttag = 1
+					}
+				}
+			} else {
+				if info.SubType == "中标" || info.SubType == "成交" {
+					if info.Bidamount > 0 {
+						project.Bidamount = info.Bidamount
+						project.Bidamounttag = 0
+					} else {
+						flag := false
+						if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
+							for _, res := range project.InfoFiled {
+								if res.ContractCode != "" && res.ContractCode == info.ContractCode {
+									flag = true
+									break
+								}
+								if res.Bidamount == project.Bidamount {
+									flag = true
+									break
+								}
+							}
+							if !flag {
+								project.Bidamount = project.Bidamount + info.Bidamount
+								project.Bidamounttag = 0
+							} else {
+								if info.Budget > 0 && project.Bidamount > info.Bidamount {
+									project.Bidamount = info.Bidamount
+									project.Bidamounttag = 0
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	} else {
+		project.Bidamounttag = 1
+	}
+
+}
+
+//结构体转map
+func StructToMap(filed InfoField) map[string]interface{} {
+	//先转json
+	result, err := json.Marshal(filed)
+	if err != nil {
+		return nil
+	}
+	//json转map
+	res := make(map[string]interface{})
+	err = json.Unmarshal(result, &res)
+	return res
+}
+
+func ClearData(clearMap, tmp map[string]interface{}) {
+	for k, v := range tmp {
+		if v == "" {
+			clearMap[k] = ""
+		}
+	}
+}
+
+func IsCreatePro(info *Info) (bol bool) {
+	bol = true
+	if info.SubType == "" || info.SubType == "变更" || info.SubType == "验收" || info.SubType == "违规" ||
+		info.SubType == "结果变更" || info.SubType == "其它" {
+		if info.ProjectName == "" && info.ProjectCode == "" {
+			bol = false
+		} else if info.ProjectName == "" && info.Buyer == "" {
+			bol = false
+		} else if info.ProjectCode == "" && info.Buyer == "" {
+			bol = false
+		}
+	}
+	return bol
+}
+
+func (p *ProjectTask) GetBidTypeAndBidStatus(info *Info) (string, string) {
+	p.mapBidLock.Lock()
+	defer p.mapBidLock.Unlock()
+	typeStr := bidtype[info.TopType]
+	statusStr := bidstatus[info.SubType]
+	if info.Infoformat == 2 || info.SubType == "拟建" {
+		statusStr = "拟建"
+		typeStr = ""
+	} else {
+		if bidtype[typeStr] == "" {
+			typeStr = "招标"
+		}
+		if typeStr == "招标" {
+			statusStr = typeStr
+		} else {
+			if statusStr == "" {
+				statusStr = "其它"
+			}
+		}
+	}
+	return typeStr, statusStr
+
+}

project_data/project_tool.go

@@ -0,0 +1,97 @@
+package main
+
+//项目中的字段
+var FIELDS = []string{
+	"area",
+	"city",
+	"district",
+	"projectname",
+	"projectcode",
+	"buyer",
+	"buyerclass",
+	"buyerperson",
+	"buyertel",
+	"winner",
+	"agency",
+	"topscopeclass",
+	"subscopeclass",
+	"package",
+}
+
+//招标信息字段
+var INFOFIELDS = []string{
+	"projectname",
+	"projectcode",
+	"projectscope",
+	"contractcode",
+	"title",
+	"href",
+	"publishtime",
+	"comeintime",
+	"bidopentime",
+	"toptype",
+	"subtype",
+	"buyer",
+	"buyerclass",
+	"agency",
+	"winner",
+	"winnertel",
+	"winnerperson",
+	"budget",
+	"bidamount",
+	"topscopeclass",
+	"subscopclass",
+	"infoformat",
+	"buyerperson",
+	"buyertel",
+	"area",
+	"city",
+	"district",
+	"spidercode",
+	"site",
+	"review_experts",
+	"purchasing",
+	"project_scale",
+	"project_duration",
+	"project_timeunit",
+	"project_startdate",
+	"project_completedate",
+	"payway",
+	"contract_guarantee",
+	"bid_guarantee",
+	"qualifies",
+	//"entidlist",
+	"tag_rule",
+}
+
+// 包信息
+var PackageEle = []string{
+	"origin",
+	"name",
+	//"text",
+	"budget",
+	"winner",
+	"bidamount",
+	"bidamounttype",
+	"currency",
+	"bidstatus",
+}
+
+var bidstatus = map[string]string{
+	"采购意向": "采购意向",
+	"预告":   "预告",
+	"中标":   "中标",
+	"成交":   "成交",
+	"废标":   "废标",
+	"流标":   "流标",
+	"合同":   "合同",
+}
+
+var bidtype = map[string]string{
+	"招标": "招标",
+	"邀标": "邀标",
+	"询价": "询价",
+	"单一": "单一",
+	"竞价": "竞价",
+	"竞谈": "竞谈",
+}

project_data/task.go

@@ -0,0 +1,398 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/robfig/cron"
+	"log"
+	"mongodb"
+	"qfw/util"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+)
+
+/**
+任务入口
+全量、增量合并
+更新、插入，内存清理
+转换成info对象
+**/
+
+var PreRegexp = map[string][]*regexp.Regexp{}
+var BackRegexp = map[string][]*regexp.Regexp{}
+var BackRepRegexp = map[string][]RegexpInfo{}
+var BlackRegexp = map[string][]*regexp.Regexp{}
+
+var (
+	//从标题获取项目编号
+	titleGetPc  = regexp.MustCompile("^([-0-9a-zA-Z第号采招政询电审竞#]{8,}[-0-9a-zA-Z#]+)")
+	titleGetPc1 = regexp.MustCompile("[\\[【(（](.{0,6}(编号|编码|项号|包号|代码|标段?号)[:：为])?([-0-9a-zA-Z第号采招政询电审竞#]{5,}([\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+[\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+)?)[\\]】)）]")
+	titleGetPc2 = regexp.MustCompile("([-0-9a-zA-Z第号采政招询电审竞#]{8,}[-0-9a-zA-Z#]+)(.{0,5}公告)?$")
+	//项目编号过滤
+	pcReplace = regexp.MustCompile("([\\[【(（〖〔《｛﹝{](重|第?[二三四再]次.{0,4})[\\]】)）〗〕》｝﹞}])$|[\\[\\]【】()（）〖〗〔〕《》｛｝﹝﹞－；{}– 　]+|(号|重|第?[二三四五再]次(招标)?)$|[ _＿]+|((采购)?项目|采购(项目)?)$")
+	//项目编号只是数字或只是字母4个以下
+	StrOrNum = regexp.MustCompile("^[0-9_-]{1,4}$|^[a-zA-Z_-]{1,4}$")
+	//纯数字或纯字母
+	StrOrNum2 = regexp.MustCompile("^[0-9_-]+$|^[a-zA-Z_-]+$")
+	//含分包词，招标未识别分包  合并到一个项目
+	KeyPackage = regexp.MustCompile("[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}(包|段)|(包|段)[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}")
+)
+
+type RegexpInfo struct {
+	regs   *regexp.Regexp
+	repstr string
+}
+
+func NewPT() *ProjectTask {
+	p := &ProjectTask{
+		InitMinTime: int64(1325347200),
+		name:        "全/增量对象",
+		thread:      Thread,
+		updatePool:  make(chan []map[string]interface{}, 5000),
+		//savePool:    make(chan map[string]interface{}, 2000),
+		wg:        sync.WaitGroup{},
+		AllIdsMap: make(map[string]*ID, 5000000),
+		mapPb:     make(map[string]*Key, 1500000),
+		mapPn:     make(map[string]*Key, 5000000),
+		mapPc:     make(map[string]*Key, 5000000),
+		saveSize:  100,
+
+		//saveSign:   make(chan bool, 1),
+		//updateSign: make(chan bool, 1),
+		coll:       ProjectColl,
+		validTime:  int64(util.IntAllDef(Sysconfig["validdays"], 150) * 86400),
+		statusTime: int64(util.IntAllDef(Sysconfig["statusdays"], 15) * 86400),
+		jgTime:     int64(util.IntAllDef(7, 7) * 86400),
+	}
+	return p
+}
+
+var P_QL *ProjectTask
+var sp = make(chan bool, 5)
+
+//初始化全量合并对象
+func init() {
+	P_QL = NewPT()
+	go P_QL.updateAllQueue()
+	//go P_QL.clearMem()
+}
+
+func (p *ProjectTask) updateAllQueue() {
+	arru := make([][]map[string]interface{}, p.saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-p.updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == p.saveSize {
+				sp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-sp
+					}()
+					MongoTool.UpSertBulk(p.coll, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, p.saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				sp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-sp
+					}()
+					MongoTool.UpSertBulk(p.coll, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, p.saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+//项目合并内存更新
+func (p *ProjectTask) clearMem() {
+	c := cron.New()
+	//在内存中保留最近6个月的信息
+	//跑全量时每5分钟跑一次，跑增量时400分钟跑一次
+	_ = c.AddFunc("50 0/5 * * * *", func() {
+		if (p.currentType == "ql" && SingleClear == 0) || p.clearContimes >= 80 {
+			SingleClear = 1
+			//跳过的次数清零
+			p.clearContimes = 0
+			//信息进入查找对比全局锁
+			p.findLock.Lock()
+			//defer p.findLock.Unlock()
+			//合并进行的任务都完成
+			p.wg.Wait()
+			//遍历id
+			//所有内存中的项目信息
+			p.AllIdsMapLock.Lock()
+			log.Println("清除开始")
+			//清除计数
+			clearNum := 0
+			for k, v := range p.AllIdsMap {
+				if p.currentTime-v.P.LastTime > p.validTime {
+					clearNum++
+					//删除id的map
+					delete(p.AllIdsMap, k)
+					//删除pb
+					if v.P.Buyer != "" {
+						ids := p.mapPb[v.P.Buyer]
+						if ids != nil {
+							ids.Lock.Lock()
+							ids.Arr = deleteSlice(ids.Arr, k, "pb")
+							if len(ids.Arr) == 0 {
+								delete(p.mapPb, v.P.Buyer)
+							}
+							ids.Lock.Unlock()
+						}
+					}
+					//删除mapPn
+					for _, vn := range append([]string{v.P.ProjectName}, v.P.MPN...) {
+						if vn != "" {
+							ids := p.mapPn[vn]
+							if ids != nil {
+								ids.Lock.Lock()
+								ids.Arr = deleteSlice(ids.Arr, k, "pn")
+								if len(ids.Arr) == 0 {
+									delete(p.mapPn, vn)
+								}
+								ids.Lock.Unlock()
+							}
+						}
+					}
+					//删除mapPc
+					for _, vn := range append([]string{v.P.ProjectCode}, v.P.MPC...) {
+						if vn != "" {
+							ids := p.mapPc[vn]
+							if ids != nil {
+								ids.Lock.Lock()
+								ids.Arr = deleteSlice(ids.Arr, k, "pc")
+								if len(ids.Arr) == 0 {
+									delete(p.mapPc, vn)
+								}
+								ids.Lock.Unlock()
+							}
+						}
+					}
+					v = nil
+				}
+			}
+			p.AllIdsMapLock.Unlock()
+			p.findLock.Unlock()
+			SingleClear = 0
+			log.Println("清除完成:", clearNum, len(p.AllIdsMap), len(p.mapPn), len(p.mapPc), len(p.mapPb))
+		} else {
+			p.clearContimes++
+		}
+	})
+	c.Start()
+}
+
+//全量合并
+func (p *ProjectTask) taskQl() {
+	defer util.Catch()
+	p.thread = util.IntAllDef(Thread, 4)
+	p.enter(MongoTool.DbName, ExtractColl, nil)
+
+}
+
+//增量合并
+func (p *ProjectTask) taskZl(udpInfo map[string]interface{}) {
+	defer util.Catch()
+	//1、检查pubilshtime索引
+	db, _ := udpInfo["db"].(string)
+	if db == "" {
+		db = MongoTool.DbName
+	}
+	coll, _ := udpInfo["coll"].(string)
+	if coll == "" {
+		coll = ExtractColl
+	}
+	thread := util.IntAllDef(Thread, 4)
+	if thread > 0 {
+		p.thread = thread
+	}
+	//开始id和结束id
+	q, _ := udpInfo["query"].(map[string]interface{})
+	gtid := udpInfo["gtid"].(string)
+	lteid := udpInfo["lteid"].(string)
+	q = map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  mongodb.StringTOBsonId(gtid),
+			"$lte": mongodb.StringTOBsonId(lteid),
+		},
+	}
+	p.enter(db, coll, q)
+	if udpInfo["stop"] == nil {
+		for i := 0; i < 5; i++ {
+			sp <- true
+		}
+		for i := 0; i < 5; i++ {
+			<-sp
+		}
+		log.Println("保存完成", p.pici)
+	}
+}
+
+func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
+	defer util.Catch()
+	defer func() {
+		p.Brun = false
+	}()
+	p.Brun = true
+	count := 0
+
+	q = map[string]interface{}{"updatetime": map[string]interface{}{"$gt": 1643262000}}
+	util.Debug("start project", q, p.pici)
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+
+	fields := map[string]interface{}{"kvtext": 0, "repeat_reason": 0}
+	ms := sess.DB(db).C(coll).Find(q).Select(fields).Sort("publishtime")
+	query := ms.Iter()
+	var lastid interface{}
+
+	for tmp := make(map[string]interface{}); query.Next(&tmp); count++ {
+		if count%2000 == 0 {
+			util.Debug("current ---", count, lastid)
+		}
+		if util.ObjToString(tmp["useable"]) == "0" {
+			continue
+		}
+		lastid = tmp["_id"]
+		info := ParseInfo(tmp)
+		p.currentTime = info.Publishtime
+		p.startProjectMerge(info, tmp)
+	}
+
+	log.Println("over...", count)
+
+}
+
+func ParseInfo(tmp map[string]interface{}) (info *Info) {
+	bys, _ := json.Marshal(tmp)
+	var thisinfo *Info
+	_ = json.Unmarshal(bys, &thisinfo)
+	if thisinfo == nil {
+		return nil
+	}
+
+	if len(thisinfo.Topscopeclass) == 0 {
+		thisinfo.Topscopeclass = []string{}
+	}
+	if len(thisinfo.Subscopeclass) == 0 {
+		thisinfo.Subscopeclass = []string{}
+	}
+	if thisinfo.SubType == "" {
+		thisinfo.SubType = util.ObjToString(tmp["bidstatus"])
+	}
+
+	//从标题中查找项目编号
+	res := titleGetPc.FindStringSubmatch(thisinfo.Title)
+	if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
+		thisinfo.PTC = res[1]
+	} else {
+		res = titleGetPc1.FindStringSubmatch(thisinfo.Title)
+		if len(res) > 3 && len(res[3]) > 6 && thisinfo.ProjectCode != res[3] && !numCheckPc.MatchString(res[3]) && !_zimureg1.MatchString(res[3]) {
+			thisinfo.PTC = res[3]
+		} else {
+			res = titleGetPc2.FindStringSubmatch(thisinfo.Title)
+			if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
+				thisinfo.PTC = res[1]
+			}
+		}
+	}
+
+	if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
+		//thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
+		//if thisinfo.ProjectName != "" {
+		thisinfo.pnbval++
+		//}
+	}
+
+	if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
+		if thisinfo.ProjectCode != "" {
+			//thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
+			if thisinfo.pnbval == 0 && len([]rune(thisinfo.ProjectCode)) < 5 {
+				thisinfo.ProjectCode = StrOrNum.ReplaceAllString(thisinfo.ProjectCode, "")
+			}
+		} else {
+			//thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
+			if thisinfo.pnbval == 0 && len([]rune(thisinfo.PTC)) < 5 {
+				thisinfo.PTC = StrOrNum.ReplaceAllString(thisinfo.PTC, "")
+			}
+		}
+		if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
+			thisinfo.pnbval++
+		}
+	}
+	if thisinfo.ProjectCode == thisinfo.PTC || strings.Index(thisinfo.ProjectCode, thisinfo.PTC) > -1 {
+		thisinfo.PTC = ""
+	}
+
+	if thisinfo.Buyer != "" && len([]rune(thisinfo.Buyer)) > 2 {
+		thisinfo.pnbval++
+	} else {
+		thisinfo.Buyer = ""
+	}
+	//清理评审专家名单
+	if len(thisinfo.ReviewExperts) > 0 {
+		thisinfo.ReviewExperts = ClearRp(thisinfo.ReviewExperts)
+	}
+	//winners整理
+	thisinfo.Winners = strings.Split(util.ObjToString(tmp["s_winner"]), ",")
+
+	thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
+	thisinfo.LenPTC = len([]rune(thisinfo.PTC))
+	thisinfo.LenPN = len([]rune(thisinfo.ProjectName))
+
+	//处理分包中数据异常问题
+	for k, tmp := range thisinfo.Package {
+		if ps, ok := tmp.([]map[string]interface{}); ok {
+			for i, p := range ps {
+				name, _ := p["name"].(string)
+				if len([]rune(name)) > 100 {
+					p["name"] = fmt.Sprint([]rune(name[:100]))
+				}
+				ps[i] = p
+			}
+			thisinfo.Package[k] = ps
+		}
+	}
+	return thisinfo
+}
+
+//从数组中删除元素
+func deleteSlice(arr []string, v, stype string) []string {
+	for k, v1 := range arr {
+		if v1 == v {
+			ts := time.Now().Unix()
+			arr = append(arr[:k], arr[k+1:]...)
+			rt := time.Now().Unix() - ts
+			if rt > 0 {
+				log.Println("deleteSlice", stype, rt, v, len(arr))
+			}
+			return arr
+		}
+	}
+	return arr
+}
+
+//校验评审专家
+func ClearRp(tmp []string) []string {
+	arrTmp := []string{}
+	for _, v := range tmp {
+		// 汉字过滤(全汉字，2-4个字)
+		if ok, _ := regexp.MatchString("^[\\p{Han}]{2,4}$", v); !ok {
+			continue
+		}
+		arrTmp = append(arrTmp, v)
+	}
+	return arrTmp
+}

qyxy_data/main.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"mongodb"
+	"qfw/util"
+	"sync"
+)
+
+var (
+	Mgo, Mgo1 *mongodb.MongodbSim
+)
+
+func init() {
+	//Mgo = &mongodb.MongodbSim{
+	//	MongodbAddr: "192.168.3.207:29092",
+	//	Size:        10,
+	//	DbName:      "wjh",
+	//}
+	//Mgo.InitPool()
+
+	Mgo1 = &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083", // 172.17.4.187:27082,172.17.145.163:27083
+		Size:        15,
+		DbName:      "mixdata",
+		UserName:    "SJZY_RWESBid_Other",
+		Password:    "SJZY@O17t8herB3B",
+	}
+	Mgo1.InitPool()
+}
+
+func main() {
+	sess := Mgo1.GetMgoConn()
+	defer Mgo1.DestoryMongoConn(sess)
+
+	ch := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+
+	field := map[string]interface{}{"bid_contracttype": -1, "bid_unittype": -1, "bid_projectname": -1, "bid_purchasing": -1, "bid_area": -1}
+	query := sess.DB("mixdata").C("qyxy_tmp").Find(nil).Select(field).Iter()
+	count, taskcount := 0, 0
+	for tmp := make(map[string]interface{}); query.Next(&tmp); count++ {
+		if count%50 == 0 {
+			util.Debug("current ---", count, taskcount)
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			name := util.ObjToString(tmp["company_name"])
+			info, _ := Mgo1.FindOne("qyxy_std", map[string]interface{}{"company_name": name})
+			if len(*info) > 0 {
+				Mgo1.Save("qyxy_oprd", *info)
+				Mgo1.UpdateById("qyxy_tmp", tmp["_id"], map[string]interface{}{"$set": map[string]interface{}{"exist": true}})
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	util.Debug("over ---", count, taskcount)
+
+	c := make(chan bool, 1)
+	<-c
+}

qyxy_es/config.json

@@ -0,0 +1,53 @@
+{
+  "mgodb": "192.168.3.207:27092",
+  "dbsize": 12,
+  "dbname": "wjh",
+  "dbcoll": "oprd_qyxy",
+  "uname": "SJZY_RWMIX_Other",
+  "upwd": "SJZY@M34I6x7D9ata",
+  "tasktime": 0,
+  "updatetime": 0,
+  "elastic": {
+    "addr": "http://192.168.3.206:9800",
+    "index": "oprd_qyxy_v1",
+    "itype": "qyxy",
+    "pool": 12,
+    "esfields": [
+      "_id",
+      "company_name",
+      "history_name",
+      "credit_no",
+      "org_code",
+      "tax_code",
+      "company_code",
+      "company_type",
+      "company_type_old",
+      "company_area",
+      "company_city",
+      "company_district",
+      "company_phone",
+      "company_email",
+      "updatetime",
+      "legal_person",
+      "establish_date",
+      "lastupdatetime",
+      "capital",
+      "currency",
+      "operation_startdate",
+      "operation_enddate",
+      "authority",
+      "issue_date",
+      "company_status",
+      "company_address",
+      "business_scope",
+      "cancel_date",
+      "revoke_date",
+      "employee_name",
+      "stock_name",
+      "company_shortname",
+      "website_url",
+      "partners",
+      "employees"
+    ]
+  }
+}

qyxy_es/main.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"mongodb"
+	qu "qfw/util"
+	es "qfw/util/elastic"
+)
+
+var (
+	Sysconfig  map[string]interface{}
+	Mgo        *mongodb.MongodbSim
+	Dbname     string
+	Dbcoll     string
+	Es         *es.Elastic
+	Index      string
+	Itype      string
+	EsFields   []string
+	TaskTime   int
+	Updatetime int64
+)
+var EsSaveCache = make(chan map[string]interface{}, 5000)
+var SP = make(chan bool, 5)
+
+func init() {
+	qu.ReadConfig(&Sysconfig)
+	Dbname = Sysconfig["dbname"].(string)
+	Dbcoll = Sysconfig["dbcoll"].(string)
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: Sysconfig["mgodb"].(string),
+		Size:        qu.IntAllDef(Sysconfig["dbsize"], 5),
+		DbName:      Dbname,
+		//UserName:    Sysconfig["uname"].(string),
+		//Password:    Sysconfig["upwd"].(string),
+	}
+	Mgo.InitPool()
+	//es
+	econf := Sysconfig["elastic"].(map[string]interface{})
+	Index = econf["index"].(string)
+	Itype = econf["itype"].(string)
+	Es = &es.Elastic{
+		S_esurl: econf["addr"].(string),
+		I_size:  qu.IntAllDef(econf["pool"], 12),
+	}
+	Es.InitElasticSize()
+	EsFields = qu.ObjArrToStringArr(econf["esfields"].([]interface{}))
+	//TaskTime = qu.IntAll(Sysconfig["tasktime"])
+	Updatetime = qu.Int64All(Sysconfig["updatetime"])
+
+}
+
+func main() {
+	go SaveEs()
+
+	//go TimeTask()
+	go StdAll()
+	//go StdAdd()
+	ch := make(chan bool, 1)
+	<-ch
+}

qyxy_es/task.go

@@ -0,0 +1,371 @@
+package main
+
+import (
+	"fmt"
+	"github.com/cron"
+	"go.mongodb.org/mongo-driver/bson"
+	"log"
+	"qfw/util"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	partner  = []string{"identify_no", "stock_type", "stock_name", "identify_type", "stock_capital", "stock_realcapital"}
+	employee = []string{"employee_name", "position"}
+)
+
+//定时任务
+func TimeTask() {
+	c := cron.New()
+	cronstr := "0 0 15 ? * Tue" //每周二15点执行
+	//cronstr := "0 */" + fmt.Sprint(TaskTime) + " * * * ?" //每TaskTime小时执行一次
+	err := c.AddFunc(cronstr, func() { StdAdd() })
+	if err != nil {
+		util.Debug(err)
+		return
+	}
+	c.Start()
+}
+
+// StdAdd 增量数据
+func StdAdd() {
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
+	//q := bson.M{"_id": "affe29f8d061f3faa4170cafba41f316"}
+	q := bson.M{"updatetime": bson.M{"$gt": Updatetime}}
+	util.Debug(q)
+	it := sess.DB(Dbname).C(Dbcoll).Find(q).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%200 == 0 {
+			log.Println("current:", count)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			esMap := map[string]interface{}{}
+			//生索引字段处理
+			for _, field := range EsFields {
+				if tmp[field] == nil {
+					continue
+				}
+				if field == "company_name" {
+					esMap[field] = tmp["company_name"]
+					esMap["name"] = tmp["company_name"]
+				} else if field == "history_name" {
+					var nameArr []string
+					names := util.ObjToString(tmp["history_name"])
+					if strings.Contains(names, ",") {
+						nameArr = append(nameArr, strings.Split(names, ",")...)
+					}
+					if len(nameArr) > 0 {
+						esMap["history_name"] = nameArr
+					}
+				} else if field == "establish_date" {
+					// 成立日期修改成时间戳
+					location, err := time.ParseInLocation(util.Date_Short_Layout, util.ObjToString(tmp["establish_date"]), time.Local)
+					if err != nil {
+						util.Debug(err)
+					} else {
+						esMap["establish_date"] = location.Unix()
+					}
+				} else if field == "lastupdatetime" {
+					esMap["lastupdatetime"] = tmp["update_time_msql"]
+				} else if field == "bid_projectname" {
+					if pname, ok := tmp["bid_projectname"].([]interface{}); ok {
+						p1 := util.ObjArrToStringArr(pname)
+						esMap["bid_projectname"] = strings.Join(p1, ",")
+					}
+				} else if field == "bid_purchasing" {
+					if pur, ok := tmp["bid_purchasing"].([]interface{}); ok {
+						p1 := util.ObjArrToStringArr(pur)
+						esMap["bid_purchasing"] = strings.Join(p1, ",")
+					}
+				} else if field == "bid_area" {
+					if areas, ok := tmp["bid_area"].([]interface{}); ok {
+						p1 := util.ObjArrToStringArr(areas)
+						esMap["bid_area"] = strings.Join(p1, ",")
+					}
+				} else if field == "partners" {
+					if ps, ok := tmp["partners"].([]interface{}); ok {
+						var parr []map[string]interface{}
+						for _, v := range ps {
+							p := make(map[string]interface{})
+							v1 := v.(map[string]interface{})
+							for _, field := range partner {
+								if v1[field] == nil {
+									continue
+								}
+								if field == "stock_capital" || field == "stock_realcapital" {
+									if v, err := strconv.ParseFloat(util.ObjToString(v1[field]), 64); err == nil {
+										v, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", v), 64) //保留小数点两位
+										p[field] = v
+									}
+								} else {
+									p[field] = v1[field]
+								}
+							}
+							if len(p) > 0 {
+								parr = append(parr, p)
+							}
+						}
+						if len(parr) > 0 {
+							esMap[field] = parr
+						}
+					}
+				} else if field == "employees" {
+					if ps, ok := tmp["employees"].([]interface{}); ok {
+						var parr []map[string]interface{}
+						for _, v := range ps {
+							p := make(map[string]interface{})
+							v1 := v.(map[string]interface{})
+							for _, field := range employee {
+								if v1[field] == nil {
+									continue
+								} else {
+									p[field] = v1[field]
+								}
+							}
+							if len(p) > 0 {
+								parr = append(parr, p)
+							}
+						}
+						if len(parr) > 0 {
+							esMap[field] = parr
+						}
+					}
+				} else {
+					esMap[field] = tmp[field]
+				}
+			}
+			company_type := util.ObjToString(tmp["company_type"])
+			company_name := util.ObjToString(tmp["company_name"])
+			if company_type == "个体工商户" {
+				if len([]rune(company_name)) >= 5 {
+					esMap["company_type_int"] = 31
+				} else {
+					esMap["company_type_int"] = 32
+				}
+			} else if company_type == "其他" || company_type == "" {
+				if len([]rune(company_name)) >= 4 {
+					esMap["company_type_int"] = 21
+				} else {
+					esMap["company_type_int"] = 22
+				}
+			} else {
+				if company_type == "内资分公司" {
+					esMap["company_type_int"] = 12
+				} else if len([]rune(company_name)) >= 4 {
+					esMap["company_type_int"] = 11
+				} else {
+					esMap["company_type_int"] = 13
+				}
+			}
+			EsSaveCache <- esMap // 保存es
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("Run Over...Count:", count)
+}
+
+// StdAll 存量数据生es
+func StdAll() {
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+	//q := bson.M{"_id": mongodb.StringTOBsonId("61d7f129de77c557ce0457ca")}
+	it := sess.DB(Dbname).C(Dbcoll).Find(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%200 == 0 {
+			log.Println("current:", count)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			esMap := map[string]interface{}{}
+			//生索引字段处理
+			for _, field := range EsFields {
+				if tmp[field] == nil {
+					continue
+				}
+				if field == "company_name" {
+					esMap[field] = tmp["company_name"]
+					esMap["name"] = tmp["company_name"]
+				} else if field == "history_name" {
+					var nameArr []string
+					for _, v := range strings.Split(util.ObjToString(tmp["history_name"]), "；") {
+						if v != "" {
+							nameArr = append(nameArr, v)
+						}
+					}
+					if len(nameArr) > 0 {
+						esMap["history_name"] = nameArr
+					}
+				} else if field == "establish_date" && util.ObjToString(tmp["establish_date"]) != "" {
+					// 成立日期修改成时间戳
+					location, err := time.ParseInLocation(util.Date_Short_Layout, util.ObjToString(tmp["establish_date"]), time.Local)
+					if err != nil {
+						util.Debug(err)
+					} else {
+						esMap["establish_date"] = location.Unix()
+					}
+				} else if field == "lastupdatetime" {
+					esMap["lastupdatetime"] = tmp["update_time_msql"]
+				} else if field == "bid_projectname" {
+					if pname, ok := tmp["bid_projectname"].([]interface{}); ok {
+						p1 := util.ObjArrToStringArr(pname)
+						esMap["bid_projectname"] = strings.Join(p1, ",")
+					}
+				} else if field == "bid_purchasing" {
+					if pur, ok := tmp["bid_purchasing"].([]interface{}); ok {
+						p1 := util.ObjArrToStringArr(pur)
+						esMap["bid_purchasing"] = strings.Join(p1, ",")
+					}
+				} else if field == "bid_area" {
+					if areas, ok := tmp["bid_area"].([]interface{}); ok {
+						p1 := util.ObjArrToStringArr(areas)
+						esMap["bid_area"] = strings.Join(p1, ",")
+					}
+				} else if field == "partners" {
+					if ps, ok := tmp["partners"].([]interface{}); ok {
+						var parr []map[string]interface{}
+						for _, v := range ps {
+							p := make(map[string]interface{})
+							v1 := v.(map[string]interface{})
+							for _, field := range partner {
+								if v1[field] == nil {
+									continue
+								}
+								if field == "stock_capital" || field == "stock_realcapital" {
+									text := util.ObjToString(v1[field])
+									if strings.Contains(text, "万元") {
+										text = strings.Replace(text, "万元", "", -1)
+									}
+									if v, err := strconv.ParseFloat(text, 64); err == nil {
+										v, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", v), 64) //保留小数点两位
+										p[field] = v
+									}
+								} else {
+									p[field] = v1[field]
+								}
+							}
+							if len(p) > 0 {
+								parr = append(parr, p)
+							}
+						}
+						if len(parr) > 0 {
+							esMap[field] = parr
+						}
+					}
+				} else if field == "employees" {
+					if ps, ok := tmp["employees"].([]interface{}); ok {
+						var parr []map[string]interface{}
+						for _, v := range ps {
+							p := make(map[string]interface{})
+							v1 := v.(map[string]interface{})
+							for _, field := range employee {
+								if v1[field] == nil {
+									continue
+								} else {
+									p[field] = v1[field]
+								}
+							}
+							if len(p) > 0 {
+								parr = append(parr, p)
+							}
+						}
+						if len(parr) > 0 {
+							esMap[field] = parr
+						}
+					}
+				} else {
+					esMap[field] = tmp[field]
+				}
+			}
+			company_type := util.ObjToString(tmp["company_type"])
+			company_name := util.ObjToString(tmp["company_name"])
+			if company_type == "个体工商户" {
+				if len([]rune(company_name)) >= 5 {
+					esMap["company_type_int"] = 31
+				} else {
+					esMap["company_type_int"] = 32
+				}
+			} else if company_type == "其他" || company_type == "" {
+				if len([]rune(company_name)) >= 4 {
+					esMap["company_type_int"] = 21
+				} else {
+					esMap["company_type_int"] = 22
+				}
+			} else {
+				if company_type == "内资分公司" {
+					esMap["company_type_int"] = 12
+				} else if len([]rune(company_name)) >= 4 {
+					esMap["company_type_int"] = 11
+				} else {
+					esMap["company_type_int"] = 13
+				}
+			}
+			EsSaveCache <- esMap // 保存es
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("Run Over...Count:", count)
+}
+
+// SaveEs 过滤后数据存库
+func SaveEs() {
+	log.Println("Es Save...")
+	arru := make([]map[string]interface{}, 100)
+	indexu := 0
+	for {
+		select {
+		case v := <-EsSaveCache:
+			arru[indexu] = v
+			indexu++
+			if indexu == 100 {
+				SP <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-SP
+					}()
+					Es.BulkSave(Index, Itype, &arru, true)
+				}(arru)
+				arru = make([]map[string]interface{}, 100)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				SP <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-SP
+					}()
+					Es.BulkSave(Index, Itype, &arru, true)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, 100)
+				indexu = 0
+			}
+		}
+	}
+}

sql_data/config.json

@@ -0,0 +1,108 @@
+{
+  "qyxy_field": {
+    "company_id": "string",
+    "company_name": "string",
+    "company_code": "string",
+    "company_address": "string",
+    "company_area": "string",
+    "company_city": "string",
+    "company_district": "string",
+    "company_email": "string",
+    "company_phone": "string",
+    "company_status": "string",
+    "company_type": "string",
+    "company_type_old": "string",
+    "credit_no": "string",
+    "org_code": "string",
+    "tax_code": "string",
+    "capital": "string",
+    "legal_person": "string",
+    "issue_date": "string",
+    "establish_date": "string",
+    "bid_type": "string",
+    "partners": ""
+  },
+  "qyxy_partner_field": {
+    "company_id": "string",
+    "stock_name": "string",
+    "identify_no": "string",
+    "stock_capital": "string",
+    "stock_realcapital": "string",
+    "stock_type": "string"
+  },
+  "qyxy_employees_field": {
+    "company_id": "string",
+    "employee_name": "string",
+    "position": "string"
+  },
+  "bidding_field": {
+    "_id": "",
+    "buyerzipcode": "string",
+    "winnertel": "string",
+    "winnerperson": "string",
+    "contractcode": "string",
+    "winneraddr": "string",
+    "agencyaddr": "string",
+    "buyeraddr": "string",
+    "signaturedate": "int64",
+    "projectperiod": "string",
+    "projectaddr": "string",
+    "agencytel": "string",
+    "agencyperson": "string",
+    "buyerperson": "string",
+    "agency": "string",
+    "projectscope": "string",
+    "projectcode": "string",
+    "bidopentime": "int64",
+    "supervisorrate": "float64",
+    "buyertel": "string",
+    "bidamount": "float64",
+    "winner": "string",
+    "buyer": "string",
+    "budget": "float64",
+    "projectname": "string",
+    "bidstatus": "string",
+    "buyerclass": "string",
+    "s_topscopeclass": "string",
+    "s_subscopeclass": "string",
+    "area": "string",
+    "city": "string",
+    "district": "string",
+    "s_winner": "string",
+    "title": "string",
+    "detail": "string",
+    "site": "string",
+    "comeintime": "int64",
+    "href": "string",
+    "infoformat": "int32",
+    "publishtime": "int64",
+    "s_sha": "string",
+    "spidercode": "string",
+    "subtype": "string",
+    "toptype": "string",
+    "purchasing": "string",
+    "channel": "string",
+    "project_scale": "string",
+    "project_duration": "int32",
+    "project_timeunit": "string",
+    "project_startdate": "int64",
+    "project_completedate": "int64",
+    "payway": "string",
+    "contract_guarantee": "bool",
+    "bid_guarantee": "bool",
+    "funds": "string",
+    "bidmethod": "string",
+    "bidendtime": "int64",
+    "bidopenaddress": "string",
+    "docamount": "float64",
+    "agencyrate": "float64",
+    "agencyfee": "float64",
+    "bidway": "string",
+    "tag_rule": "string"
+  },
+  "project_field": {
+    "_id": "",
+    "projectname": "string",
+    "title": "string"
+  }
+}

sql_data/init.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"mongodb"
+	"qfw/util"
+)
+
+var (
+	Sysconfig     map[string]interface{}
+	Mgo           *mongodb.MongodbSim
+	MysqlTool     *Mysql
+	QyxyFieldsMap map[string]interface{}
+	QyxyPartner   map[string]interface{}
+	BiddingMap    map[string]interface{}
+	ProjectMap    map[string]interface{}
+)
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: "192.168.3.207:27092",
+		Size:        10,
+		DbName:      "wjh",
+	}
+	Mgo.InitPool()
+
+	MysqlTool = &Mysql{
+		Address:  "192.168.3.109:4000", // 192.168.3.109:4000
+		UserName: "jianyu",
+		PassWord: "top@123",
+		DBName:   "privatedata",
+	}
+	MysqlTool.Init()
+
+	QyxyFieldsMap = Sysconfig["qyxy_field"].(map[string]interface{})
+	QyxyPartner = Sysconfig["qyxy_partner_field"].(map[string]interface{})
+	BiddingMap = Sysconfig["bidding_field"].(map[string]interface{})
+	ProjectMap = Sysconfig["bidding_field"].(map[string]interface{})
+}

sql_data/main.go

@@ -0,0 +1,168 @@
+package main
+
+import (
+	"mongodb"
+	"qfw/util"
+	"reflect"
+	"sync"
+)
+
+func main() {
+	//taskQyxy()
+	taskBidding()
+}
+
+func taskQyxy() {
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+
+	q := map[string]interface{}{"_id": mongodb.StringTOBsonId("61d552398e4217aff3dd7462")}
+	it := sess.DB("wjh").C("oprd_qyxy").Find(q).Select(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%2000 == 0 {
+			util.Debug("current:", count)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			save := make(map[string]interface{})
+			for s, s2 := range QyxyFieldsMap {
+				if s == "company_id" {
+					save[s] = mongodb.BsonIdToSId(tmp["_id"])
+				} else if s == "partners" {
+					if partners, ok := tmp["partners"].([]interface{}); ok {
+						for _, p := range partners {
+							p1 := p.(map[string]interface{})
+							ptSave := make(map[string]interface{})
+							for s3, s4 := range QyxyPartner {
+								if s3 == "company_id" {
+									ptSave["company_id"] = mongodb.BsonIdToSId(tmp["_id"])
+								} else {
+									if p1[s3] != nil && reflect.TypeOf(p1[s3]).String() == s4 {
+										ptSave[s3] = p1[s3]
+									}
+								}
+							}
+							if len(ptSave) > 0 {
+								MysqlTool.Insert("qyxy_partner", ptSave)
+							}
+						}
+					}
+				} else {
+					if tmp[s] != nil && reflect.TypeOf(tmp[s]).String() == s2 {
+						save[s] = tmp[s]
+					}
+				}
+			}
+			if len(save) > 0 {
+				MysqlTool.Insert("qyxy", save)
+			}
+		}(tmp)
+	}
+	wg.Wait()
+}
+
+func taskBidding() {
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+
+	q := map[string]interface{}{"_id": mongodb.StringTOBsonId("5d4a77cda5cb26b9b7e6986a")}
+	it := sess.DB("wjh").C("oprd_bidding").Find(q).Select(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%2000 == 0 {
+			util.Debug("current:", count)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			save := make(map[string]interface{})
+			for s, s2 := range BiddingMap {
+				if s == "_id" {
+					save[s] = mongodb.BsonIdToSId(tmp["_id"])
+				} else if s == "contract_guarantee" || s == "bid_guarantee" {
+					if tmp[s] != nil {
+						if tmp[s] == true {
+							save[s] = 1
+						} else {
+							save[s] = 0
+						}
+					}
+				} else {
+					if s == "budget" {
+						util.Debug(reflect.TypeOf(tmp[s]).String())
+					}
+					if tmp[s] != nil && reflect.TypeOf(tmp[s]).String() == s2 {
+						save[s] = tmp[s]
+					}
+				}
+			}
+			if len(save) > 0 {
+				MysqlTool.Insert("bidding", save)
+			}
+		}(tmp)
+	}
+	wg.Wait()
+}
+
+func taskProject() {
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+
+	q := map[string]interface{}{"_id": mongodb.StringTOBsonId("5d4a77cda5cb26b9b7e6986a")}
+	it := sess.DB("wjh").C("oprd_project").Find(q).Select(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%2000 == 0 {
+			util.Debug("current:", count)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			save := make(map[string]interface{})
+			for s, s2 := range BiddingMap {
+				if s == "_id" {
+					save[s] = mongodb.BsonIdToSId(tmp["_id"])
+				} else if s == "contract_guarantee" || s == "bid_guarantee" {
+					if tmp[s] != nil {
+						if tmp[s] == true {
+							save[s] = 1
+						} else {
+							save[s] = 0
+						}
+					}
+				} else {
+					if tmp[s] != nil && reflect.TypeOf(tmp[s]).String() == s2 {
+						save[s] = tmp[s]
+					}
+				}
+			}
+			if len(save) > 0 {
+				MysqlTool.Insert("bidding", save)
+			}
+		}(tmp)
+	}
+	wg.Wait()
+}

sql_data/mysql.go

@@ -0,0 +1,504 @@
+package main
+
+import (
+	"bytes"
+	"database/sql"
+	"fmt"
+	"log"
+	"reflect"
+	"strings"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+type Mysql struct {
+	Address      string  //数据库地址:端口
+	UserName     string  //用户名
+	PassWord     string  //密码
+	DBName       string  //数据库名
+	DB           *sql.DB //数据库连接池对象
+	MaxOpenConns int     //用于设置最大打开的连接数，默认值为0表示不限制。
+	MaxIdleConns int     //用于设置闲置的连接数。
+}
+
+func (m *Mysql) Init() {
+	if m.MaxOpenConns <= 0 {
+		m.MaxOpenConns = 20
+	}
+	if m.MaxIdleConns <= 0 {
+		m.MaxIdleConns = 20
+	}
+	var err error
+	m.DB, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4", m.UserName, m.PassWord, m.Address, m.DBName))
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	m.DB.SetMaxOpenConns(m.MaxOpenConns)
+	m.DB.SetMaxIdleConns(m.MaxIdleConns)
+	m.DB.SetConnMaxLifetime(time.Minute * 3)
+	err = m.DB.Ping()
+	if err != nil {
+		log.Println(err)
+	}
+}
+
+//新增
+func (m *Mysql) Insert(tableName string, data map[string]interface{}) int64 {
+	return m.InsertByTx(nil, tableName, data)
+}
+
+//带有事务的新增
+func (m *Mysql) InsertByTx(tx *sql.Tx, tableName string, data map[string]interface{}) int64 {
+	fields := []string{}
+	values := []interface{}{}
+	placeholders := []string{}
+	if tableName == "dataexport_order" {
+		if _, ok := data["user_nickname"]; ok {
+			data["user_nickname"] = ""
+		}
+	}
+	for k, v := range data {
+		fields = append(fields, k)
+		values = append(values, v)
+		placeholders = append(placeholders, "?")
+	}
+	q := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", tableName, strings.Join(fields, ","), strings.Join(placeholders, ","))
+	log.Println("mysql", q, values)
+	return m.InsertBySqlByTx(tx, q, values...)
+}
+
+//sql语句新增
+func (m *Mysql) InsertBySql(q string, args ...interface{}) int64 {
+	return m.InsertBySqlByTx(nil, q, args...)
+}
+
+//带有事务的sql语句新增
+func (m *Mysql) InsertBySqlByTx(tx *sql.Tx, q string, args ...interface{}) int64 {
+	result, _ := m.ExecBySqlByTx(tx, q, args...)
+	if result == nil {
+		return -1
+	}
+	id, err := result.LastInsertId()
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	return id
+}
+
+//批量新增
+func (m *Mysql) InsertIgnoreBatch(tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.InsertIgnoreBatchByTx(nil, tableName, fields, values)
+}
+
+//带事务的批量新增
+func (m *Mysql) InsertIgnoreBatchByTx(tx *sql.Tx, tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.insertOrReplaceBatchByTx(tx, "INSERT", "IGNORE", tableName, fields, values)
+}
+
+//批量新增
+func (m *Mysql) InsertBatch(tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.InsertBatchByTx(nil, tableName, fields, values)
+}
+
+//带事务的批量新增
+func (m *Mysql) InsertBatchByTx(tx *sql.Tx, tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.insertOrReplaceBatchByTx(tx, "INSERT", "", tableName, fields, values)
+}
+
+//批量更新
+func (m *Mysql) ReplaceBatch(tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.ReplaceBatchByTx(nil, tableName, fields, values)
+}
+
+//带事务的批量更新
+func (m *Mysql) ReplaceBatchByTx(tx *sql.Tx, tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.insertOrReplaceBatchByTx(tx, "REPLACE", "", tableName, fields, values)
+}
+
+func (m *Mysql) insertOrReplaceBatchByTx(tx *sql.Tx, tp string, afterInsert, tableName string, fields []string, values []interface{}) (int64, int64) {
+	placeholders := []string{}
+	for range fields {
+		placeholders = append(placeholders, "?")
+	}
+	placeholder := strings.Join(placeholders, ",")
+	array := []string{}
+	for i := 0; i < len(values)/len(fields); i++ {
+		array = append(array, fmt.Sprintf("(%s)", placeholder))
+	}
+	q := fmt.Sprintf("%s %s INTO %s (%s) VALUES %s", tp, afterInsert, tableName, strings.Join(fields, ","), strings.Join(array, ","))
+	result, _ := m.ExecBySqlByTx(tx, q, values...)
+	if result == nil {
+		return -1, -1
+	}
+	v1, e1 := result.RowsAffected()
+	if e1 != nil {
+		log.Println(e1)
+		return -1, -1
+	}
+	v2, e2 := result.LastInsertId()
+	if e2 != nil {
+		log.Println(e2)
+		return -1, -1
+	}
+	return v1, v2
+}
+
+//sql语句执行
+func (m *Mysql) ExecBySql(q string, args ...interface{}) (sql.Result, error) {
+	return m.ExecBySqlByTx(nil, q, args...)
+}
+
+//sql语句执行,带有事务
+func (m *Mysql) ExecBySqlByTx(tx *sql.Tx, q string, args ...interface{}) (sql.Result, error) {
+	var stmtIns *sql.Stmt
+	var err error
+	if tx == nil {
+		stmtIns, err = m.DB.Prepare(q)
+	} else {
+		stmtIns, err = tx.Prepare(q)
+	}
+	if err != nil {
+		log.Println(err)
+		return nil, err
+	}
+	defer stmtIns.Close()
+	result, err := stmtIns.Exec(args...)
+	if err != nil {
+		log.Println(args, err)
+		return nil, err
+	}
+	return result, nil
+}
+
+/*不等于 map[string]string{"ne":"1"}
+ *不等于多个 map[string]string{"notin":[]interface{}{1,2}}
+ *字段为空 map[string]string{"name":"$isNull"}
+ *字段不为空 map[string]string{"name":"$isNotNull"}
+ */
+func (m *Mysql) Find(tableName string, query map[string]interface{}, fields, order string, start, pageSize int) *[]map[string]interface{} {
+	fs := []string{}
+	vs := []interface{}{}
+	for k, v := range query {
+		rt := reflect.TypeOf(v)
+		rv := reflect.ValueOf(v)
+		if rt.Kind() == reflect.Map {
+			for _, rv_k := range rv.MapKeys() {
+				if rv_k.String() == "ne" {
+					fs = append(fs, fmt.Sprintf("%s!=?", k))
+					vs = append(vs, rv.MapIndex(rv_k).Interface())
+				}
+				if rv_k.String() == "notin" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						for _, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							fs = append(fs, fmt.Sprintf("%s!=?", k))
+							vs = append(vs, v)
+						}
+					}
+				}
+				if rv_k.String() == "in" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						_fs := fmt.Sprintf("%s in (?", k)
+						for k, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							if k > 0 {
+								_fs += ",?"
+							}
+							vs = append(vs, v)
+						}
+						_fs += ")"
+						fs = append(fs, _fs)
+					}
+				}
+			}
+		} else {
+			if v == "$isNull" {
+				fs = append(fs, fmt.Sprintf("%s is null", k))
+			} else if v == "$isNotNull" {
+				fs = append(fs, fmt.Sprintf("%s is not null", k))
+			} else {
+				fs = append(fs, fmt.Sprintf("%s=?", k))
+				vs = append(vs, v)
+			}
+		}
+	}
+	var buffer bytes.Buffer
+	buffer.WriteString("select ")
+	if fields == "" {
+		buffer.WriteString("*")
+	} else {
+		buffer.WriteString(fields)
+	}
+	buffer.WriteString(" from ")
+	buffer.WriteString(tableName)
+	if len(fs) > 0 {
+		buffer.WriteString(" where ")
+		buffer.WriteString(strings.Join(fs, " and "))
+	}
+	if order != "" {
+		buffer.WriteString(" order by ")
+		buffer.WriteString(order)
+	}
+	if start > -1 && pageSize > 0 {
+		buffer.WriteString(" limit ")
+		buffer.WriteString(fmt.Sprint(start))
+		buffer.WriteString(",")
+		buffer.WriteString(fmt.Sprint(pageSize))
+	}
+	q := buffer.String()
+	log.Println(q, vs)
+	return m.SelectBySql(q, vs...)
+}
+
+//sql语句查询
+func (m *Mysql) SelectBySql(q string, args ...interface{}) *[]map[string]interface{} {
+	return m.SelectBySqlByTx(nil, q, args...)
+}
+func (m *Mysql) SelectBySqlByTx(tx *sql.Tx, q string, args ...interface{}) *[]map[string]interface{} {
+	return m.Select(0, nil, tx, q, args...)
+}
+func (m *Mysql) Select(bath int, f func(l *[]map[string]interface{}), tx *sql.Tx, q string, args ...interface{}) *[]map[string]interface{} {
+	var stmtOut *sql.Stmt
+	var err error
+	if tx == nil {
+		stmtOut, err = m.DB.Prepare(q)
+	} else {
+		stmtOut, err = tx.Prepare(q)
+	}
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	defer stmtOut.Close()
+	rows, err := stmtOut.Query(args...)
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	if rows != nil {
+		defer rows.Close()
+	}
+	columns, err := rows.Columns()
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	list := []map[string]interface{}{}
+	for rows.Next() {
+		scanArgs := make([]interface{}, len(columns))
+		values := make([]interface{}, len(columns))
+		ret := make(map[string]interface{})
+		for k, _ := range values {
+			scanArgs[k] = &values[k]
+		}
+		err = rows.Scan(scanArgs...)
+		if err != nil {
+			log.Println(err)
+			break
+		}
+		for i, col := range values {
+			if v, ok := col.([]uint8); ok {
+				ret[columns[i]] = string(v)
+			} else {
+				ret[columns[i]] = col
+			}
+		}
+		list = append(list, ret)
+		if bath > 0 && len(list) == bath {
+			f(&list)
+			list = []map[string]interface{}{}
+		}
+	}
+	if bath > 0 && len(list) > 0 {
+		f(&list)
+		list = []map[string]interface{}{}
+	}
+	return &list
+}
+func (m *Mysql) SelectByBath(bath int, f func(l *[]map[string]interface{}), q string, args ...interface{}) {
+	m.SelectByBathByTx(bath, f, nil, q, args...)
+}
+func (m *Mysql) SelectByBathByTx(bath int, f func(l *[]map[string]interface{}), tx *sql.Tx, q string, args ...interface{}) {
+	m.Select(bath, f, tx, q, args...)
+}
+func (m *Mysql) FindOne(tableName string, query map[string]interface{}, fields, order string) *map[string]interface{} {
+	list := m.Find(tableName, query, fields, order, 0, 1)
+	if list != nil && len(*list) == 1 {
+		temp := (*list)[0]
+		return &temp
+	}
+	return nil
+}
+
+//修改
+func (m *Mysql) Update(tableName string, query, update map[string]interface{}) bool {
+	return m.UpdateByTx(nil, tableName, query, update)
+}
+
+//带事务的修改
+func (m *Mysql) UpdateByTx(tx *sql.Tx, tableName string, query, update map[string]interface{}) bool {
+	q_fs := []string{}
+	u_fs := []string{}
+	values := []interface{}{}
+	for k, v := range update {
+		q_fs = append(q_fs, fmt.Sprintf("%s=?", k))
+		values = append(values, v)
+	}
+	for k, v := range query {
+		u_fs = append(u_fs, fmt.Sprintf("%s=?", k))
+		values = append(values, v)
+	}
+	q := fmt.Sprintf("update %s set %s where %s", tableName, strings.Join(q_fs, ","), strings.Join(u_fs, " and "))
+	log.Println(q, values)
+	return m.UpdateOrDeleteBySqlByTx(tx, q, values...) >= 0
+}
+
+//删除
+func (m *Mysql) Delete(tableName string, query map[string]interface{}) bool {
+	return m.DeleteByTx(nil, tableName, query)
+}
+func (m *Mysql) DeleteByTx(tx *sql.Tx, tableName string, query map[string]interface{}) bool {
+	fields := []string{}
+	values := []interface{}{}
+	for k, v := range query {
+		fields = append(fields, fmt.Sprintf("%s=?", k))
+		values = append(values, v)
+	}
+	q := fmt.Sprintf("delete from %s where %s", tableName, strings.Join(fields, " and "))
+	log.Println(q, values)
+	return m.UpdateOrDeleteBySqlByTx(tx, q, values...) > 0
+}
+
+//修改或删除
+func (m *Mysql) UpdateOrDeleteBySql(q string, args ...interface{}) int64 {
+	return m.UpdateOrDeleteBySqlByTx(nil, q, args...)
+}
+
+//带事务的修改或删除
+func (m *Mysql) UpdateOrDeleteBySqlByTx(tx *sql.Tx, q string, args ...interface{}) int64 {
+	result, err := m.ExecBySqlByTx(tx, q, args...)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	count, err := result.RowsAffected()
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	return count
+}
+
+//总数
+func (m *Mysql) Count(tableName string, query map[string]interface{}) int64 {
+	fields := []string{}
+	values := []interface{}{}
+	for k, v := range query {
+		rt := reflect.TypeOf(v)
+		rv := reflect.ValueOf(v)
+		if rt.Kind() == reflect.Map {
+			for _, rv_k := range rv.MapKeys() {
+				if rv_k.String() == "ne" {
+					fields = append(fields, fmt.Sprintf("%s!=?", k))
+					values = append(values, rv.MapIndex(rv_k).Interface())
+				}
+				if rv_k.String() == "notin" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						for _, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							fields = append(fields, fmt.Sprintf("%s!=?", k))
+							values = append(values, v)
+						}
+					}
+				}
+				if rv_k.String() == "in" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						_fs := fmt.Sprintf("%s in (?", k)
+						for k, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							if k > 0 {
+								_fs += ",?"
+							}
+							values = append(values, v)
+						}
+						_fs += ")"
+						fields = append(fields, _fs)
+					}
+				}
+			}
+		} else if v == "$isNull" {
+			fields = append(fields, fmt.Sprintf("%s is null", k))
+		} else if v == "$isNotNull" {
+			fields = append(fields, fmt.Sprintf("%s is not null", k))
+		} else {
+			fields = append(fields, fmt.Sprintf("%s=?", k))
+			values = append(values, v)
+		}
+	}
+	q := fmt.Sprintf("select count(1) as count from %s", tableName)
+	if len(query) > 0 {
+		q += fmt.Sprintf(" where %s", strings.Join(fields, " and "))
+	}
+	log.Println(q, values)
+	return m.CountBySql(q, values...)
+}
+func (m *Mysql) CountBySql(q string, args ...interface{}) int64 {
+	stmtIns, err := m.DB.Prepare(q)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	defer stmtIns.Close()
+
+	rows, err := stmtIns.Query(args...)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	if rows != nil {
+		defer rows.Close()
+	}
+	var count int64 = -1
+	if rows.Next() {
+		err = rows.Scan(&count)
+		if err != nil {
+			log.Println(err)
+		}
+	}
+	return count
+}
+
+//执行事务
+func (m *Mysql) ExecTx(msg string, f func(tx *sql.Tx) bool) bool {
+	tx, err := m.DB.Begin()
+	if err != nil {
+		log.Println(msg, "获取事务错误", err)
+	} else {
+		if f(tx) {
+			if err := tx.Commit(); err != nil {
+				log.Println(msg, "提交事务错误", err)
+			} else {
+				return true
+			}
+		} else {
+			if err := tx.Rollback(); err != nil {
+				log.Println(msg, "事务回滚错误", err)
+			}
+		}
+	}
+	return false
+}
+
+/*************方法命名不规范，上面有替代方法*************/
+func (m *Mysql) Query(query string, args ...interface{}) *[]map[string]interface{} {
+	return m.SelectBySql(query, args...)
+}
+
+func (m *Mysql) QueryCount(query string, args ...interface{}) (count int) {
+	count = -1
+	if !strings.Contains(strings.ToLower(query), "count(*)") {
+		fmt.Println("QueryCount need query like < select count(*) from ..... >")
+		return
+	}
+	count = int(m.CountBySql(query, args...))
+	return
+}