索引程序修改

esmgocount/src/config.json

@@ -32,8 +32,8 @@
     ],
 	"esAddr":"http://172.17.145.170:9800",
 	"esIndex":"bidding",
-    "esDataAddr":"http://172.17.4.189:9800",
-	"esDataIndex":"bidding",
+    "esAddr2":"http://172.17.4.189:9800",
+	"esIndex2":"bidding",
 	"jkmail": {
         "to": "wangjianghan@topnet.net.cn",
         "api": "http://10.171.112.160:19281/_send/_mail"

esmgocount/src/main.go

@@ -30,14 +30,14 @@ type T struct {
 
 //es数据最报警
 var (
-	config               map[string]interface{}
-	to                   string
-	api                  string
-	esAddr, esDataAddr   string
-	esIndex, esDataIndex string
-	Ts                   = []*T{}
-	esQ                  = `{"query": {"range": {"id": {"gte": "%s","lt": "%s"}}}}`
-	esQ1                 = `{"query": {"bool": {"must": [{"range": {"id": {"gte": "%s","lt": "%s"}}},{"terms": {"bidding.site": ["元博网（采购与招标网）","中国招标与采购网"]}}]}}}`
+	config            map[string]interface{}
+	to                string
+	api               string
+	esAddr, esAddr2   string
+	esIndex, esIndex2 string
+	Ts                = []*T{}
+	esQ               = `{"query": {"range": {"id": {"gte": "%s","lt": "%s"}}}}`
+	esQ1              = `{"query": {"bool": {"must": [{"range": {"id": {"gte": "%s","lt": "%s"}}},{"terms": {"bidding.site": ["元博网（采购与招标网）","中国招标与采购网"]}}]}}}`
 )
 
 func init() {
@@ -47,8 +47,8 @@ func init() {
 	api, _ = jkmail["api"].(string)
 	esAddr, _ = config["esAddr"].(string)
 	esIndex, _ = config["esIndex"].(string)
-	esDataAddr, _ = config["esDataAddr"].(string)
-	esDataIndex, _ = config["esDataIndex"].(string)
+	esAddr2, _ = config["esAddr2"].(string)
+	esIndex2, _ = config["esIndex2"].(string)
 	tasks, _ := config["task"].([]interface{})
 	for _, t := range tasks {
 		bs, _ := json.Marshal(t)
@@ -65,7 +65,7 @@ func main() {
 	if len(Ts) > 0 {
 		c := cron.New()
 		for _, v := range Ts {
-			c.AddFunc(v.Cron, v.task)
+			_ = c.AddFunc(v.Cron, v.task)
 		}
 		c.Start()
 		defer c.Stop()
@@ -98,8 +98,11 @@ func (t *T) task() {
 	eq1 := fmt.Sprintf(esQ1, st1, et1)
 	es := elastic.Elastic{S_esurl: esAddr, I_size: 1}
 	es.InitElasticSize()
+	es2 := elastic.Elastic{S_esurl: esAddr2, I_size: 1}
+	es2.InitElasticSize()
 	count := int(es.Count(esIndex, esIndex, eq))
-	count1 := int(es.Count(esIndex, esIndex, eq1))
+	count2 := int(es2.Count(esIndex2, esIndex2, eq)) // es 新集群
+	count1 := int(es.Count(esIndex, esIndex, eq1))   // 竞品
 	switch t.Type {
 	case "alert":
 		if count < t.Min || count > t.Max {
@@ -107,7 +110,7 @@ func (t *T) task() {
 			t.SendMail(report)
 		}
 	case "report":
-		report := fmt.Sprintf("报告%s,统计结果：%d", t.Name, count)
+		report := fmt.Sprintf("统计报告%s，统计结果旧es库数量：%d，新es库数量：%d", t.Name, count, count2)
 		if len(t.Mgo) > 5 {
 			fs := strings.Split(t.Mgo, "|")
 			fmgo := mongodb.NewMgoWithUser(fs[0], fs[3], fs[1], fs[2], 1)
@@ -136,8 +139,8 @@ func (t *T) task() {
 					}
 				}
 			}
-			report += ",mgo统计" + fmt.Sprint(count3) + ",差值:" + fmt.Sprint(count3-count) + ",mgo总入库量" + fmt.Sprint(count2)
-			report += "<br>" + "竟品统计结果：" + strconv.Itoa(count1) + ",mgo统计" + fmt.Sprint(count5) + ",差值:" + fmt.Sprint(count5-count1) + ",mgo总入库量" + fmt.Sprint(count4)
+			report += "，mgo统计" + fmt.Sprint(count3) + "，差值:" + fmt.Sprint(count3-count) + "，mgo总入库量" + fmt.Sprint(count2)
+			report += "<br>" + "竟品统计结果：" + strconv.Itoa(count1) + "，mgo统计" + fmt.Sprint(count5) + "，差值:" + fmt.Sprint(count5-count1) + "，mgo总入库量" + fmt.Sprint(count4)
 		}
 		t.SendMail(report)
 	}

fullproject/src_v1/config.json

@@ -4,11 +4,11 @@
     "statusdays": 15,
 	"mongodbServers": "192.168.3.207:27092",
     "mongodbPoolSize": 10,
-    "mongodbName": "zhaolongyue",
+    "mongodbName": "wjh",
 	"hints":"_id_1_publishtime_1",
-    "extractColl": "extract_test_new",
-    "extractColl1": "extract",
-    "projectColl": "projectset-test1",
+    "extractColl": "bidding",
+    "extractColl1": "bidding",
+    "projectColl": "projectset-test",
     "backupFlag": true,
     "siteColl": "site",
     "thread": 1,

fullproject/src_v1/main.go

@@ -100,8 +100,8 @@ func mainT() {
 }
 
 func main() {
-	sid = "61d3b61345a326c6c35080f8"
-	eid = "629496011cd2d8ec2137ed5b"
+	sid = "62a311cb1cd2d8ec214ea212"
+	eid = "62a311cb1cd2d8ec214ea214"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()

monitor/task.go

@@ -1,9 +1,12 @@
 package main
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"github.com/cron"
 	"go.mongodb.org/mongo-driver/bson"
+	"io/ioutil"
 	"mongodb"
 	"net/http"
 	"qfw/util"
@@ -11,13 +14,23 @@ import (
 	"time"
 )
 
-var taskA = 0		// 增量统计跳过
+var taskA = 0 // 增量统计跳过
+
+var (
+	WebUrl     = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=45962efc-ca87-4996-9ffa-08bf6608ab7a"
+	skip, send = 0, 0
+)
 
 func TimeTask() {
 	//StartTask()
 	c := cron.New()
-	cronstr := "0 0 * * * ?" // 每小时执行一次
-	_ = c.AddFunc(cronstr, func() { StartTask() })
+	//cronstr := "0 0 * * * ?" // 每小时执行一次
+	cronstr := "0 */10 * * * ?" // 每10min执行一次
+	//_ = c.AddFunc(cronstr, func() { StartTask() })
+	_ = c.AddFunc(cronstr, func() {
+		util.Debug("Start Task...")
+		Attachment()
+	})
 	c.Start()
 }
 
@@ -27,11 +40,33 @@ func StartTask() {
 	result = EsCheck(result)
 	if result != "" {
 		util.Debug(result)
-		SendMail(result)
 	}
 	util.Debug("Task Over...")
 }
 
+// @Description 附件识别监控
+// @Author J 2022/6/9 5:58 PM
+func Attachment() {
+	info, _ := Mgo.Find("ocr_goods_over", nil, map[string]interface{}{"_id": -1}, nil, true, -1, -1)
+	lastT := util.Int64All((*info)[0]["import_time"])
+	if time.Now().Unix()-lastT > 30*60 {
+		if skip == 0 {
+			skip += 1
+			util.Debug("跳过第一次", (*info)[0]["_id"])
+		} else {
+			send++
+			if send <= 3 {
+				SendMsg("当前最后一个id段数据累积时间已经超过30分钟, 最后一段数据入库时间为：" + util.FormatDateByInt64(&lastT, util.Date_Full_Layout))
+			}
+		}
+	} else {
+		if skip != 0 || send != 0 {
+			skip = 0
+			send = 0
+		}
+	}
+}
+
 // 统计mgo数据
 func MgoCount() string {
 	defer util.Catch()
@@ -55,7 +90,7 @@ func MgoCount() string {
 	if int64(count1) != LastStockSize {
 		result = fmt.Sprintf("bindding数据存量统计异常，统计条件%v，统计结果%d，上次统计数据量%d", q1, count1, LastStockSize)
 		result += "<br>"
-	}else {
+	} else {
 		//	无异常，替换成当前时间点(当前时间点往前一个小时的id)存量的统计结果
 		LastStockId = currentLastId
 		q2 := bson.M{"_id": bson.M{"$lte": mongodb.StringTOBsonId(LastStockId)}}
@@ -73,8 +108,8 @@ func MgoCount() string {
 			result += fmt.Sprintf("bindding数据增量统计异常，统计条件%v，统计结果%d", q3, count3)
 			result += "<br><br>"
 			taskA = 0
-		}else {
-			taskA ++
+		} else {
+			taskA++
 		}
 	}
 	return result
@@ -103,3 +138,15 @@ func EsCheck(result string) string {
 func SendMail(report string) {
 	_, _ = http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, to, "异常报告！", report))
 }
+
+func SendMsg(content string) {
+	client := &http.Client{}
+	data := map[string]interface{}{"msgtype": "text", "text": map[string]interface{}{
+		"content": content, "mentioned_list": []string{"@all"},
+	}}
+	bytesData, _ := json.Marshal(data)
+	req, _ := http.NewRequest("POST", WebUrl, bytes.NewReader(bytesData))
+	resp, _ := client.Do(req)
+	body, _ := ioutil.ReadAll(resp.Body)
+	fmt.Println(string(body))
+}

udpcreateindex/src/biddingall.go

@@ -4,32 +4,16 @@ import (
 	"fmt"
 	"go.mongodb.org/mongo-driver/bson"
 	"mongodb"
-	"qfw/util/elastic"
-	"regexp"
-	"time"
-	//"fmt"
-	"log"
 	qutil "qfw/util"
-	//elastic "qfw/util/elastic"
-	"reflect"
+	"regexp"
+
 	"strings"
 	"sync"
 )
 
-//对字段处理 bidamount  budget
-//招标数据表和抽取表一一对应开始更新
-/*
-	注意：
-	1、biddingall任务跑历史数据生成索引并更新bidding表
-	2、调用biddingall任务时config.json中indexfields配置要有purchasing、purchasinglist、filetext
-*/
-
-var RegSpace = regexp.MustCompile("[\\s\u3000\u2003\u00a0]+")
-
-func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
+func (t *TaskInfo) biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 	defer qutil.Catch()
-	thread := 20
-	var mpool = make(chan bool, thread)
+	var mpool = make(chan bool, t.thread)
 	q, _ := mapInfo["query"].(map[string]interface{})
 	if q == nil {
 		q = map[string]interface{}{
@@ -49,56 +33,38 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 		q["_id"] = tmpQ
 	}
 	//bidding库
-	session := mgo.GetMgoConn()
-	defer mgo.DestoryMongoConn(session)
+	biddingConn := biddingMgo.GetMgoConn()
+	defer biddingMgo.DestoryMongoConn(biddingConn)
 	//extract库
-	extractsession := extractmgo.GetMgoConn()
-	defer extractmgo.DestoryMongoConn(extractsession)
+	extractConn := extractMgo.GetMgoConn()
+	defer extractMgo.DestoryMongoConn(extractConn)
 	//连接信息
 	c, _ := mapInfo["coll"].(string)
 	if c == "" {
 		c, _ = bidding["collect"].(string)
+	} else {
+		currentColl = c
 	}
 
-	extractc, _ := bidding["extractcollect"].(string)
-	db, _ := bidding["db"].(string)
-	extractdb, _ := bidding["extractdb"].(string)
-	index, _ := bidding["index"].(string)
-	itype, _ := bidding["type"].(string)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
-	fields := strings.Split(bidding["fields"].(string), ",")
+	extractc, _ := extract["collect"].(string)
+	count, _ := biddingConn.DB(biddingMgo.DbName).C(c).Find(&q).Count()
 	//线程池
 	UpdatesLock := sync.Mutex{}
 
-	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	qutil.Debug("查询语句:", q, "同步总数:", count)
 	//查询招标数据
-	query := session.DB(db).C(c).Find(q).Select(bson.M{
+	query := biddingConn.DB(biddingMgo.DbName).C(c).Find(q).Select(bson.M{
 		"projectinfo.attachment": 0,
 		"contenthtml":            0,
 		"publishdept":            0, // 6.30		迭代报错，字段值乱码
 	}).Sort("_id").Iter()
 	//查询抽取结果
-	extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Sort("_id").Iter()
+	extractResult := extractConn.DB(extractMgo.DbName).C(extractc).Find(q).Sort("_id").Iter()
 	n := 0
-	//更新数组
-	arr := [][]map[string]interface{}{}
-	arrEs := []map[string]interface{}{}
 	//对比两张表数据，减少查询次数
 	var compare map[string]interface{}
 	bnil := false
 	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
-		// if qutil.IntAll(tmp["dataging"]) == 1 { //dataging=1不生索引
-		// 	tmp = make(map[string]interface{})
-		// 	continue
-		// }
-		//if qutil.IntAll(tmp["repeat"]) != 0 {
-		//	esQ := `{"query": {"bool": {"must": [{"term": {"id": "`+ mongodb.BsonIdToSId(tmp["_id"]) +`"}}]}}}`
-		//	esData := elastic.Get(index, itype, esQ)
-		//	if len(*esData) > 0 {
-		//		elastic.DelById(index, itype, mongodb.BsonIdToSId(tmp["_id"]))
-		//	}
-		//}
-
 		if sensitive := qutil.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
 			tmp = make(map[string]interface{})
 			continue
@@ -109,7 +75,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 		for {
 			if compare == nil {
 				compare = make(map[string]interface{})
-				if !extractquery.Next(compare) {
+				if !extractResult.Next(compare) {
 					break
 				}
 			}
@@ -126,7 +92,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 							modifyinfo[k] = true
 						}
 					}
-					for _, k := range fields { //fields更新到mongo的字段
+					for _, k := range biddingMgoFields { //fields更新到mongo的字段
 						v1 := compare[k] //extract
 						v2 := tmp[k]     //bidding
 						if v2 == nil && v1 != nil {
@@ -173,64 +139,8 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				<-mpool
 			}()
 			if !bnil && compare != nil && len(compare) > 0 {
-				subscopeclass, _ := compare["subscopeclass"].([]interface{}) //subscopeclass
-				if subscopeclass != nil {
-					m1 := map[string]bool{}
-					newclass := []string{}
-					for _, sc := range subscopeclass {
-						sclass, _ := sc.(string)
-						if !m1[sclass] {
-							m1[sclass] = true
-							newclass = append(newclass, sclass)
-						}
-					}
-					update["s_subscopeclass"] = strings.Join(newclass, ",")
-					update["subscopeclass"] = newclass
-				}
-				topscopeclass, _ := compare["topscopeclass"].([]interface{}) //topscopeclass
-				if topscopeclass != nil {
-					m2 := map[string]bool{}
-					newclass := []string{}
-					for _, tc := range topscopeclass {
-						tclass, _ := tc.(string)
-						tclass = reg_letter.ReplaceAllString(tclass, "") // 去除字母
-						if !m2[tclass] {
-							m2[tclass] = true
-							newclass = append(newclass, tclass)
-						}
-					}
-					update["s_topscopeclass"] = strings.Join(newclass, ",")
-				}
-				//处理中标企业
-				//				winner, _ := compare["winner"].(string)
-				//				m1 := map[string]bool{}
-				//				if winner != "" {
-				//					m1[winner] = true
-				//				}
-				// 多包标记
-				if package1 := compare["package"]; package1 != nil {
-					packageM, _ := package1.(map[string]interface{})
-					for _, p := range packageM {
-						pm, _ := p.(map[string]interface{})
-						if qutil.ObjToString(pm["winner"]) != "" || qutil.Float64All(pm["budget"]) > 0 ||
-							qutil.Float64All(pm["bidamount"]) > 0 {
-							update["multipackage"] = int32(1)
-							break
-						}
-					}
-				} else {
-					update["multipackage"] = int32(0)
-				}
+				FieldMethod(compare, update)
 				compare = nil
-				//				if len(m1) > 0 {
-				//					//str := ","
-				//					winnerarr := []string{}
-				//					for k, _ := range m1 {
-				//						//str += k + ","
-				//						winnerarr = append(winnerarr, k)
-				//					}
-				//					update["s_winner"] = strings.Join(winnerarr, ",")
-				//				}
 			} else {
 				area := qutil.ObjToString(tmp["area"])
 				city := qutil.ObjToString(tmp["city"])
@@ -249,301 +159,51 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 			for tk, tv := range update {
 				tmp[tk] = tv
 			}
-			tmp_up := []map[string]interface{}{}
-			extupdate := make(map[string]interface{})
+
 			if tmp["s_winner"] != "" {
-				//sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
-				var cid []string
-				//for _, w := range sWinnerarr {
-				//	if w != "" {
-				//		id := redis.GetStr("qyxy_id", w)
-				//		if id == "" {
-				//			ents, _ := mgostandard.Find("qyxy_std", map[string]interface{}{"company_name": w}, map[string]interface{}{"updatetime": -1}, nil, false, -1, -1)
-				//			if len(*ents) > 0 {
-				//				id = qutil.ObjToString((*ents)[0]["_id"])
-				//			}else {
-				//				ent, _ := qyxydb.FindOne("company_history_name", map[string]interface{}{"history_name": w})
-				//				if len(*ent) > 0 {
-				//					id = qutil.ObjToString((*ent)["company_id"])
-				//				}
-				//			}
-				//		}
-				//		if id == "" {
-				//			id = "-"
-				//		}
-				//		cid = append(cid, id)
-				//	}
-				//}
+				cid := FieldFun(tmp)
 				if len(cid) > 0 {
 					tmp["entidlist"] = cid
 					update["entidlist"] = cid
-					extupdate["entidlist"] = cid
-				}
-			}
-			// 临时
-			//if tmp["purchasing_tag"] != nil {
-			//	extupdate["purchasing_tag"] = tmp["purchasing_tag"]
-			//}
-			if len(extupdate) > 0 {
-				tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
-				tmp_up = append(tmp_up, map[string]interface{}{"$set": extupdate})
-				UpdataMgoCache <- tmp_up
-			}
-			//对projectscope字段的索引处理
-			ps, _ := tmp["projectscope"].(string)
-			if len(ps) > ESLEN {
-				tmp["projectscope"] = string(([]rune(ps))[:4000])
-			}
-			//对标的物为空处理
-			filetext := getFileText(tmp)
-			filetextS := filetext
-			if len([]rune(filetextS)) > 10 { //attach_text
-				tmp["filetext"] = filetext
-			}
-			if purchasing, ok := tmp["purchasing"].(string); ok && purchasing == "" {
-				delete(tmp, "purchasing")
-			}
-			if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok && len(purchasinglist) == 0 {
-				delete(tmp, "purchasinglist")
-			}
-			//数据为空处理
-			for _, f := range []string{"bidstatus", "city", "district", "channel"} {
-				if fVal, ok := tmp[f].(string); ok && fVal == "" {
-					delete(tmp, f)
+					tmp_up := []map[string]interface{}{}
+					tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
+					tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
+					updateExtractPool <- tmp_up
 				}
 			}
 
-			//预算和中标金额
-			//			if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
-			//				tmp["budget"] = nil
-			//			} else if sbd, ok := tmp["budget"].(string); ok {
-			//				tmp["budget"] = ObjToMoney([]interface{}{sbd, sbd})[0]
-			//			}
-			//			if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
-			//				tmp["bidamount"] = nil
-			//			} else if sbd, ok := tmp["bidamount"].(string); ok {
-			//				tmp["bidamount"] = ObjToMoney([]interface{}{sbd, sbd})[0]
-			//			}
-
+			clearMap(tmp)
 			//go IS.Add("bidding")
 			UpdatesLock.Lock()
 			if qutil.IntAll(update["extracttype"]) != -1 {
-				newTmp := map[string]interface{}{}
-				for field, ftype := range biddingIndexFieldsMap {
-					if tmp[field] != nil && del[field] == nil {
-						//qutil.Debug(field, tmp[field], reflect.TypeOf(tmp[field]).String(), ftype)
-						if field == "projectinfo" {
-							mp, _ := tmp[field].(map[string]interface{})
-							if mp != nil {
-								newmap := map[string]interface{}{}
-								for k, ktype := range projectinfoFieldsMap {
-									mpv := mp[k]
-									if mpv != nil && reflect.TypeOf(mpv).String() == ktype {
-										newmap[k] = mp[k]
-									}
-								}
-								if len(newmap) > 0 {
-									newTmp[field] = newmap
-								}
-							}
-						} else if field == "purchasinglist" { //标的物处理
-							purchasinglist_new := []map[string]interface{}{}
-							if pcl, _ := tmp[field].([]interface{}); len(pcl) > 0 {
-								for _, ls := range pcl {
-									lsm_new := make(map[string]interface{})
-									lsm := ls.(map[string]interface{})
-									for pf, pftype := range purchasinglistFieldsMap {
-										lsmv := lsm[pf]
-										if lsmv != nil && reflect.TypeOf(lsmv).String() == pftype {
-											lsm_new[pf] = lsm[pf]
-										}
-									}
-									if lsm_new != nil && len(lsm_new) > 0 {
-										purchasinglist_new = append(purchasinglist_new, lsm_new)
-									}
-								}
-							}
-							if len(purchasinglist_new) > 0 {
-								newTmp[field] = purchasinglist_new
-							}
-						} else if field == "winnerorder" { //中标候选
-							winnerorder_new := []map[string]interface{}{}
-							if winnerorder, _ := tmp[field].([]interface{}); len(winnerorder) > 0 {
-								for _, win := range winnerorder {
-									winMap_new := make(map[string]interface{})
-									winMap := win.(map[string]interface{})
-									for wf, wftype := range winnerorderlistFieldsMap {
-										wfv := winMap[wf]
-										if wfv != nil && reflect.TypeOf(wfv).String() == wftype {
-											if wf == "sort" && qutil.Int64All(wfv) > 100 {
-												continue
-											}
-											winMap_new[wf] = winMap[wf]
-										}
-									}
-									if winMap_new != nil && len(winMap_new) > 0 {
-										winnerorder_new = append(winnerorder_new, winMap_new)
-									}
-								}
-							}
-							if len(winnerorder_new) > 0 {
-								newTmp[field] = winnerorder_new
-							}
-						} else if field == "qualifies" {
-							//项目资质
-							qs := []string{}
-							if q, _ := tmp[field].([]interface{}); len(q) > 0 {
-								for _, v := range q {
-									v1 := v.(map[string]interface{})
-									qs = append(qs, qutil.ObjToString(v1["key"]))
-								}
-							}
-							if len(qs) > 0 {
-								newTmp[field] = strings.Join(qs, ",")
-							}
-						} else if field == "review_experts" {
-							// 评审专家
-							if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
-								arr1 := qutil.ObjArrToStringArr(arr)
-								newTmp[field] = strings.Join(arr1, ",")
-							}
-						} else if field == "entidlist" {
-							newTmp[field] = tmp[field]
-						} else if field == "bidopentime" {
-							if tmp[field] != nil && tmp["bidendtime"] == nil {
-								newTmp["bidendtime"] = tmp[field]
-								newTmp[field] = tmp[field]
-							} else if tmp[field] == nil && tmp["bidendtime"] != nil {
-								newTmp["bidendtime"] = tmp[field]
-								newTmp[field] = tmp["bidendtime"]
-							} else {
-								if tmp["bidopentime"] != nil {
-									newTmp[field] = tmp["bidopentime"]
-								}
-							}
-						} else if field == "detail" { //过滤
-							detail, _ := tmp[field].(string)
-							if len([]rune(detail)) > detailLength {
-								detail = detail[:detailLength]
-							}
-							newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
-						} else if field == "_id" || field == "topscopeclass" { //不做处理
-							newTmp[field] = tmp[field]
-						} else if field == "publishtime" || field == "comeintime" {
-							//字段类型不正确，特别处理
-							if tmp[field] != nil && qutil.Int64All(tmp[field]) > 0 {
-								newTmp[field] = qutil.Int64All(tmp[field])
-							}
-						} else { //其它字段判断数据类型，不正确舍弃
-							if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {
-								continue
-							} else {
-								if fieldval != "" {
-									newTmp[field] = fieldval
-								}
-							}
-						}
-					}
-				}
-				YuceEndtime(newTmp)
-				arrEs = append(arrEs, newTmp)
+				newTmp := t.GetEsField(tmp, update)
+				saveEsPool <- newTmp
 			}
 			if len(update) > 0 {
 				delete(update, "winnerorder") //winnerorder不需要更新到bindding表，删除
-				queryId := map[string]interface{}{"_id": tmp["_id"]}
-				set := map[string]interface{}{"$set": update}
-				if len(del) > 0 { //删除的数据
-					set["$unset"] = del
-				}
-				arr = append(arr, []map[string]interface{}{queryId, set})
-			}
-			if len(arr) >= BulkSize {
-				mgo.UpdateBulkAll(db, c, arr...)
-				arr = [][]map[string]interface{}{}
-			}
-			if len(arrEs) >= BulkSize {
-				tmps := arrEs
-				if StopFlag {
-					qutil.Debug("es队列紧张，暂停10s执行")
-					time.Sleep(time.Second * 10)
+				if len(del) > 0 {             //删除的数据
+					updateBiddingPool <- []map[string]interface{}{{
+						"_id": tmp["_id"],
+					},
+						{"$set": update, "$unset": del},
+					}
+				} else {
+					updateBiddingPool <- []map[string]interface{}{{
+						"_id": tmp["_id"],
+					},
+						{"$set": update},
+					}
 				}
-				elastic.BulkSave(index, itype, &tmps, true)
-				//if len(multiIndex) == 2 {
-				//	elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
-				//}
-				//if other_index != "" && other_itype != "" { //备份库同时生索引
-				//	bidding_other_es.BulkSave(other_index, other_itype, &tmps, true)
-				//}
-				arrEs = []map[string]interface{}{}
 			}
+
 			UpdatesLock.Unlock()
 		}(tmp, update, compare, del, bnil)
-		if n%10000 == 0 {
-			log.Println("current:", n, _id)
+		if n%20000 == 0 {
+			qutil.Debug("current:", n, _id)
 		}
 		tmp = make(map[string]interface{})
 	}
-	for i := 0; i < thread; i++ {
-		mpool <- true
-	}
-	UpdatesLock.Lock()
-	if len(arr) > 0 {
-		mgo.UpdateBulkAll(db, c, arr...)
-	}
-	if len(arrEs) > 0 {
-		tmps := arrEs
-		if StopFlag {
-			qutil.Debug("es队列紧张，暂停10s执行")
-			time.Sleep(time.Second * 10)
-		}
-		elastic.BulkSave(index, itype, &tmps, true)
-		//if len(multiIndex) == 2 {
-		//	elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
-		//}
-		//if other_index != "" && other_itype != "" { //备份库同时生索引
-		//	bidding_other_es.BulkSave(other_index, other_itype, &tmps, true)
-		//}
-	}
-	UpdatesLock.Unlock()
-	log.Println(mapInfo, "create bidding index...over", n)
-}
-
-//更新extract表
-func UpdateExtract() {
-	qutil.Debug("Update Extract...")
-	extract := qutil.ObjToString(bidding["extractcollect"])
-	arru := make([][]map[string]interface{}, 200)
-	indexu := 0
-	for {
-		select {
-		case v := <-UpdataMgoCache:
-			arru[indexu] = v
-			indexu++
-			if indexu == 200 {
-				SP <- true
-				go func(arru [][]map[string]interface{}) {
-					defer func() {
-						<-SP
-					}()
-					extractmgo.UpdateBulk(extract, arru...)
-				}(arru)
-				arru = make([][]map[string]interface{}, 200)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				SP <- true
-				go func(arru [][]map[string]interface{}) {
-					defer func() {
-						<-SP
-					}()
-					extractmgo.UpdateBulk(extract, arru...)
-				}(arru[:indexu])
-				arru = make([][]map[string]interface{}, 200)
-				indexu = 0
-			}
-		}
-	}
+	qutil.Debug(mapInfo, "create bidding index...over", n)
 }
 
 //城市标准校验

udpcreateindex/src/biddingdata.go

@@ -2,20 +2,12 @@ package main
 
 import (
 	"go.mongodb.org/mongo-driver/bson"
-	"log"
 	"mongodb"
 	qutil "qfw/util"
-	elastic "qfw/util/elastic"
-	"reflect"
-	"strings"
-
-	//	"strings"
-	"sync"
 )
 
-func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
+func (t *TaskInfo) biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 	defer qutil.Catch()
-	qutil.Debug(mapInfo)
 	q, _ := mapInfo["query"].(map[string]interface{})
 	if q == nil {
 		q = map[string]interface{}{
@@ -35,23 +27,19 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 		q["_id"] = tmpQ
 	}
 	//bidding库
-	session := mgo.GetMgoConn()
-	defer mgo.DestoryMongoConn(session)
-	//连接信息
+	biddingConn := biddingMgo.GetMgoConn()
+	defer biddingMgo.DestoryMongoConn(biddingConn)
 	//连接信息
 	c, _ := mapInfo["coll"].(string)
 	if c == "" {
 		c, _ = bidding["collect"].(string)
+	} else {
+		currentColl = c
 	}
-	db, _ := bidding["db"].(string)
-	index := "bidding_all"
-	itype := "bidding"
-	count, _ := session.DB(db).C(c).Find(&q).Count()
-	//线程池
-	UpdatesLock := sync.Mutex{}
-	qutil.Debug("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	count, _ := biddingConn.DB(biddingMgo.DbName).C(c).Find(&q).Count()
+	qutil.Debug("查询语句:", q, "同步总数:", count, "elastic库:")
 	//查询招标数据
-	query := session.DB(db).C(c).Find(q).Select(bson.M{
+	query := biddingConn.DB(biddingMgo.DbName).C(c).Find(q).Select(bson.M{
 		"projectinfo.attachment": 0,
 		"contenthtml":            0,
 		"publishdept":            0,
@@ -60,167 +48,25 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 		"project_scale":          0,
 		"bidstatus":              0,
 		"china_bidding":          0,
+		"purchasinglist":         0,
 	}).Sort("_id").Iter()
 	n := 0
 	//更新数组
-	arrEs := []map[string]interface{}{}
-	thread := 10
-	espool := make(chan bool, 5)
-	var mpool = make(chan bool, thread)
+	var mpool = make(chan bool, t.thread)
 	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
 		if n%20000 == 0 {
-			log.Println("current:", n, tmp["_id"])
+			qutil.Debug("current:", n, tmp["_id"])
 		}
 		mpool <- true
 		go func(tmp map[string]interface{}) {
 			defer func() {
 				<-mpool
 			}()
-			if purchasing, ok := tmp["purchasing"].(string); ok && purchasing == "" {
-				delete(tmp, "purchasing")
-			}
-			if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok && len(purchasinglist) == 0 {
-				delete(tmp, "purchasinglist")
-			}
-			//数据为空处理
-			for _, f := range []string{"bidstatus", "city", "district", "channel"} {
-				if fVal, ok := tmp[f].(string); ok && fVal == "" {
-					delete(tmp, f)
-				}
-			}
-			UpdatesLock.Lock()
-			newTmp := map[string]interface{}{}
-			for field, ftype := range biddingIndexFieldsMap {
-				if tmp[field] != nil { //
-					if field == "projectinfo" {
-						mp, _ := tmp[field].(map[string]interface{})
-						if mp != nil {
-							newmap := map[string]interface{}{}
-							for k, ktype := range projectinfoFieldsMap {
-								mpv := mp[k]
-								if mpv != nil && reflect.TypeOf(mpv).String() == ktype {
-									newmap[k] = mp[k]
-								}
-							}
-							if len(newmap) > 0 {
-								newTmp[field] = newmap
-							}
-						}
-					} else if field == "purchasinglist" { //标的物处理
-						purchasinglist_new := []map[string]interface{}{}
-						if pcl, _ := tmp[field].([]interface{}); len(pcl) > 0 {
-							for _, ls := range pcl {
-								lsm_new := make(map[string]interface{})
-								lsm := ls.(map[string]interface{})
-								for pf, pftype := range purchasinglistFieldsMap {
-									lsmv := lsm[pf]
-									if lsmv != nil && reflect.TypeOf(lsmv).String() == pftype {
-										lsm_new[pf] = lsm[pf]
-									}
-								}
-								if lsm_new != nil && len(lsm_new) > 0 {
-									purchasinglist_new = append(purchasinglist_new, lsm_new)
-								}
-							}
-						}
-						if len(purchasinglist_new) > 0 {
-							newTmp[field] = purchasinglist_new
-						}
-					} else if field == "winnerorder" { //中标候选
-						winnerorder_new := []map[string]interface{}{}
-						if winnerorder, _ := tmp[field].([]interface{}); len(winnerorder) > 0 {
-							for _, win := range winnerorder {
-								winMap_new := make(map[string]interface{})
-								winMap := win.(map[string]interface{})
-								for wf, wftype := range winnerorderlistFieldsMap {
-									wfv := winMap[wf]
-									if wfv != nil && reflect.TypeOf(wfv).String() == wftype {
-										if wf == "sort" && qutil.Int64All(wfv) > 100 {
-											continue
-										}
-										winMap_new[wf] = winMap[wf]
-									}
-								}
-								if winMap_new != nil && len(winMap_new) > 0 {
-									winnerorder_new = append(winnerorder_new, winMap_new)
-								}
-							}
-						}
-						if len(winnerorder_new) > 0 {
-							newTmp[field] = winnerorder_new
-						}
-					} else if field == "qualifies" {
-						//项目资质
-						qs := []string{}
-						if q, _ := tmp[field].([]interface{}); len(q) > 0 {
-							for _, v := range q {
-								v1 := v.(map[string]interface{})
-								qs = append(qs, qutil.ObjToString(v1["key"]))
-							}
-						}
-						if len(qs) > 0 {
-							newTmp[field] = strings.Join(qs, ",")
-						}
-					} else if field == "review_experts" {
-						// 评审专家
-						if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
-							arr1 := qutil.ObjArrToStringArr(arr)
-							newTmp[field] = strings.Join(arr1, ",")
-						}
-					} else if field == "entidlist" {
-						newTmp[field] = tmp[field]
-					} else if field == "bidopentime" {
-						if tmp[field] != nil && tmp["bidendtime"] == nil {
-							newTmp["bidendtime"] = tmp[field]
-						}
-						if tmp[field] == nil && tmp["bidendtime"] != nil {
-							newTmp[field] = tmp["bidendtime"]
-						}
-					} else if field == "_id" || field == "topscopeclass" { //不做处理
-						newTmp[field] = tmp[field]
-					} else if field == "publishtime" || field == "comeintime" {
-						//字段类型不正确，特别处理
-						if tmp[field] != nil && qutil.Int64All(tmp[field]) > 0 {
-							newTmp[field] = qutil.Int64All(tmp[field])
-						}
-					} else if field == "s" {
-						newTmp[field] = tmp[field]
-					} else { //其它字段判断数据类型，不正确舍弃
-						if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {
-							continue
-						} else {
-							if fieldval != "" {
-								newTmp[field] = fieldval
-							}
-						}
-					}
-				}
-			}
+			newTmp := t.GetEsField(tmp, nil)
 			newTmp["extracttype"] = qutil.IntAll(tmp["extracttype"])
-			arrEs = append(arrEs, newTmp)
-			if len(arrEs) >= BulkSizeBack {
-				tmps := arrEs
-				espool <- true
-				go func(tmps []map[string]interface{}) {
-					defer func() {
-						<-espool
-					}()
-					elastic.BulkSave(index, itype, &tmps, true)
-				}(tmps)
-				arrEs = []map[string]interface{}{}
-			}
-			UpdatesLock.Unlock()
+			saveEsAllPool <- newTmp
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
-	for i := 0; i < thread; i++ {
-		mpool <- true
-	}
-	UpdatesLock.Lock()
-	if len(arrEs) > 0 {
-		tmps := arrEs
-		elastic.BulkSave(index, itype, &tmps, true)
-	}
-	UpdatesLock.Unlock()
-	log.Println(mapInfo, "create biddingdata index...over", n)
+	qutil.Debug(mapInfo, "create biddingdata index...over", n)
 }

udpcreateindex/src/biddingdeletebyextract.go

@@ -7,7 +7,6 @@ import (
 	qutil "qfw/util"
 
 	elastic "qfw/util/elastic"
-
 )
 
 //根据抽取表repeat=1，删除es中重复数据
@@ -25,18 +24,17 @@ func biddingDelByExtract(data []byte, mapInfo map[string]interface{}) {
 			"repeat": 1,
 		}
 	}
-	extractc, _ := bidding["extractcollect"].(string)
-	extractdb, _ := bidding["extractdb"].(string)
+	extractc, _ := extract["collect"].(string)
 	index, _ := bidding["index"].(string)
 	itype, _ := bidding["type"].(string)
 
 	//extract
-	extractsession := extractmgo.GetMgoConn()
-	defer extractmgo.DestoryMongoConn(extractsession)
+	extractConn := extractMgo.GetMgoConn()
+	defer extractMgo.DestoryMongoConn(extractConn)
 	//查询数据
-	count, _ := extractsession.DB(extractdb).C(extractc).Find(&q).Count()
+	count, _ := extractConn.DB(extractMgo.DbName).C(extractc).Find(&q).Count()
 	log.Println("查询语句:", q, "删除同步总数:", count, "elastic库:", index)
-	extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Select(
+	extractquery := extractConn.DB(extractMgo.DbName).C(extractc).Find(q).Select(
 		bson.M{"_id": 1},
 	).Sort("_id").Iter()
 	log.Println("开始迭代...")
@@ -49,11 +47,11 @@ func biddingDelByExtract(data []byte, mapInfo map[string]interface{}) {
 		if elastic.DelById(index, itype, _id) { //删除
 			dnum++
 		}
-		if other_index != "" && other_itype != "" {
-			bidding_other_es.DelById(other_index, other_itype, _id)
-		}
+		//if other_index != "" && other_itype != "" {
+		//	bidding_other_es.DelById(other_index, other_itype, _id)
+		//}
 
-		if n%savesizei == 0 {
+		if n%200 == 0 {
 			log.Println("当前:", n)
 		}
 		tmp = make(map[string]interface{})

udpcreateindex/src/biddingdeletebyextracttype.go

@@ -5,9 +5,6 @@ import (
 	"log"
 	"mongodb"
 	qutil "qfw/util"
-
-	elastic "qfw/util/elastic"
-
 )
 
 //根据bidding表extracttype=-1，删除es中重复数据
@@ -25,18 +22,15 @@ func biddingDelByExtracttype(data []byte, mapInfo map[string]interface{}) {
 			"extracttype": -1,
 		}
 	}
-	db, _ := bidding["db"].(string)
 	c, _ := bidding["collect"].(string)
-	index, _ := bidding["index"].(string)
-	itype, _ := bidding["type"].(string)
 
 	//bidding
-	session := mgo.GetMgoConn()
-	defer mgo.DestoryMongoConn(session)
+	conn := biddingMgo.GetMgoConn()
+	defer biddingMgo.DestoryMongoConn(conn)
 	//查询数据
 	//count, _ := session.DB(db).C(c).Find(&q).Count()
 	//log.Println("查询语句:", q, "删除同步总数:", count, "elastic库:", index)
-	biddingquery := session.DB(db).C(c).Find(q).Select(
+	biddingquery := conn.DB(biddingMgo.DbName).C(c).Find(q).Select(
 		bson.M{"_id": 1},
 	).Sort("_id").Iter()
 	log.Println("开始迭代...")
@@ -46,14 +40,11 @@ func biddingDelByExtracttype(data []byte, mapInfo map[string]interface{}) {
 	for tmp := make(map[string]interface{}); biddingquery.Next(tmp); i = i + 1 {
 		n++
 		_id := mongodb.BsonIdToSId(tmp["_id"])
-		if elastic.DelById(index, itype, _id) { //删除
+		if Es2.DelById(qutil.ObjToString(biddingIndex["index"]), qutil.ObjToString(biddingIndex["type"]), _id) { //删除
+			//Es2.DelById(qutil.ObjToString(biddingIndex["idnex"]), qutil.ObjToString(biddingIndex["type"]), _id)
 			dnum++
 		}
-		if other_index != "" && other_itype != "" {
-			bidding_other_es.DelById(other_index, other_itype, _id)
-		}
-
-		if n%savesizei == 0 {
+		if n%200 == 0 {
 			log.Println("当前:", n)
 		}
 		tmp = make(map[string]interface{})

udpcreateindex/src/biddingindex.go

@@ -2,29 +2,19 @@ package main
 
 import (
 	"encoding/json"
-	"go.mongodb.org/mongo-driver/bson"
-	"mongodb"
-	"qfw/util/redis"
-	"reflect"
-	//"fmt"
-	"log"
 	mu "mfw/util"
-	"net"
-	qutil "qfw/util"
-	elastic "qfw/util/elastic"
+	"mongodb"
+	"qfw/util"
 	"regexp"
 	"strings"
-	"sync"
 	"time"
 )
 
-var date1 = regexp.MustCompile("20[0-2][0-9][年|\\-\\/|.][0-9]{1,2}[月|\\-|\\/|.][0-9]{1,2}[日]?")
-
-//对字段处理 bidamount  budget
 //招标数据表和抽取表一一对应开始更新
 
-func biddingTask(data []byte, mapInfo map[string]interface{}, tasktype string) {
-	defer qutil.Catch()
+func (t *TaskInfo) biddingTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+
 	q, _ := mapInfo["query"].(map[string]interface{})
 	bkey, _ := mapInfo["bkey"].(string)
 	if q == nil {
@@ -35,107 +25,88 @@ func biddingTask(data []byte, mapInfo map[string]interface{}, tasktype string) {
 			},
 		}
 	}
-
-	//logger.SetRollingDaily("./logs", "id.log")
-	//连接信息
-	c, _ := bidding["collect"].(string)
-	extractc, _ := bidding["extractcollect"].(string)
-	db, _ := bidding["db"].(string)
-	extractdb, _ := bidding["extractdb"].(string)
-	index, _ := bidding["index"].(string)
-	itype, _ := bidding["type"].(string)
-
 	//extract库
-	extractsession := extractmgo.GetMgoConn()
-	defer extractmgo.DestoryMongoConn(extractsession)
-	extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Sort("_id").Iter()
+	extractConn := extractMgo.GetMgoConn()
+	defer extractMgo.DestoryMongoConn(extractConn)
+	extractc, _ := extract["collect"].(string)
+	extractResult := extractConn.DB(extractMgo.DbName).C(extractc).Find(q).Sort("_id").Iter()
 	eMap := map[string]map[string]interface{}{}
 	extCount, repeatCount := 0, 0
-	for tmp := make(map[string]interface{}); extractquery.Next(tmp); extCount++ {
-		if qutil.IntAll(tmp["repeat"]) == 1 {
+	for tmp := make(map[string]interface{}); extractResult.Next(tmp); extCount++ {
+		if util.IntAll(tmp["repeat"]) == 1 {
 			repeatCount++
 		}
 		tid := mongodb.BsonIdToSId(tmp["_id"])
 		eMap[tid] = tmp
 		tmp = make(map[string]interface{})
 	}
+	util.Debug("抽取表 重复数据量：", extCount, repeatCount)
 
 	//bidding库
-	session := mgo.GetMgoConn()
-	count, _ := session.DB(db).C(c).Find(&q).Count()
-	log.Println("抽取表 重复数据量：", extCount, repeatCount)
-	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	biddingConn := biddingMgo.GetMgoConn()
+	c, _ := bidding["collect"].(string)
+	count, _ := biddingConn.DB(biddingMgo.DbName).C(c).Find(&q).Count()
+	util.Debug("查询语句:", q, "同步总数:", count)
 	n1, n2 := 0, 0
 	if count < 200000 {
 		var res []map[string]interface{}
-		//res := make([]map[string]interface{}, 1)
-		result := session.DB(db).C(c).Find(q).Select(map[string]interface{}{
-			"projectinfo.attachment": 0,
-			"contenthtml":            0,
+		result := biddingConn.DB(biddingMgo.DbName).C(c).Find(q).Select(map[string]interface{}{
+			"contenthtml": 0,
 		}).Iter()
 		for tmp := make(map[string]interface{}); result.Next(tmp); {
 			res = append(res, tmp)
 			tmp = make(map[string]interface{})
 		}
-		mgo.DestoryMongoConn(session)
-		log.Println("查询结果", "bidding:", count, "抽取:", extCount)
-		if int64(len(res)) != count {
-			time.Sleep(20 * time.Second)
-			toadd := &net.UDPAddr{
-				IP:   net.ParseIP("127.0.0.1"),
-				Port: qutil.IntAll(Sysconfig["udpport"]),
-			}
-			udpclient.WriteUdp(data, mu.OP_TYPE_DATA, toadd)
-		}
-		n1, n2 = doIndex(res, eMap, index, itype, db, c, bkey, tasktype)
-		//if int64(n1 + n2) != count {
-		//	log.Println("任务错误，结果不一致")
+		biddingMgo.DestoryMongoConn(biddingConn)
+		util.Debug("查询结果", "bidding:", count, "抽取:", extCount)
+		//if int64(len(res)) != count {
+		//	time.Sleep(20 * time.Second)
+		//	toadd := &net.UDPAddr{
+		//		IP:   net.ParseIP("127.0.0.1"),
+		//		Port: util.IntAll(Sysconfig["udpport"]),
+		//	}
+		//	udpclient.WriteUdp(data, mu.OP_TYPE_DATA, toadd)
 		//}
+		n1, n2 = t.doIndex(res, eMap, bkey)
 	} else {
-		log.Println("数据量太大，放弃！", count)
-		mgo.DestoryMongoConn(session)
+		util.Debug("数据量太大，放弃！", count)
+		biddingMgo.DestoryMongoConn(biddingConn)
 	}
-	log.Println(mapInfo, "create bidding index...over", "all:", count, "bidding size:", n1, "，es size:", n2)
-	if tasktype == "bidding_history" {
-		qutil.Debug(tasktype)
+	util.Debug(mapInfo, "create bidding index...over", "all:", count, "bidding size:", n1, "，es size:", n2)
+	if t.stype == "bidding_history" {
 		// 历史判重id段结束之后 生全量数据索引
-		biddingDataTask(data, mapInfo)
+		t.stype = "biddingdata"
+		t.thread = 30
+		t.biddingDataTask(data, mapInfo)
 	}
 }
 
-func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interface{}, index, itype, db, c, bkey, tasktype string) (int, int) {
+func (t *TaskInfo) doIndex(infos []map[string]interface{}, eMap map[string]map[string]interface{}, bkey string) (int, int) {
 	n1, n2 := 0, 0 //bidding数量，索引数量
-	//线程池
-	UpdatesLock := sync.Mutex{}
-	fields := strings.Split(bidding["fields"].(string), ",")
-	//更新数组
-	arr := [][]map[string]interface{}{}
-	arrEs := []map[string]interface{}{}
 	//对比两张表数据，减少查询次数
 	var compare map[string]interface{}
-	log.Println("开始迭代..")
+	util.Debug("start ...")
 	for n, tmp := range infos {
 		n1++
-		if sensitive := qutil.ObjToString(tmp["sensitive"]); sensitive == "测试" || sensitive == "异常" { //bidding中有敏感词，不生索引
+		if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
 			tmp = make(map[string]interface{})
 			continue
 		}
 		tid := mongodb.BsonIdToSId(tmp["_id"])
-		//loginfo := make(map[string]interface{}) // 日志
 		update := map[string]interface{}{} //要更新的mongo数据
 		//对比方法----------------
 		if eMap[tid] != nil {
 			compare = eMap[tid]
-			if tasktype == "bidding" {
+			if t.stype == "bidding" {
 				// 增量id段 正常数据
-				if num := qutil.IntAll(compare["dataging"]); num == 1 { //extract中dataging=1跳过
+				if num := util.IntAll(compare["dataging"]); num == 1 { //extract中dataging=1跳过
 					tmp = make(map[string]interface{})
 					compare = nil
 					continue
 				}
 				delete(eMap, tid)
 			}
-			if tasktype == "bidding_history" {
+			if t.stype == "bidding_history" {
 				//增量id段 历史数据
 				if compare["history_updatetime"] == nil { //extract中history_updatetime不存在跳过
 					tmp = make(map[string]interface{})
@@ -152,7 +123,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 				}
 			}
 			//更新bidding表，生成索引
-			for _, k := range fields {
+			for _, k := range biddingMgoFields {
 				v1 := compare[k] //extract
 				v2 := tmp[k]     //bidding
 				if v2 == nil && v1 != nil && !modifyinfo[k] {
@@ -167,68 +138,26 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 					}
 				}
 			}
-			if qutil.IntAll(compare["repeat"]) == 1 {
+			if util.IntAll(compare["repeat"]) == 1 {
 				update["extracttype"] = -1
 			} else {
 				update["extracttype"] = 1
 			}
 		} else {
 			compare = nil
-			if qutil.IntAll(tmp["dataging"]) == 1 { //修改未抽取的bidding数据的dataging
+			if util.IntAll(tmp["dataging"]) == 1 { //修改未抽取的bidding数据的dataging
 				update["dataging"] = 0
 			}
 		}
 		//下面可以多线程跑的--->
 		//处理分类
 		if compare != nil { //extract
-			subscopeclass, _ := compare["subscopeclass"].([]interface{}) //subscopeclass
-			if subscopeclass != nil {
-				//str := ","
-				m1 := map[string]bool{}
-				newclass := []string{}
-				for _, sc := range subscopeclass {
-					sclass, _ := sc.(string)
-					if !m1[sclass] {
-						m1[sclass] = true
-						//str += sclass + ","
-						newclass = append(newclass, sclass)
-					}
-				}
-				update["s_subscopeclass"] = strings.Join(newclass, ",")
-				update["subscopeclass"] = newclass
-			}
-			topscopeclass, _ := compare["topscopeclass"].([]interface{}) //topscopeclass
-			if topscopeclass != nil {
-				m2 := map[string]bool{}
-				newclass := []string{}
-				for _, tc := range topscopeclass {
-					tclass, _ := tc.(string)
-					tclass = reg_letter.ReplaceAllString(tclass, "") // 去除字母
-					if !m2[tclass] {
-						m2[tclass] = true
-						newclass = append(newclass, tclass)
-					}
-				}
-				update["s_topscopeclass"] = strings.Join(newclass, ",")
-			}
-			if package1 := compare["package"]; package1 != nil {
-				packageM, _ := package1.(map[string]interface{})
-				for _, p := range packageM {
-					pm, _ := p.(map[string]interface{})
-					if qutil.ObjToString(pm["winner"]) != "" || qutil.Float64All(pm["budget"]) > 0 ||
-						qutil.Float64All(pm["bidamount"]) > 0 {
-						update["multipackage"] = 1
-						break
-					}
-				}
-			} else {
-				update["multipackage"] = 0
-			}
+			FieldMethod(compare, update)
 			compare = nil
 		} else {
-			area := qutil.ObjToString(tmp["area"])
-			city := qutil.ObjToString(tmp["city"])
-			district := qutil.ObjToString(tmp["district"])
+			area := util.ObjToString(tmp["area"])
+			city := util.ObjToString(tmp["city"])
+			district := util.ObjToString(tmp["district"])
 			rdata := standardCheckCity(area, city, district)
 			if len(rdata) > 0 {
 				for k, v := range rdata {
@@ -246,260 +175,57 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		for tk, tv := range update {
 			tmp[tk] = tv
 		}
+
+		extractMap := make(map[string]interface{})
 		if tmp["s_winner"] != "" {
-			sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
-			var cid []string
-			for _, w := range sWinnerarr {
-				if w != "" {
-					id := redis.GetStr("qyxy_id", w)
-					if id == "" {
-						ents, _ := mgostandard.Find("qyxy_std", bson.M{"company_name": w}, bson.M{"updatetime": -1}, nil, false, -1, -1)
-						if len(*ents) > 0 {
-							id = qutil.ObjToString((*ents)[0]["_id"])
-							redis.PutCKV("qyxy_id", w, id)
-						} else {
-							ent, _ := qyxydb.FindOne("company_history_name", bson.M{"history_name": w})
-							if len(*ent) > 0 {
-								id = qutil.ObjToString((*ent)["company_id"])
-								redis.PutCKV("qyxy_id", w, id)
-							}
-						}
-					}
-					if id == "" {
-						id = "-"
-					}
-					cid = append(cid, id)
-					//ent, _ := mgostandard.FindOne("qyxy_historyname", map[string]interface{}{"company_name": w})
-					//if len(*ent) > 0 {
-					//	cid = append(cid, qutil.ObjToString((*ent)["company_id"]))
-					//}else {
-					//	ent, _ = mgostandard.FindOne("qyxy_std", map[string]interface{}{"company_name": w})
-					//	if len(*ent) > 0 {
-					//		cid = append(cid, qutil.ObjToString((*ent)["_id"]))
-					//	}
-					//}
-				}
-			}
+			cid := FieldFun(tmp)
 			if len(cid) > 0 {
 				tmp["entidlist"] = cid
 				update["entidlist"] = cid
-				tmp_up := []map[string]interface{}{}
-				tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
-				tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
-				UpdataMgoCache <- tmp_up
+				extractMap["entidlist"] = cid
 			}
 		}
-		//对projectscope字段的索引处理
-		ps, _ := tmp["projectscope"].(string)
-		if len(ps) > ESLEN {
-			tmp["projectscope"] = string(([]rune(ps))[:4000])
-		}
-		//对标的物为空处理
-		if filetext := getFileText(tmp); len(filetext) > 10 { //attach_text
-			// if site, _ := tmp["site"].(string); site == "中国招标投标公共服务平台" { //site:中国招标投标公共服务平台 detail替换成filetext 并加入标记filedetail=1
-			// 	tmp["detail"] = filetext    //更新es中detail
-			// 	update["detail"] = filetext //更新mongo中detail
-			// 	update["filedetail"] = 1    //mongo中打标记
-			// }
-			tmp["filetext"] = filetext
-		}
-		if purchasing, ok := tmp["purchasing"].(string); ok && purchasing == "" {
-			delete(tmp, "purchasing")
-		}
-		if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok && len(purchasinglist) == 0 {
-			delete(tmp, "purchasinglist")
-		}
-		//数据为空处理
-		for _, f := range []string{"bidstatus", "city", "district", "channel"} {
-			if fVal, ok := tmp[f].(string); ok && fVal == "" {
-				delete(tmp, f)
+		// 6.10 剑鱼发布信息分类处理, 写在这里是为了修改抽取表
+		TypeMethod(tmp, update, extractMap)
+		if len(extractMap) > 0 {
+			if extractMap["toptype"] != nil && extractMap["subtype"] == nil {
+				updateExtractPool <- []map[string]interface{}{
+					{"_id": tmp["_id"]},
+					{"$set": extractMap, "$unset": map[string]interface{}{"subtype": ""}},
+				}
+			} else {
+				updateExtractPool <- []map[string]interface{}{
+					{"_id": tmp["_id"]},
+					{"$set": extractMap},
+				}
 			}
 		}
-		UpdatesLock.Lock()
-		//		for k1, _ := range tmp {
-		//			if strings.HasSuffix(k1, "_b") || k1 == "contenthtml" {
-		//				delete(tmp, k1)
-		//			}
-		//		}
-		go IS.Add("bidding")
-		if qutil.IntAll(update["extracttype"]) != -1 {
-			n2++
-			newTmp := map[string]interface{}{}                //最终生索引的数据
-			for field, ftype := range biddingIndexFieldsMap { //
-				if tmp[field] != nil { //
-					if field == "projectinfo" {
-						mp, _ := tmp[field].(map[string]interface{})
-						if mp != nil {
-							newmap := map[string]interface{}{}
-							for k, ktype := range projectinfoFieldsMap {
-								mpv := mp[k]
-								if mpv != nil && reflect.TypeOf(mpv).String() == ktype {
-									newmap[k] = mp[k]
-								}
-							}
-							if len(newmap) > 0 {
-								newTmp[field] = newmap
-							}
-						}
-					} else if field == "purchasinglist" { //标的物处理
-						purchasinglist_new := []map[string]interface{}{}
-						if pcl, _ := tmp[field].([]interface{}); len(pcl) > 0 {
-							for _, ls := range pcl {
-								lsm_new := make(map[string]interface{})
-								lsm := ls.(map[string]interface{})
-								for pf, pftype := range purchasinglistFieldsMap {
-									lsmv := lsm[pf]
-									if lsmv != nil && reflect.TypeOf(lsmv).String() == pftype {
-										lsm_new[pf] = lsm[pf]
-									}
-								}
-								if lsm_new != nil && len(lsm_new) > 0 {
-									purchasinglist_new = append(purchasinglist_new, lsm_new)
-								}
-							}
-						}
-						if len(purchasinglist_new) > 0 {
-							newTmp[field] = purchasinglist_new
-						}
-					} else if field == "winnerorder" { //中标候选
-						winnerorder_new := []map[string]interface{}{}
-						if winnerorder, _ := tmp[field].([]interface{}); len(winnerorder) > 0 {
-							for _, win := range winnerorder {
-								winMap_new := make(map[string]interface{})
-								winMap := win.(map[string]interface{})
-								for wf, wftype := range winnerorderlistFieldsMap {
-									wfv := winMap[wf]
-									if wfv != nil && reflect.TypeOf(wfv).String() == wftype {
-										if wf == "sort" && qutil.Int64All(wfv) > 100 {
-											continue
-										}
-										winMap_new[wf] = winMap[wf]
-									}
-								}
-								if winMap_new != nil && len(winMap_new) > 0 {
-									winnerorder_new = append(winnerorder_new, winMap_new)
-								}
-							}
-						}
-						if len(winnerorder_new) > 0 {
-							newTmp[field] = winnerorder_new
-						}
-					} else if field == "qualifies" {
-						//项目资质
-						qs := []string{}
-						if q, _ := tmp[field].([]interface{}); len(q) > 0 {
-							for _, v := range q {
-								v1 := v.(map[string]interface{})
-								qs = append(qs, qutil.ObjToString(v1["key"]))
-							}
-						}
-						if len(qs) > 0 {
-							newTmp[field] = strings.Join(qs, ",")
-						}
-					} else if field == "detail" { //过滤
-						detail, _ := tmp[field].(string)
-						if len([]rune(detail)) > detailLength {
-							detail = detail[:detailLength]
-						}
-						newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
-					} else if field == "_id" || field == "topscopeclass" { //不做处理
-						newTmp[field] = tmp[field]
-					} else if field == "publishtime" || field == "comeintime" {
-						//字段类型不正确，特别处理
-						if tmp[field] != nil && qutil.Int64All(tmp[field]) > 0 {
-							newTmp[field] = qutil.Int64All(tmp[field])
-						}
 
-					} else if field == "review_experts" {
-						// 评审专家
-						if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
-							arr1 := qutil.ObjArrToStringArr(arr)
-							newTmp[field] = strings.Join(arr1, ",")
-						}
-					} else if field == "entidlist" {
-						newTmp[field] = tmp[field]
-					} else if field == "bidopentime" {
-						if tmp[field] != nil && tmp["bidendtime"] == nil {
-							newTmp["bidendtime"] = tmp[field]
-							newTmp[field] = tmp[field]
-						} else if tmp[field] == nil && tmp["bidendtime"] != nil {
-							newTmp["bidendtime"] = tmp["bidendtime"]
-							newTmp[field] = tmp["bidendtime"]
-						} else {
-							if tmp["bidopentime"] != nil {
-								newTmp[field] = tmp["bidopentime"]
-							}
-						}
-					} else { //其它字段判断数据类型，不正确舍弃
-						if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {
-							continue
-						} else {
-							if fieldval != "" {
-								newTmp[field] = fieldval
-							}
-						}
-					}
-				}
-			}
-			YuceEndtime(newTmp)                      // 预测结果时间
-			newTmp["createtime"] = time.Now().Unix() // es库数据创建时间，只有增量数据有
-			if qutil.ObjToString(newTmp["spidercode"]) == "a_jyxxfbpt_gg" {
+		clearMap(tmp)
+		//go IS.Add("bidding")
+		if util.IntAll(update["extracttype"]) != -1 {
+			n2++
+			newTmp := t.GetEsField(tmp, update)
+			if util.ObjToString(newTmp["spidercode"]) == "a_jyxxfbpt_gg" {
 				// 剑鱼信息发布数据 通过udp通知信息发布程序
 				go UdpMethod(mongodb.BsonIdToSId(newTmp["_id"]))
 			}
-			arrEs = append(arrEs, newTmp)
+			saveEsPool <- newTmp
+
 		}
 		if len(update) > 0 {
 			delete(update, "winnerorder") //winnerorder不需要更新到bindding表，删除
-			arr = append(arr, []map[string]interface{}{
-				{"_id": tmp["_id"]},
+			updateBiddingPool <- []map[string]interface{}{{
+				"_id": tmp["_id"],
+			},
 				{"$set": update},
-			})
-		}
-		if len(arr) >= BulkSize-1 {
-			mgo.UpdateBulkAll(db, c, arr...)
-			arr = [][]map[string]interface{}{}
-		}
-		if len(arrEs) >= BulkSize-1 {
-			tmps := arrEs
-			if StopFlag {
-				qutil.Debug("es队列紧张，暂停10s执行")
-				time.Sleep(time.Second * 10)
 			}
-			elastic.BulkSave(index, itype, &tmps, true)
-			if other_index != "" && other_itype != "" {
-				elastic.BulkSave(other_index, other_itype, &tmps, true)
-			}
-			if len(multiIndex) == 2 {
-				elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
-			}
-			arrEs = []map[string]interface{}{}
 		}
-		UpdatesLock.Unlock()
-		if n%100 == 0 {
-			log.Println("current:", n)
+		if n%200 == 0 {
+			util.Debug("current:", n)
 		}
 		tmp = make(map[string]interface{})
 	}
-	UpdatesLock.Lock()
-	if len(arr) > 0 {
-		mgo.UpdateBulkAll(db, c, arr...)
-	}
-	if len(arrEs) > 0 {
-		tmps := arrEs
-		if StopFlag {
-			qutil.Debug("es队列紧张，暂停10s执行")
-			time.Sleep(time.Second * 10)
-		}
-		elastic.BulkSave(index, itype, &tmps, true)
-		if other_index != "" && other_itype != "" {
-			bidding_other_es.BulkSave(other_index, other_itype, &tmps, true)
-		}
-		if len(multiIndex) == 2 {
-			elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
-		}
-	}
-	UpdatesLock.Unlock()
 	return n1, n2
 }
 
@@ -515,7 +241,7 @@ var MSG_SERVER = "123.56.236.148:7070"
 var DesLen = 120
 
 func inits() {
-	ser := qutil.ObjToString(Sysconfig["msg_server"])
+	ser := util.ObjToString(Sysconfig["msg_server"])
 	if ser != "" {
 		MSG_SERVER = ser
 	}
@@ -525,7 +251,7 @@ func inits() {
 		MsgServerAddr:   MSG_SERVER,
 		CanHandleEvents: []int{},
 		OnConnectSuccess: func() {
-			log.Println("c.")
+			util.Debug("剑鱼关键词 client")
 		},
 		ReadBufferSize:  10,
 		WriteBufferSize: 10,
@@ -534,18 +260,17 @@ func inits() {
 
 }
 
-//var clientlock = &sync.Mutex{}
 var keypool = make(chan bool, 1)
 
 func DealInfo(obj, update *map[string]interface{}) {
-	defer qutil.Catch()
+	defer util.Catch()
 	if (*obj)["keywords"] != nil && (*obj)["description"] != nil {
 		return
 	} else {
 		(*update)["keywords"] = ""
 		(*update)["description"] = ""
 	}
-	title := qutil.ObjToString((*obj)["title"])
+	title := util.ObjToString((*obj)["title"])
 	var m [][]string
 	select {
 	case <-func() <-chan bool {
@@ -593,9 +318,9 @@ func DealInfo(obj, update *map[string]interface{}) {
 	(*update)["keywords"] = keywords
 	content := ""
 	if (*obj)["detail_bak"] != nil {
-		content = qutil.ObjToString((*obj)["detail_bak"])
+		content = util.ObjToString((*obj)["detail_bak"])
 	} else {
-		content = qutil.ObjToString((*obj)["detail"])
+		content = util.ObjToString((*obj)["detail"])
 	}
 	//内容替换
 	content = strings.Replace(content, " ", "", -1)
@@ -606,7 +331,6 @@ func DealInfo(obj, update *map[string]interface{}) {
 	if strings.HasPrefix(content, ",") {
 		content = content[1:]
 	}
-	//log.Println(content)
 	tc := []rune(content)
 	ltc := len(tc)
 	description := content
@@ -618,104 +342,51 @@ func DealInfo(obj, update *map[string]interface{}) {
 	return
 }
 
-// 预测结果时间
-func YuceEndtime(tmp map[string]interface{}) {
-	flag := true
-	scope := []string{"服务采购_法律咨询", "服务采购_会计", "服务采购_物业", "服务采购_审计", "服务采购_安保", "服务采购_仓储物流",
-		"服务采购_广告宣传印刷"}
-	subscopeclass := qutil.ObjToString(tmp["s_subscopeclass"])
-	for _, v := range scope {
-		if strings.Contains(subscopeclass, v) {
-			flag = false
-			break
-		}
-	}
-	if flag {
-		return
-	}
-	subtype := qutil.ObjToString(tmp["subtype"])
-	if subtype == "成交" || subtype == "合同" {
-		// yucestarttime、yuceendtime
-		yucestarttime, yuceendtime := int64(0), int64(0)
-		// 项目周期中
-		if qutil.ObjToString(tmp["projectperiod"]) != "" {
-			dateStr := date1.FindStringSubmatch(qutil.ObjToString(tmp["projectperiod"]))
-			if len(dateStr) == 2 {
-				sdate := FormatDateStr(dateStr[0])
-				edate := FormatDateStr(dateStr[1])
-				if sdate < edate && sdate != 0 && edate != 0 {
-					yucestarttime = sdate
-					yuceendtime = edate
+// @Description tmp修改索引，update 修改bidding表，extractM修改抽取表
+// @Author J 2022/6/10 10:29 AM
+func TypeMethod(tmp, update, extractM map[string]interface{}) {
+	if jyData, ok := tmp["jyfb_data"].(map[string]interface{}); ok {
+		if t := util.ObjToString(jyData["type"]); t != "" {
+			switch t {
+			//case "采购信息":
+			case "招标公告":
+				if util.ObjToString(tmp["toptype"]) != "招标" {
+					tmp["toptype"] = "招标"
+					update["toptype"] = "招标"
+					extractM["toptype"] = "招标"
+					delete(tmp, "subtype")
+					delete(update, "subtype")
 				}
-			}
-		}
-		if yucestarttime > 0 && yuceendtime > yucestarttime {
-			tmp["yuceendtime"] = yuceendtime
-			return
-		}
-		// 预测开始时间 合同签订日期
-		if yucestarttime == 0 {
-			if qutil.IntAll(tmp["signaturedate"]) <= 0 {
-				if qutil.IntAll(tmp["publishtime"]) <= 0 {
-					return
-				} else {
-					yucestarttime = qutil.Int64All(tmp["publishtime"])
+			case "采购意向":
+				if util.ObjToString(tmp["toptype"]) != "采购意向" {
+					tmp["toptype"] = "采购意向"
+					tmp["subtype"] = "采购意向"
+					update["toptype"] = "采购意向"
+					update["subtype"] = "采购意向"
+					extractM["toptype"] = "采购意向"
+					extractM["subtype"] = "采购意向"
+				}
+			case "招标预告":
+				if util.ObjToString(tmp["toptype"]) != "预告" {
+					tmp["toptype"] = "预告"
+					update["toptype"] = "预告"
+					extractM["toptype"] = "预告"
+					delete(tmp, "subtype")
+					delete(update, "subtype")
+				}
+			case "招标结果":
+				if util.ObjToString(tmp["toptype"]) != "结果" {
+					tmp["toptype"] = "结果"
+					update["toptype"] = "结果"
+					extractM["toptype"] = "结果"
+					delete(tmp, "subtype")
+					delete(update, "subtype")
 				}
-			} else {
-				yucestarttime = qutil.Int64All(tmp["signaturedate"])
-			}
-		}
-		// 预测结束时间
-		if yucestarttime > 0 && yuceendtime == 0 {
-			if qutil.IntAll(tmp["project_duration"]) > 0 && qutil.ObjToString(tmp["project_timeunit"]) != "" {
-				yuceendtime = YcEndTime(yucestarttime, qutil.IntAll(tmp["project_duration"]), qutil.ObjToString(tmp["project_timeunit"]))
-				tmp["yuceendtime"] = yuceendtime
 			}
 		}
 	}
 }
 
-func YcEndTime(starttime int64, num int, unit string) int64 {
-	yuceendtime := int64(0)
-	if unit == "日历天" || unit == "天" || unit == "日" {
-		yuceendtime = starttime + int64(num*86400)
-	} else if unit == "周" {
-		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num*7).Unix()
-	} else if unit == "月" {
-		yuceendtime = time.Unix(starttime, 0).AddDate(0, num, 0).Unix()
-	} else if unit == "年" {
-		yuceendtime = time.Unix(starttime, 0).AddDate(num, 0, 0).Unix()
-	} else if unit == "工作日" {
-		n := num / 7 * 2
-		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num+n).Unix()
-	}
-	return yuceendtime
-}
-
-func FormatDateStr(ds string) int64 {
-
-	ds = strings.Replace(ds, "年", "-", -1)
-	ds = strings.Replace(ds, "月", "-", -1)
-	ds = strings.Replace(ds, "日", "", -1)
-	ds = strings.Replace(ds, "/", "-", -1)
-	ds = strings.Replace(ds, ".", "-", -1)
-
-	location, err := time.ParseInLocation(qutil.Date_Short_Layout, ds, time.Local)
-	if err != nil {
-		qutil.Debug(err)
-		return 0
-	} else {
-		return location.Unix()
-	}
-}
-
-type Request struct {
-	InfoId string
-}
-type Response struct {
-	Rep []map[string]interface{}
-}
-
 // @Description rpc调用信息发布程序接口
 // @Author J 2022/4/13 9:13 AM
 func UdpMethod(id string) {
@@ -724,6 +395,6 @@ func UdpMethod(id string) {
 		"stype":  "jyfb_data_over",
 	}
 	datas, _ := json.Marshal(mapinfo)
-	qutil.Debug(JyUdpAddr, string(datas))
+	util.Debug(JyUdpAddr, string(datas))
 	_ = udpclient.WriteUdp(datas, mu.OP_TYPE_DATA, JyUdpAddr)
 }

udpcreateindex/src/biddingindexback.go

@@ -2,25 +2,11 @@ package main
 
 import (
 	"go.mongodb.org/mongo-driver/bson"
-	"log"
 	"mongodb"
 	qutil "qfw/util"
-	elastic "qfw/util/elastic"
-	"reflect"
-	"strings"
-
-	//elastic "qfw/util/elastic_v5"
-	"regexp"
-	//	"strings"
-	"sync"
-)
-
-var (
-	BulkSizeBack = 400
-	ESLEN        = 32766
 )
 
-func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
+func (t *TaskInfo) biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 	defer qutil.Catch()
 	q, _ := mapInfo["query"].(map[string]interface{})
 	if q == nil {
@@ -32,35 +18,30 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 		}
 	}
 	//bidding库
-	session := mgo.GetMgoConn()
-	defer mgo.DestoryMongoConn(session)
-	//连接信息
-	c, _ := mapInfo["coll"].(string)
+	biddingConn := biddingMgo.GetMgoConn()
+	defer biddingMgo.DestoryMongoConn(biddingConn)
+	c := qutil.ObjToString(mapInfo["coll"])
 	if c == "" {
-		//		c, _ = bidding["collect"].(string)
-		c, _ = biddingback["collect"].(string)
+		c = bidding["collect"].(string)
+	} else {
+		currentColl = c
 	}
-	db, _ := biddingback["db"].(string)
-	index, _ := biddingback["index"].(string)
-	itype, _ := biddingback["type"].(string)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
-	//线程池
-	UpdatesLock := sync.Mutex{}
-	qutil.Debug("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	count, _ := biddingConn.DB(biddingMgo.DbName).C(c).Find(&q).Count()
+	qutil.Debug("查询语句:", q, "同步总数:", count)
 	//查询招标数据
-	query := session.DB(db).C(c).Find(q).Select(bson.M{
+	query := biddingConn.DB(biddingMgo.DbName).C(c).Find(q).Select(bson.M{
 		"projectinfo.attachment": 0,
 		"contenthtml":            0,
 		"publishdept":            0,
 	}).Sort("_id").Iter()
 	//查询抽取结果
 	n := 0
-	//更新数组
-	arrEs := []map[string]interface{}{}
-	thread := 10
-	espool := make(chan bool, 5)
-	var mpool = make(chan bool, thread)
+
+	var mpool = make(chan bool, t.thread)
 	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+		if n%20000 == 0 {
+			qutil.Debug("current:", n, tmp["_id"])
+		}
 		if sensitive := qutil.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
 			tmp = make(map[string]interface{})
 			continue
@@ -69,228 +50,15 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 			tmp = make(map[string]interface{})
 			continue
 		}
-		if n%20000 == 0 {
-			log.Println("current:", n, tmp["_id"])
-		}
 		mpool <- true
 		go func(tmp map[string]interface{}) {
 			defer func() {
 				<-mpool
 			}()
-			subscopeclass, _ := tmp["subscopeclass"].([]interface{}) //subscopeclass
-			if subscopeclass != nil {
-				m1 := map[string]bool{}
-				newclass := []string{}
-				for _, sc := range subscopeclass {
-					sclass, _ := sc.(string)
-					if !m1[sclass] {
-						m1[sclass] = true
-						newclass = append(newclass, sclass)
-					}
-				}
-				tmp["s_subscopeclass"] = strings.Join(newclass, ",")
-				tmp["subscopeclass"] = newclass
-			}
-			topscopeclass, _ := tmp["topscopeclass"].([]interface{}) //topscopeclass
-			if topscopeclass != nil {
-				m2 := map[string]bool{}
-				newclass := []string{}
-				for _, tc := range topscopeclass {
-					tclass, _ := tc.(string)
-					tclass = reg_letter.ReplaceAllString(tclass, "") // 去除字母
-					if !m2[tclass] {
-						m2[tclass] = true
-						newclass = append(newclass, tclass)
-					}
-				}
-				tmp["s_topscopeclass"] = strings.Join(newclass, ",")
-			}
-			//对projectscope字段的索引处理
-			ps, _ := tmp["projectscope"].(string)
-			if len(ps) > ESLEN {
-				tmp["projectscope"] = string(([]rune(ps))[:4000])
-			}
-			//对标的物为空处理
-			filetext := getFileText(tmp)
-			filetextS := filetext
-			if len([]rune(filetextS)) > 10 { //attach_text
-				tmp["filetext"] = filetext
-			}
-			if purchasing, ok := tmp["purchasing"].(string); ok && purchasing == "" {
-				delete(tmp, "purchasing")
-			}
-			if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok && len(purchasinglist) == 0 {
-				delete(tmp, "purchasinglist")
-			}
-			//数据为空处理
-			for _, f := range []string{"bidstatus", "city", "district", "channel"} {
-				if fVal, ok := tmp[f].(string); ok && fVal == "" {
-					delete(tmp, f)
-				}
-			}
-			UpdatesLock.Lock()
-			newTmp := map[string]interface{}{}
-			for field, ftype := range biddingIndexFieldsMap {
-				if tmp[field] != nil { //
-					if field == "projectinfo" {
-						mp, _ := tmp[field].(map[string]interface{})
-						if mp != nil {
-							newmap := map[string]interface{}{}
-							for k, ktype := range projectinfoFieldsMap {
-								mpv := mp[k]
-								if mpv != nil && reflect.TypeOf(mpv).String() == ktype {
-									newmap[k] = mp[k]
-								}
-							}
-							if len(newmap) > 0 {
-								newTmp[field] = newmap
-							}
-						}
-					} else if field == "purchasinglist" { //标的物处理
-						purchasinglist_new := []map[string]interface{}{}
-						if pcl, _ := tmp[field].([]interface{}); len(pcl) > 0 {
-							for _, ls := range pcl {
-								lsm_new := make(map[string]interface{})
-								lsm := ls.(map[string]interface{})
-								for pf, pftype := range purchasinglistFieldsMap {
-									lsmv := lsm[pf]
-									if lsmv != nil && reflect.TypeOf(lsmv).String() == pftype {
-										lsm_new[pf] = lsm[pf]
-									}
-								}
-								if lsm_new != nil && len(lsm_new) > 0 {
-									purchasinglist_new = append(purchasinglist_new, lsm_new)
-								}
-							}
-						}
-						if len(purchasinglist_new) > 0 {
-							newTmp[field] = purchasinglist_new
-						}
-					} else if field == "winnerorder" { //中标候选
-						winnerorder_new := []map[string]interface{}{}
-						if winnerorder, _ := tmp[field].([]interface{}); len(winnerorder) > 0 {
-							for _, win := range winnerorder {
-								winMap_new := make(map[string]interface{})
-								winMap := win.(map[string]interface{})
-								for wf, wftype := range winnerorderlistFieldsMap {
-									wfv := winMap[wf]
-									if wfv != nil && reflect.TypeOf(wfv).String() == wftype {
-										if wf == "sort" && qutil.Int64All(wfv) > 100 {
-											continue
-										}
-										winMap_new[wf] = winMap[wf]
-									}
-								}
-								if winMap_new != nil && len(winMap_new) > 0 {
-									winnerorder_new = append(winnerorder_new, winMap_new)
-								}
-							}
-						}
-						if len(winnerorder_new) > 0 {
-							newTmp[field] = winnerorder_new
-						}
-					} else if field == "qualifies" {
-						//项目资质
-						qs := []string{}
-						if q, _ := tmp[field].([]interface{}); len(q) > 0 {
-							for _, v := range q {
-								v1 := v.(map[string]interface{})
-								qs = append(qs, qutil.ObjToString(v1["key"]))
-							}
-						}
-						if len(qs) > 0 {
-							newTmp[field] = strings.Join(qs, ",")
-						}
-					} else if field == "review_experts" {
-						// 评审专家
-						if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
-							arr1 := qutil.ObjArrToStringArr(arr)
-							newTmp[field] = strings.Join(arr1, ",")
-						}
-					} else if field == "entidlist" {
-						newTmp[field] = tmp[field]
-					} else if field == "bidopentime" {
-						if tmp[field] != nil && tmp["bidendtime"] == nil {
-							newTmp["bidendtime"] = tmp[field]
-							newTmp[field] = tmp[field]
-						} else if tmp[field] == nil && tmp["bidendtime"] != nil {
-							newTmp["bidendtime"] = tmp[field]
-							newTmp[field] = tmp["bidendtime"]
-						} else {
-							if tmp["bidopentime"] != nil {
-								newTmp[field] = tmp["bidopentime"]
-							}
-						}
-					} else if field == "detail" { //过滤
-						detail, _ := tmp[field].(string)
-						if len([]rune(detail)) > detailLength {
-							detail = detail[:detailLength]
-						}
-						newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
-					} else if field == "_id" || field == "topscopeclass" { //不做处理
-						newTmp[field] = tmp[field]
-					} else if field == "publishtime" || field == "comeintime" {
-						//字段类型不正确，特别处理
-						if tmp[field] != nil && qutil.Int64All(tmp[field]) > 0 {
-							newTmp[field] = qutil.Int64All(tmp[field])
-						}
-					} else { //其它字段判断数据类型，不正确舍弃
-						if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {
-							continue
-						} else {
-							if fieldval != "" {
-								newTmp[field] = fieldval
-							}
-						}
-					}
-				}
-			}
-			arrEs = append(arrEs, newTmp)
-			if len(arrEs) >= BulkSizeBack {
-				tmps := arrEs
-				espool <- true
-				go func(tmps []map[string]interface{}) {
-					defer func() {
-						<-espool
-					}()
-					elastic.BulkSave(index, itype, &tmps, true)
-				}(tmps)
-				arrEs = []map[string]interface{}{}
-			}
-			UpdatesLock.Unlock()
+			newTmp := t.GetEsField(tmp, nil)
+			saveEsElsePool <- newTmp
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
-	for i := 0; i < thread; i++ {
-		mpool <- true
-	}
-	UpdatesLock.Lock()
-	if len(arrEs) > 0 {
-		tmps := arrEs
-		elastic.BulkSave(index, itype, &tmps, true)
-	}
-	UpdatesLock.Unlock()
-	log.Println(mapInfo, "create biddingback index...over", n)
-}
-
-var filterReg = regexp.MustCompile("<[^>]+>")
-var filterSpace = regexp.MustCompile("<[^>]*?>|[\\s\u3000\u2003\u00a0]")
-
-func FilterDetail(text string) string {
-	text = filterReg.ReplaceAllString(text, "")
-	for _, s := range FilterKeyword {
-		reg := regexp.MustCompile(s)
-		text = reg.ReplaceAllString(text, "")
-	}
-	return text
-}
-
-func FilterDetailSpace(text string) string {
-	return filterSpace.ReplaceAllString(text, "")
-}
-
-// 正则判断是否包含
-func checkContains(s, sub string) bool {
-	reg := regexp.MustCompile(`(?i)(^|([\s\t\n]+))(` + sub + `)($|([\s\t\n]+))`)
-	return reg.MatchString(s)
+	qutil.Debug(mapInfo, "create biddingback index...over", n)
 }

udpcreateindex/src/biddingindexback2.go

@@ -1,170 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"go.mongodb.org/mongo-driver/bson"
-	"log"
-	"mongodb"
-	qutil "qfw/util"
-	elastic "qfw/util/elastic"
-	"strings"
-
-	//elastic "qfw/util/elastic_v5"
-	//	"strings"
-	"sync"
-	"time"
-
-)
-
-func biddingBackTask2(data []byte, mapInfo map[string]interface{}) {
-	defer qutil.Catch()
-	q, _ := mapInfo["query"].(map[string]interface{})
-	if q == nil {
-		q = map[string]interface{}{
-			"_id": bson.M{
-				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
-				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
-			},
-		}
-	}
-	c, _ := mapInfo["coll"].(string)
-	if c == "" {
-		c, _ = biddingback["collect"].(string)
-	}
-	cs := strings.Split(c, ",")
-	for _, c := range cs {
-		//bidding库
-		session := mgo.GetMgoConn()
-		defer mgo.DestoryMongoConn(session)
-		//连接信息
-
-		db, _ := biddingback["db"].(string)
-		index, _ := biddingback["index"].(string)
-		itype, _ := biddingback["type"].(string)
-		count, _ := session.DB(db).C(c).Find(&q).Count()
-		//线程池
-		UpdatesLock := sync.Mutex{}
-		log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
-		query := session.DB(db).C(c).Find(q).Select(bson.M{
-			"contenthtml": 0,
-			"s_sha":       0,
-		}).Sort("_id").Iter()
-		//查询抽取结果
-		n := 0
-		//更新数组
-		arrEs := []map[string]interface{}{}
-		//对比两张表数据，减少查询次数
-		thread := qutil.IntAll(mapInfo["thread"])
-		//不传为0只生成招标索引,1生成招标+预览,2只生成预览
-		if thread < 1 {
-			thread = 3
-		}
-		log.Println("es线程数:", thread)
-		espool := make(chan bool, thread)
-		now1 := time.Now().Unix()
-		for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
-			if qutil.IntAll(tmp["extracttype"]) == -1 {
-				tmp = make(map[string]interface{})
-				continue
-			}
-			ct := qutil.Int64All(tmp["comeintime"])
-			pt := qutil.Int64All(tmp["publishtime"])
-			if pt > ct+86400 || pt > now1 { //时间问题,需要更新
-				if ct > now1 {
-					ct = now1
-				}
-				tmp["publishtime"] = ct
-			}
-
-			ps, _ := tmp["projectscope"].(string)
-			if ps == "" {
-				tmp["projectscope"] = "" //= tmp["detail"]
-			}
-			if len(ps) > ESLEN {
-				tmp["projectscope"] = string(([]rune(ps))[:4000])
-			}
-			if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
-				tmp["budget"] = nil
-			} else if sbd, ok := tmp["budget"].(string); ok {
-				tmp["budget"] = ObjToMoney([]interface{}{sbd, sbd})[0]
-			}
-			if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
-				tmp["bidamount"] = nil
-			} else if sbd, ok := tmp["bidamount"].(string); ok {
-				tmp["bidamount"] = ObjToMoney([]interface{}{sbd, sbd})[0]
-			}
-			UpdatesLock.Lock()
-			newTmp := map[string]interface{}{}
-			for _, v := range biddingIndexFields {
-				if tmp[v] != nil {
-					if "projectinfo" == v {
-						//处理附件 content
-						mp, _ := tmp[v].(map[string]interface{})
-						if mp != nil {
-							newmap := map[string]interface{}{}
-							for _, v1 := range projectinfoFields {
-								if mp[v1] != nil {
-									newmap[v1] = mp[v1]
-								}
-							}
-							if len(newmap) > 0 {
-								newTmp[v] = newmap
-							}
-							// attachments := mp["attachments"]
-							// con := ""
-							// if attachments != nil {
-							// 	am, _ := attachments.(map[string]interface{})
-							// 	if am != nil {
-							// 		for _, v1 := range am {
-							// 			vm, _ := v1.(map[string]interface{})
-							// 			if vm != nil {
-							// 				c, _ := vm["content"].(string)
-							// 				con += c
-							// 			}
-							// 		}
-							// 	}
-							// }
-							// if con != "" {
-							// 	con = FilterDetailSpace(con)
-							// 	newTmp["attachments"] = con
-							// }
-						}
-					} else {
-						if v == "detail" {
-							detail, _ := tmp[v].(string)
-							newTmp[v] = FilterDetail(detail)
-						} else {
-							newTmp[v] = tmp[v]
-						}
-					}
-				} else if v == "budget" || v == "bidamount" {
-					newTmp[v] = nil
-				}
-			}
-			arrEs = append(arrEs, newTmp)
-			if len(arrEs) >= BulkSizeBack {
-				tmps := arrEs
-				espool <- true
-				go func(tmps []map[string]interface{}) {
-					defer func() {
-						<-espool
-					}()
-					elastic.BulkSave(index, itype, &tmps, true)
-				}(tmps)
-				arrEs = []map[string]interface{}{}
-			}
-			UpdatesLock.Unlock()
-			if n%1000 == 0 {
-				log.Println("current:", n, mongodb.BsonIdToSId(tmp["_id"]))
-			}
-			tmp = make(map[string]interface{})
-		}
-		UpdatesLock.Lock()
-		if len(arrEs) > 0 {
-			tmps := arrEs
-			elastic.BulkSave(index, itype, &tmps, true)
-		}
-		UpdatesLock.Unlock()
-		log.Println(mapInfo, "create biddingback2 index...over", c, n)
-	}
-}

udpcreateindex/src/biddingmerge.go

@@ -8,7 +8,6 @@ import (
 	qutil "qfw/util"
 	"strings"
 	"sync"
-
 )
 
 //对字段处理 bidamount  budget
@@ -28,33 +27,30 @@ func biddingMergeTask(data []byte, mapInfo map[string]interface{}) {
 		}
 	}
 	//bidding库
-	session := mgo.GetMgoConn()
-	defer mgo.DestoryMongoConn(session)
+	session := biddingMgo.GetMgoConn()
+	defer biddingMgo.DestoryMongoConn(session)
 	//extract库
-	extractsession := extractmgo.GetMgoConn()
-	defer extractmgo.DestoryMongoConn(extractsession)
+	extractsession := extractMgo.GetMgoConn()
+	defer extractMgo.DestoryMongoConn(extractsession)
 	//连接信息
 	c, _ := mapInfo["coll"].(string)
 	if c == "" {
 		c, _ = bidding["collect"].(string)
 	}
 
-	extractc, _ := bidding["extractcollect"].(string)
-	db, _ := bidding["db"].(string)
-	extractdb, _ := bidding["extractdb"].(string)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
-	fields := strings.Split(bidding["fields"].(string), ",")
+	extractc, _ := extract["collect"].(string)
+	count, _ := session.DB(biddingMgo.DbName).C(c).Find(&q).Count()
 	//线程池
 	UpdatesLock := sync.Mutex{}
 
 	log.Println("查询语句:", q, "同步总数:", count)
 	//查询招标数据
-	query := session.DB(db).C(c).Find(q).Select(bson.M{
+	query := session.DB(biddingMgo.DbName).C(c).Find(q).Select(bson.M{
 		"projectinfo.attachment": 0,
 		"contenthtml":            0,
 	}).Sort("_id").Iter()
 	//查询抽取结果
-	extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Sort("_id").Iter()
+	extractquery := extractsession.DB(extractMgo.DbName).C(extractc).Find(q).Sort("_id").Iter()
 
 	n := 0
 
@@ -80,7 +76,7 @@ func biddingMergeTask(data []byte, mapInfo map[string]interface{}) {
 				if cid == tid {
 					bnil = false
 					//更新bidding表，生成索引
-					for _, k := range fields {
+					for _, k := range biddingMgoFields {
 						v1 := compare[k]
 						v2 := tmp[k]
 						if v2 == nil && v1 != nil {
@@ -176,14 +172,7 @@ func biddingMergeTask(data []byte, mapInfo map[string]interface{}) {
 			for tk, tv := range update {
 				tmp[tk] = tv
 			}
-			//对projectscope字段的索引处理
-			ps, _ := tmp["projectscope"].(string)
-			if ps == "" {
-				tmp["projectscope"] = "" //= tmp["detail"]
-			}
-			if len(ps) > ESLEN {
-				tmp["projectscope"] = string(([]rune(ps))[:4000])
-			}
+
 			if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
 				tmp["budget"] = nil
 			} else if sbd, ok := tmp["budget"].(string); ok {
@@ -211,10 +200,10 @@ func biddingMergeTask(data []byte, mapInfo map[string]interface{}) {
 					},
 				})
 			}
-			if len(arr) >= BulkSize-1 {
-				mgo.UpdateBulkAll(db, c, arr...)
-				arr = [][]map[string]interface{}{}
-			}
+			//if len(arr) >= BulkSize-1 {
+			//	mgo.UpdateBulkAll(db, c, arr...)
+			//	arr = [][]map[string]interface{}{}
+			//}
 
 			UpdatesLock.Unlock()
 		}(tmp, update, compare, bnil)
@@ -226,10 +215,10 @@ func biddingMergeTask(data []byte, mapInfo map[string]interface{}) {
 	for i := 0; i < thread; i++ {
 		mpool <- true
 	}
-	UpdatesLock.Lock()
-	if len(arr) > 0 {
-		mgo.UpdateBulkAll(db, c, arr...)
-	}
-	UpdatesLock.Unlock()
+	//UpdatesLock.Lock()
+	//if len(arr) > 0 {
+	//	mgo.UpdateBulkAll(db, c, arr...)
+	//}
+	//UpdatesLock.Unlock()
 	log.Println(mapInfo, "merge bidding...over", n)
 }

udpcreateindex/src/biddingtask.go

@@ -0,0 +1,523 @@
+package main
+
+import (
+	"mongodb"
+	"qfw/util"
+	"qfw/util/redis"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+	u "util"
+)
+
+var date1 = regexp.MustCompile("20[0-2][0-9][年|\\-/.][0-9]{1,2}[月|\\-/.][0-9]{1,2}[日]?")
+
+// @Description 合并extract 字段到bidding表
+// @Author J 2022/6/7 2:25 PM
+func MergeExtract(tmp, compare, update map[string]interface{}, extractMap map[string]map[string]interface{}, tasktype string) (map[string]interface{}, map[string]interface{}) {
+	tid := mongodb.BsonIdToSId(tmp["_id"])
+	if extractMap[tid] != nil {
+		compare = extractMap[tid]
+		if tasktype == "bidding" {
+			// 增量id段 正常数据
+			if num := util.IntAll(compare["dataging"]); num == 1 { //extract中dataging=1跳过
+				tmp = make(map[string]interface{})
+				compare = nil
+				return compare, update
+			}
+			delete(extractMap, tid)
+		}
+		if tasktype == "bidding_history" {
+			//增量id段 历史数据
+			if compare["history_updatetime"] == nil { //extract中history_updatetime不存在跳过
+				tmp = make(map[string]interface{})
+				compare = nil
+				return compare, update
+			}
+			delete(extractMap, tid)
+		}
+		//更新bidding表，生成索引;bidding表modifyinfo中的字段不更新
+		modifyinfo := make(map[string]bool)
+		if tmpmodifyinfo, ok := tmp["modifyinfo"].(map[string]interface{}); ok && tmpmodifyinfo != nil {
+			for k, _ := range tmpmodifyinfo {
+				modifyinfo[k] = true
+			}
+		}
+		//更新bidding表，生成索引
+		for _, k := range biddingMgoFields {
+			v1 := compare[k] //extract
+			v2 := tmp[k]     //bidding
+			if v2 == nil && v1 != nil && !modifyinfo[k] {
+				update[k] = v1
+			} else if v2 != nil && v1 != nil && !modifyinfo[k] {
+				//update[k+"_b"] = v2
+				update[k] = v1
+			} else if v2 != nil && v1 == nil {
+				//update[k+"_b"] = v2
+				if k == "area" || k == "city" || k == "district" {
+					update[k] = ""
+				}
+			}
+		}
+		if util.IntAll(compare["repeat"]) == 1 {
+			update["extracttype"] = -1
+		} else {
+			update["extracttype"] = 1
+		}
+	} else {
+		compare = nil
+		if util.IntAll(tmp["dataging"]) == 1 { //修改未抽取的bidding数据的dataging
+			update["dataging"] = 0
+		}
+	}
+	return compare, update
+}
+
+// @Description subscopeclass、topscopeclass、package
+// @Author J 2022/6/7 5:54 PM
+func FieldMethod(compare, update map[string]interface{}) {
+	subscopeclass, _ := compare["subscopeclass"].([]interface{}) //subscopeclass
+	if subscopeclass != nil {
+		m1 := map[string]bool{}
+		newclass := []string{}
+		for _, sc := range subscopeclass {
+			sclass, _ := sc.(string)
+			if !m1[sclass] {
+				m1[sclass] = true
+				newclass = append(newclass, sclass)
+			}
+		}
+		update["s_subscopeclass"] = strings.Join(newclass, ",")
+		update["subscopeclass"] = newclass
+	}
+	topscopeclass, _ := compare["topscopeclass"].([]interface{}) //topscopeclass
+	if topscopeclass != nil {
+		m2 := map[string]bool{}
+		newclass := []string{}
+		for _, tc := range topscopeclass {
+			tclass, _ := tc.(string)
+			tclass = reg_letter.ReplaceAllString(tclass, "") // 去除字母
+			if !m2[tclass] {
+				m2[tclass] = true
+				newclass = append(newclass, tclass)
+			}
+		}
+		update["s_topscopeclass"] = strings.Join(newclass, ",")
+	}
+	if package1 := compare["package"]; package1 != nil {
+		packageM, _ := package1.(map[string]interface{})
+		for _, p := range packageM {
+			pm, _ := p.(map[string]interface{})
+			if util.ObjToString(pm["winner"]) != "" || util.Float64All(pm["budget"]) > 0 ||
+				util.Float64All(pm["bidamount"]) > 0 {
+				update["multipackage"] = 1
+				break
+			}
+		}
+	} else {
+		update["multipackage"] = 0
+	}
+}
+
+// @Description ES保存字段
+// @Author J 2022/6/7 11:34 AM
+func (t *TaskInfo) GetEsField(tmp, update map[string]interface{}) map[string]interface{} {
+	newTmp := make(map[string]interface{})
+	for field, ftype := range biddingEsFields {
+		if tmp[field] != nil { //
+			if field == "projectinfo" {
+				mp, _ := tmp[field].(map[string]interface{})
+				if mp != nil {
+					newmap := map[string]interface{}{}
+					for k, ktype := range projectinfoFields {
+						mpv := mp[k]
+						if mpv != nil && reflect.TypeOf(mpv).String() == ktype {
+							newmap[k] = mp[k]
+						}
+					}
+					if len(newmap) > 0 {
+						newTmp[field] = newmap
+					}
+				}
+			} else if field == "purchasinglist" { //标的物处理
+				purchasinglist_new := []map[string]interface{}{}
+				if pcl, _ := tmp[field].([]interface{}); len(pcl) > 0 {
+					for _, ls := range pcl {
+						lsm_new := make(map[string]interface{})
+						lsm := ls.(map[string]interface{})
+						for pf, pftype := range purchasinglistFields {
+							lsmv := lsm[pf]
+							if lsmv != nil && reflect.TypeOf(lsmv).String() == pftype {
+								lsm_new[pf] = lsm[pf]
+							}
+						}
+						if lsm_new != nil && len(lsm_new) > 0 {
+							purchasinglist_new = append(purchasinglist_new, lsm_new)
+						}
+					}
+				}
+				if len(purchasinglist_new) > 0 {
+					newTmp[field] = purchasinglist_new
+				}
+			} else if field == "procurementlist" {
+				if tmp["procurementlist"] != nil {
+					var arr []interface{}
+					plist := tmp["procurementlist"].([]interface{})
+					for _, p := range plist {
+						p1 := p.(map[string]interface{})
+						p2 := make(map[string]interface{})
+						for k, v := range procurementlisFields {
+							if k == "projectname" && util.ObjToString(p1[k]) == "" {
+								p2[k] = util.ObjToString(tmp["projectname"])
+							} else if k == "buyer" && util.ObjToString(p1[k]) == "" && util.ObjToString(tmp["buyer"]) != "" {
+								p2[k] = util.ObjToString(tmp["buyer"])
+							} else if k == "expurasingtime" && util.ObjToString(p1[k]) != "" {
+								res := getMethod(util.ObjToString(p1[k]))
+								if res != 0 {
+									p2[k] = res
+								}
+							} else if p1[k] != nil && reflect.TypeOf(p1[k]).String() == v {
+								p2[k] = p1[k]
+							}
+						}
+						arr = append(arr, p2)
+					}
+					if len(arr) > 0 {
+						newTmp[field] = arr
+					}
+				}
+			} else if field == "projectscope" {
+				ps, _ := tmp["projectscope"].(string)
+				if len(ps) > pscopeLength {
+					tmp["projectscope"] = string(([]rune(ps))[:pscopeLength])
+				}
+			} else if field == "winnerorder" { //中标候选
+				winnerorder_new := []map[string]interface{}{}
+				if winnerorder, _ := tmp[field].([]interface{}); len(winnerorder) > 0 {
+					for _, win := range winnerorder {
+						winMap_new := make(map[string]interface{})
+						winMap := win.(map[string]interface{})
+						for wf, wftype := range winnerorderlistFields {
+							wfv := winMap[wf]
+							if wfv != nil && reflect.TypeOf(wfv).String() == wftype {
+								if wf == "sort" && util.Int64All(wfv) > 100 {
+									continue
+								}
+								winMap_new[wf] = winMap[wf]
+							}
+						}
+						if winMap_new != nil && len(winMap_new) > 0 {
+							winnerorder_new = append(winnerorder_new, winMap_new)
+						}
+					}
+				}
+				if len(winnerorder_new) > 0 {
+					newTmp[field] = winnerorder_new
+				}
+			} else if field == "qualifies" {
+				//项目资质
+				qs := []string{}
+				if q, _ := tmp[field].([]interface{}); len(q) > 0 {
+					for _, v := range q {
+						v1 := v.(map[string]interface{})
+						qs = append(qs, util.ObjToString(v1["key"]))
+					}
+				}
+				if len(qs) > 0 {
+					newTmp[field] = strings.Join(qs, ",")
+				}
+			} else if field == "review_experts" {
+				// 评审专家
+				if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
+					arr1 := util.ObjArrToStringArr(arr)
+					newTmp[field] = strings.Join(arr1, ",")
+				}
+			} else if field == "bidopentime" {
+				if tmp[field] != nil && tmp["bidendtime"] == nil {
+					newTmp["bidendtime"] = tmp[field]
+					newTmp[field] = tmp[field]
+				} else if tmp[field] == nil && tmp["bidendtime"] != nil {
+					newTmp["bidendtime"] = tmp[field]
+					newTmp[field] = tmp["bidendtime"]
+				} else {
+					if tmp["bidopentime"] != nil {
+						newTmp[field] = tmp["bidopentime"]
+					}
+				}
+			} else if field == "detail" { //过滤
+				detail, _ := tmp[field].(string)
+				if len([]rune(detail)) > detailLength {
+					detail = detail[:detailLength]
+				}
+				detail = filterSpace.ReplaceAllString(detail, "")
+				if t.stype == "bidding" || t.stype == "bidding_history" {
+					text, b := FilterDetail(detail)
+					newTmp[field] = util.ObjToString(tmp["title"]) + " " + text
+					update["cleartag"] = b
+				} else {
+					if tmp["cleartag"] != nil && tmp["cleartag"].(bool) {
+						text, _ := FilterDetail(detail)
+						newTmp[field] = util.ObjToString(tmp["title"]) + " " + text
+					} else {
+						newTmp[field] = util.ObjToString(tmp["title"]) + " " + detail
+					}
+				}
+			} else if field == "_id" || field == "topscopeclass" || field == "entidlist" { //不做处理
+				newTmp[field] = tmp[field]
+			} else if field == "publishtime" || field == "comeintime" {
+				//字段类型不正确，特别处理
+				if tmp[field] != nil && util.Int64All(tmp[field]) > 0 {
+					newTmp[field] = util.Int64All(tmp[field])
+				}
+			} else { //其它字段判断数据类型，不正确舍弃
+				if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {
+					continue
+				} else {
+					if fieldval != "" {
+						newTmp[field] = fieldval
+					}
+				}
+			}
+		}
+	}
+	filetext := getFileText(tmp)
+	if len([]rune(filetext)) > 10 {
+		newTmp["filetext"] = filetext
+	} else {
+		// 附件未识别到内容，有附件且附件能够下载	filetext=""
+		if pinfo, o1 := tmp["projectinfo"].(map[string]interface{}); o1 {
+			if atts, o2 := pinfo["attachments"].(map[string]interface{}); o2 {
+				tag := false
+				for _, at := range atts {
+					at1 := at.(map[string]interface{})
+					if at1["fid"] != nil {
+						tag = true
+						break
+					}
+				}
+				if tag {
+					newTmp["filetext"] = ""
+				}
+			}
+		}
+	}
+	YuceEndtime(newTmp) // 预测结果时间
+	if t.stype == "bidding" {
+		newTmp["createtime"] = time.Now().Unix() // es库数据创建时间，只有增量数据有
+	}
+	return newTmp
+}
+
+// @Description 附件内容
+// @Author J 2022/6/7 1:54 PM
+func getFileText(tmp map[string]interface{}) (filetext string) {
+	if attchMap, ok := tmp["attach_text"].(map[string]interface{}); attchMap != nil && ok {
+		for _, tmpData1 := range attchMap {
+			if tmpData2, ok := tmpData1.(map[string]interface{}); tmpData2 != nil && ok {
+				for _, result := range tmpData2 {
+					if resultMap, ok := result.(map[string]interface{}); resultMap != nil && ok {
+						if attach_url := util.ObjToString(resultMap["attach_url"]); attach_url != "" {
+							bs := u.OssGetObject(attach_url) //oss读数据
+							if utf8.RuneCountInString(filetext+bs) < fileLength {
+								filetext += bs + "\n"
+							} else {
+								if utf8.RuneCountInString(bs) > fileLength {
+									filetext = bs[0:fileLength]
+								} else {
+									filetext = bs
+								}
+								break
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	return
+}
+
+// 预测结果时间
+func YuceEndtime(tmp map[string]interface{}) {
+	flag := true
+	scope := []string{"服务采购_法律咨询", "服务采购_会计", "服务采购_物业", "服务采购_审计", "服务采购_安保", "服务采购_仓储物流",
+		"服务采购_广告宣传印刷"}
+	subscopeclass := util.ObjToString(tmp["s_subscopeclass"])
+	for _, v := range scope {
+		if strings.Contains(subscopeclass, v) {
+			flag = false
+			break
+		}
+	}
+	if flag {
+		return
+	}
+	subtype := util.ObjToString(tmp["subtype"])
+	if subtype == "成交" || subtype == "合同" {
+		// yucestarttime、yuceendtime
+		yucestarttime, yuceendtime := int64(0), int64(0)
+		// 项目周期中
+		if util.ObjToString(tmp["projectperiod"]) != "" {
+			dateStr := date1.FindStringSubmatch(util.ObjToString(tmp["projectperiod"]))
+			if len(dateStr) == 2 {
+				sdate := FormatDateStr(dateStr[0])
+				edate := FormatDateStr(dateStr[1])
+				if sdate < edate && sdate != 0 && edate != 0 {
+					yucestarttime = sdate
+					yuceendtime = edate
+				}
+			}
+		}
+		if yucestarttime > 0 && yuceendtime > yucestarttime {
+			tmp["yuceendtime"] = yuceendtime
+			return
+		}
+		// 预测开始时间 合同签订日期
+		if yucestarttime == 0 {
+			if util.IntAll(tmp["signaturedate"]) <= 0 {
+				if util.IntAll(tmp["publishtime"]) <= 0 {
+					return
+				} else {
+					yucestarttime = util.Int64All(tmp["publishtime"])
+				}
+			} else {
+				yucestarttime = util.Int64All(tmp["signaturedate"])
+			}
+		}
+		// 预测结束时间
+		if yucestarttime > 0 && yuceendtime == 0 {
+			if util.IntAll(tmp["project_duration"]) > 0 && util.ObjToString(tmp["project_timeunit"]) != "" {
+				yuceendtime = YcEndTime(yucestarttime, util.IntAll(tmp["project_duration"]), util.ObjToString(tmp["project_timeunit"]))
+				tmp["yuceendtime"] = yuceendtime
+			}
+		}
+	}
+}
+
+func YcEndTime(starttime int64, num int, unit string) int64 {
+	yuceendtime := int64(0)
+	if unit == "日历天" || unit == "天" || unit == "日" {
+		yuceendtime = starttime + int64(num*86400)
+	} else if unit == "周" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num*7).Unix()
+	} else if unit == "月" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, num, 0).Unix()
+	} else if unit == "年" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(num, 0, 0).Unix()
+	} else if unit == "工作日" {
+		n := num / 7 * 2
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num+n).Unix()
+	}
+	return yuceendtime
+}
+
+func FormatDateStr(ds string) int64 {
+	ds = strings.Replace(ds, "年", "-", -1)
+	ds = strings.Replace(ds, "月", "-", -1)
+	ds = strings.Replace(ds, "日", "", -1)
+	ds = strings.Replace(ds, "/", "-", -1)
+	ds = strings.Replace(ds, ".", "-", -1)
+
+	location, err := time.ParseInLocation(util.Date_Short_Layout, ds, time.Local)
+	if err != nil {
+		util.Debug(err)
+		return 0
+	} else {
+		return location.Unix()
+	}
+}
+
+// @Description entidlist
+// @Author J 2022/6/7 2:36 PM
+func FieldFun(tmp map[string]interface{}) (cid []string) {
+	sWinnerarr := strings.Split(util.ObjToString(tmp["s_winner"]), ",")
+	for _, w := range sWinnerarr {
+		if w != "" {
+			id := redis.GetStr("qyxy_id", w)
+			if id == "" {
+				ents, _ := standardMgo.Find("qyxy_std", map[string]interface{}{"company_name": w}, map[string]interface{}{"updatetime": -1}, nil, false, -1, -1)
+				if len(*ents) > 0 {
+					id = util.ObjToString((*ents)[0]["_id"])
+					redis.PutCKV("qyxy_id", w, id)
+				} else {
+					ent, _ := qyxyMgo.FindOne("company_history_name", map[string]interface{}{"history_name": w})
+					if len(*ent) > 0 {
+						id = util.ObjToString((*ent)["company_id"])
+						redis.PutCKV("qyxy_id", w, id)
+					}
+				}
+			}
+			if id == "" {
+				id = "-"
+			}
+			cid = append(cid, id)
+		}
+	}
+	return cid
+}
+
+var filterSpace = regexp.MustCompile("<[^>]*?>|[\\s\u3000\u2003\u00a0]")
+
+func FilterDetail(text string) (string, bool) {
+	b := false // 清理标记
+	for _, s := range FilterKeyword {
+		reg := regexp.MustCompile(s)
+		if reg.MatchString(text) {
+			text = reg.ReplaceAllString(text, "")
+			if !b {
+				b = true
+			}
+		}
+	}
+	return text, b
+}
+
+// 正则判断是否包含
+func checkContains(s, sub string) bool {
+	reg := regexp.MustCompile(`(?i)(^|([\s\t\n]+))(` + sub + `)($|([\s\t\n]+))`)
+	return reg.MatchString(s)
+}
+
+var TimeV1 = regexp.MustCompile("(\\d{4})[年.]?$")
+var TimeV2 = regexp.MustCompile("(\\d{4}[年.\\-/]?)(\\d{1,2}[月.\\-/]?$)")
+var TimeClear = regexp.MustCompile("[年|月|/|.|-]")
+
+// @Description 采购意向 预计采购时间处理
+// @Author J 2022/6/7 8:04 PM
+func getMethod(str string) int64 {
+	if TimeV1.MatchString(str) {
+		arr := TimeV1.FindStringSubmatch(str)
+		st := arr[1] + "0000"
+		parseInt, err := strconv.ParseInt(st, 10, 64)
+		if err == nil {
+			return parseInt
+		}
+	} else if TimeV2.MatchString(str) {
+		arr := TimeV2.FindStringSubmatch(str)
+		str1 := arr[2]
+		if len(str1) == 1 {
+			str1 = "0" + str1
+		}
+		str2 := TimeClear.ReplaceAllString(arr[1], "") + TimeClear.ReplaceAllString(str1, "") + "00"
+		parseInt, err := strconv.ParseInt(str2, 10, 64)
+		if err == nil {
+			return parseInt
+		}
+	}
+	return 0
+}
+
+// @Description 字段空值处理
+// @Author J 2022/6/7 8:04 PM
+func clearMap(tmp map[string]interface{}) {
+	for k := range tmp {
+		if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok && len(purchasinglist) == 0 {
+			delete(tmp, "purchasinglist")
+		} else if reflect.TypeOf(tmp[k]).String() == "string" && util.ObjToString(tmp[k]) == "" {
+			delete(tmp, k)
+		}
+	}
+}

udpcreateindex/src/bidingpurchasing.go

@@ -1,369 +0,0 @@
-package main
-
-import (
-	"go.mongodb.org/mongo-driver/bson"
-	"log"
-	"qfw/util"
-	elastic "qfw/util/elastic"
-	"sync"
-	"unicode/utf8"
-
-	u "util"
-)
-
-//定时查询bidding中extract_state为2的数据生成索引
-func biddingPurchaingTask(q map[string]interface{}) {
-	defer util.Catch()
-	//锁
-	SaveUpdageLock := sync.Mutex{}
-	//连接参数
-	c, _ := bidding["collect"].(string)   //bidding表
-	db, _ := bidding["db"].(string)       //库
-	index, _ := bidding["index"].(string) //索引别名
-	itype, _ := bidding["type"].(string)
-	//
-	session := mgo.GetMgoConn()
-	defer mgo.DestoryMongoConn(session)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
-	log.Println("biddingPurchaingTask:	", db, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
-
-	query := session.DB(db).C(c).Find(q).Select(bson.M{
-		"projectinfo.attachment": 0,
-		"contenthtml":            0,
-	}).Iter()
-	arrEs := make([]map[string]interface{}, savesizei)
-	arrMgo := [][]map[string]interface{}{}
-	var n int
-	i := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
-		n++
-		if util.IntAll(tmp["extracttype"]) == -1 { // || util.IntAll(tmp["dataging"]) == 1 { //重复数据不生索引
-			tmp = make(map[string]interface{})
-			continue
-		}
-		newTmp := map[string]interface{}{} //最终生索引的数据
-		saveArr := []map[string]interface{}{}
-		//oss拼装filetext
-		if filetext := getFileText(tmp); len(filetext) > 10 {
-			if site, _ := tmp["site"].(string); site == "中国招标投标公共服务平台" { //site:中国招标投标公共服务平台 detail替换成filetext 并加入标记filedetail=1
-				tmp["detail"] = filetext
-				saveArr = append(saveArr, map[string]interface{}{"_id": tmp["_id"]})
-				saveArr = append(saveArr, map[string]interface{}{
-					"$set": map[string]interface{}{
-						"filedetail": 1,
-						"detail":     filetext,
-					},
-				})
-			}
-			newTmp["filetext"] = filetext
-		}
-		//purchasing
-		if purchasing, ok := tmp["purchasing"].(string); ok {
-			if len(purchasing) > 0 {
-				newTmp["purchasing"] = tmp["purchasing"]
-			}
-		}
-		//purchasinglist
-		if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok {
-			util.Debug(len(purchasinglist))
-			if len(purchasinglist) > 0 {
-				purchasinglist_new := []map[string]interface{}{}
-				for _, ls := range purchasinglist {
-					lsm_new := make(map[string]interface{})
-					lsm := ls.(map[string]interface{})
-					for _, pf := range purchasinglistFields {
-						if lsm[pf] != nil {
-							lsm_new[pf] = lsm[pf]
-						}
-					}
-					if lsm_new != nil && len(lsm_new) > 0 {
-						purchasinglist_new = append(purchasinglist_new, lsm_new)
-					}
-				}
-				util.Debug(len(purchasinglist_new), purchasinglist_new)
-				if len(purchasinglist_new) > 0 {
-					newTmp["purchasinglist"] = purchasinglist_new
-				}
-			}
-		}
-
-		//处理数据
-		if tmp["supervisorrate"] != nil { //临时处理supervisorrate抽取类型为string不生索引
-			if _, ok := tmp["supervisorrate"].(string); ok { //supervisorrate数据为string类型
-				delete(tmp, "supervisorrate")
-			}
-		}
-		//对projectscope字段的索引处理
-		ps, _ := tmp["projectscope"].(string)
-		if len(ps) > ESLEN {
-			tmp["projectscope"] = string(([]rune(ps))[:4000])
-		}
-		SaveUpdageLock.Lock()
-		for _, v := range biddingIndexFields { //索引字段
-			if tmp[v] != nil {
-				if "projectinfo" == v {
-					mp, _ := tmp[v].(map[string]interface{})
-					if mp != nil {
-						newmap := map[string]interface{}{}
-						for _, v1 := range projectinfoFields {
-							if mp[v1] != nil {
-								newmap[v1] = mp[v1]
-							}
-						}
-						newTmp[v] = newmap
-						// attachments := mp["attachments"]
-						// con := ""
-						// if attachments != nil {
-						// 	am, _ := attachments.(map[string]interface{})
-						// 	if am != nil {
-						// 		for _, v1 := range am {
-						// 			vm, _ := v1.(map[string]interface{})
-						// 			if vm != nil {
-						// 				c, _ := vm["content"].(string)
-						// 				con += c
-						// 			}
-						// 		}
-						// 	}
-						// }
-						// con = FilterDetailSpace(con)
-						// if con != "" {
-						// 	newTmp["attachments"] = con
-						// }
-					}
-				} else {
-					if v == "detail" {
-						detail, _ := tmp[v].(string)
-						newTmp[v] = FilterDetail(detail)
-					} else {
-						newTmp[v] = tmp[v]
-					}
-				}
-			}
-		}
-		arrEs = append(arrEs, newTmp)
-		if len(saveArr) > 0 {
-			arrMgo = append(arrMgo, saveArr) //要更新数据
-		}
-		// arrMgo = append(arrMgo, []map[string]interface{}{ //要更新数据
-		// 	map[string]interface{}{
-		// 		"_id": tmp["_id"],
-		// 	},
-		// 	map[string]interface{}{
-		// 		"$set": map[string]interface{}{
-		// 			"extract_state": 4,
-		// 		},
-		// 	},
-		// })
-		//批量更新
-		if len(arrMgo) >= savesizei-1 {
-			mgo.UpdateBulkAll(db, c, arrMgo...)
-			arrMgo = [][]map[string]interface{}{}
-		}
-		//生索引
-		if len(arrEs) >= savesizei-1 {
-			tmps := arrEs
-			elastic.BulkSave(index, itype, &tmps, true)
-			arrEs = []map[string]interface{}{}
-		}
-		SaveUpdageLock.Unlock()
-		//计数
-		if n%savesizei == 0 {
-			log.Println("当前:", n)
-		}
-		tmp = make(map[string]interface{})
-	}
-
-	SaveUpdageLock.Lock()
-	if len(arrMgo) > 0 {
-		mgo.UpdateBulkAll(db, c, arrMgo...)
-	}
-	if len(arrEs) > 0 {
-		tmps := arrEs
-		elastic.BulkSave(index, itype, &tmps, true)
-	}
-	SaveUpdageLock.Unlock()
-	log.Println("create filetext index...over", n)
-}
-
-//定时任务site：中国招标投标公共服务平台
-/*
-	注意：
-	1、调用此任务时config.json中indexfields配置不要有purchasing、purchasinglist、filetext
-*/
-func site_attach_text(q map[string]interface{}) {
-	defer util.Catch()
-	//锁
-	SaveUpdageLock := sync.Mutex{}
-	//连接参数
-	c, _ := bidding["collect"].(string)   //bidding表
-	db, _ := bidding["db"].(string)       //库
-	index, _ := bidding["index"].(string) //索引别名
-	itype, _ := bidding["type"].(string)
-	//
-	session := mgo.GetMgoConn()
-	defer mgo.DestoryMongoConn(session)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
-	log.Println("site_attach_text:	", db, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
-
-	query := session.DB(db).C(c).Find(q).Select(bson.M{
-		"projectinfo.attachment": 0,
-		"contenthtml":            0,
-	}).Iter()
-	arrEs := make([]map[string]interface{}, savesizei)
-	arrMgo := [][]map[string]interface{}{}
-	var n int
-	var indexnum int
-	i := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
-		n++
-		//计数
-		if n%savesizei == 0 {
-			log.Println("当前:", n)
-		}
-		site, _ := tmp["site"].(string)
-		if util.IntAll(tmp["extracttype"]) == -1 || site != "中国招标投标公共服务平台" || tmp["attach_text"] == nil {
-			tmp = make(map[string]interface{})
-			continue
-		}
-		newTmp := map[string]interface{}{} //最终生索引的数据
-		saveArr := []map[string]interface{}{}
-
-		filetext := getFileText(tmp) //oss拼装filetext
-		if len(filetext) > 10 {
-			tmp["detail"] = filetext //filetext替换detail
-			saveArr = append(saveArr, map[string]interface{}{"_id": tmp["_id"]})
-			saveArr = append(saveArr, map[string]interface{}{
-				"$set": map[string]interface{}{
-					"filedetail": 1,
-					"detail":     filetext,
-				},
-			})
-			newTmp["filetext"] = filetext //
-		} else {
-			//log.Println("filetext is null string:", tmp["_id"])
-			tmp = make(map[string]interface{})
-			continue
-		}
-
-		indexnum++
-
-		//purchasing
-		if purchasing, ok := tmp["purchasing"].(string); ok {
-			if len(purchasing) > 0 {
-				newTmp["purchasing"] = tmp["purchasing"]
-			}
-		}
-		//purchasinglist
-		if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok {
-			if len(purchasinglist) > 0 {
-				purchasinglist_new := []map[string]interface{}{}
-				for _, ls := range purchasinglist {
-					lsm_new := make(map[string]interface{})
-					lsm := ls.(map[string]interface{})
-					for _, pf := range purchasinglistFields {
-						if lsm[pf] != nil {
-							lsm_new[pf] = lsm[pf]
-						}
-					}
-					if lsm_new != nil && len(lsm_new) > 0 {
-						purchasinglist_new = append(purchasinglist_new, lsm_new)
-					}
-				}
-				if len(purchasinglist_new) > 0 {
-					newTmp["purchasinglist"] = purchasinglist_new
-				}
-			}
-		}
-
-		//处理数据
-		if tmp["supervisorrate"] != nil { //临时处理supervisorrate抽取类型为string不生索引
-			if _, ok := tmp["supervisorrate"].(string); ok { //supervisorrate数据为string类型
-				delete(tmp, "supervisorrate")
-			}
-		}
-		//对projectscope字段的索引处理
-		ps, _ := tmp["projectscope"].(string)
-		if len(ps) > ESLEN {
-			tmp["projectscope"] = string(([]rune(ps))[:4000])
-		}
-		SaveUpdageLock.Lock()
-		for _, v := range biddingIndexFields { //索引字段
-			if tmp[v] != nil {
-				if "projectinfo" == v {
-					mp, _ := tmp[v].(map[string]interface{})
-					if mp != nil {
-						newmap := map[string]interface{}{}
-						for _, v1 := range projectinfoFields {
-							if mp[v1] != nil {
-								newmap[v1] = mp[v1]
-							}
-						}
-						newTmp[v] = newmap
-					}
-				} else {
-					if v == "detail" {
-						detail, _ := tmp[v].(string)
-						newTmp[v] = FilterDetail(detail)
-					} else {
-						newTmp[v] = tmp[v]
-					}
-				}
-			}
-		}
-		arrEs = append(arrEs, newTmp) //要生索引数据
-		if len(saveArr) > 0 {
-			arrMgo = append(arrMgo, saveArr) //要更新数据
-		}
-		//批量更新
-		if len(arrMgo) >= savesizei-1 {
-			mgo.UpdateBulkAll(db, c, arrMgo...)
-			arrMgo = [][]map[string]interface{}{}
-		}
-		//生索引
-		if len(arrEs) >= savesizei-1 {
-			tmps := arrEs
-			elastic.BulkSave(index, itype, &tmps, true)
-			arrEs = []map[string]interface{}{}
-		}
-		SaveUpdageLock.Unlock()
-		tmp = make(map[string]interface{})
-	}
-
-	SaveUpdageLock.Lock()
-	if len(arrMgo) > 0 {
-		mgo.UpdateBulkAll(db, c, arrMgo...)
-	}
-	if len(arrEs) > 0 {
-		tmps := arrEs
-		elastic.BulkSave(index, itype, &tmps, true)
-	}
-	SaveUpdageLock.Unlock()
-	log.Println("create filetext index...over", n, indexnum)
-}
-
-func getFileText(tmp map[string]interface{}) (filetext string) {
-	if attchMap, ok := tmp["attach_text"].(map[string]interface{}); attchMap != nil && ok {
-		for _, tmpData1 := range attchMap {
-			if tmpData2, ok := tmpData1.(map[string]interface{}); tmpData2 != nil && ok {
-				for _, result := range tmpData2 {
-					if resultMap, ok := result.(map[string]interface{}); resultMap != nil && ok {
-						if attach_url := util.ObjToString(resultMap["attach_url"]); attach_url != "" {
-							bs := u.OssGetObject(attach_url) //oss读数据
-							if utf8.RuneCountInString(filetext+bs) < fileLength {
-								filetext += bs + "\n"
-							} else {
-								if utf8.RuneCountInString(bs) > fileLength {
-									filetext = bs[0:fileLength]
-								} else {
-									filetext = bs
-								}
-								break
-							}
-						}
-					}
-				}
-			}
-		}
-	}
-	return
-}

udpcreateindex/src/buyerindex.go

@@ -7,7 +7,6 @@ import (
 	"qfw/util"
 	elastic "qfw/util/elastic"
 	"sync"
-
 )
 
 /*
@@ -85,16 +84,15 @@ func buyerTask_err(data []byte, mapInfo map[string]interface{}) {
 		}
 	}
 	//mongo
-	sess := mgostandard.GetMgoConn()
-	defer mgostandard.DestoryMongoConn(sess)
+	sess := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(sess)
 	c, _ := buyer["collect"].(string)
-	db, _ := buyer["db"].(string)
 	index, _ := buyer["index"].(string)
 	itype, _ := buyer["type"].(string)
 
-	count, _ := sess.DB(db).C(c).Find(&q).Count()
+	count, _ := sess.DB(standardMgo.DbName).C(c).Find(&q).Count()
 	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
-	it := sess.DB(db).C(c).Find(&q).Select(map[string]interface{}{
+	it := sess.DB(standardMgo.DbName).C(c).Find(&q).Select(map[string]interface{}{
 		"name":              1,
 		"buyer_name":        1,
 		"institute_type":    1,
@@ -141,7 +139,7 @@ func buyerTask_err(data []byte, mapInfo map[string]interface{}) {
 			util.Debug(savetmp)
 			buyerEsLock.Lock()
 			arrEs = append(arrEs, savetmp)
-			if len(arrEs) >= BulkSize {
+			if len(arrEs) >= MgoBulkSize {
 				tmps := arrEs
 				elastic.BulkSave(index, itype, &tmps, true)
 				arrEs = []map[string]interface{}{}
@@ -174,16 +172,16 @@ func buyerTask(data []byte, mapInfo map[string]interface{}) {
 		}
 	}
 	//mongo
-	sess := mgostandard.GetMgoConn()
-	defer mgostandard.DestoryMongoConn(sess)
+	sess := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(sess)
+
 	c, _ := buyer["collect"].(string)
-	db, _ := buyer["db"].(string)
 	index, _ := buyer["index"].(string)
 	itype, _ := buyer["type"].(string)
 
-	count, _ := sess.DB(db).C(c).Find(&q).Count()
+	count, _ := sess.DB(standardMgo.DbName).C(c).Find(&q).Count()
 	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
-	it := sess.DB(db).C(c).Find(&q).Select(map[string]interface{}{
+	it := sess.DB(standardMgo.DbName).C(c).Find(&q).Select(map[string]interface{}{
 		"buyer_name":        1,
 		"institute_type":    1,
 		"buyerclass":        1,
@@ -237,7 +235,7 @@ func buyerTask(data []byte, mapInfo map[string]interface{}) {
 			}
 			buyerEsLock.Lock()
 			arrEs = append(arrEs, savetmp)
-			if len(arrEs) >= BulkSize {
+			if len(arrEs) >= EsBulkSize {
 				tmps := arrEs
 				elastic.BulkSave(index, itype, &tmps, true)
 				arrEs = []map[string]interface{}{}

udpcreateindex/src/buyertask.go

@@ -5,7 +5,6 @@ import (
 	"log"
 	"mongodb"
 	qu "qfw/util"
-	elastic "qfw/util/elastic"
 	"sync"
 	"time"
 )
@@ -41,11 +40,11 @@ func buyerEsTaskOnce() {
 	index, _ := buyerent["index"].(string)
 	itype, _ := buyerent["type"].(string)
 	//mongo
-	sess := mgostandard.GetMgoConn()
-	defer mgostandard.DestoryMongoConn(sess)
+	sess := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(sess)
 
-	log.Println("q:", q, "db:", mgostandard.DbName, "coll:", buyer_ent)
-	it_1 := sess.DB(mgostandard.DbName).C(buyer_ent).Find(&q).Select(map[string]interface{}{
+	log.Println("q:", q, "db:", standardMgo.DbName, "coll:", buyer_ent)
+	it_1 := sess.DB(standardMgo.DbName).C(buyer_ent).Find(&q).Select(map[string]interface{}{
 		"buyer_name":        1,
 		"institute_type":    1,
 		"buyerclass":        1,
@@ -94,9 +93,9 @@ func buyerEsTaskOnce() {
 			}
 			buyerEsLock.Lock()
 			arrEs = append(arrEs, savetmp)
-			if len(arrEs) >= BulkSize {
+			if len(arrEs) >= EsBulkSize {
 				tmps := arrEs
-				elastic.BulkSave(index, itype, &tmps, true)
+				Es1.BulkSave(index, itype, &tmps, true)
 				arrEs = []map[string]interface{}{}
 			}
 			buyerEsLock.Unlock()
@@ -147,7 +146,7 @@ func buyerEsTaskOnce() {
 	buyerEsLock.Lock()
 	if len(arrEs) > 0 {
 		tmps := arrEs
-		elastic.BulkSave(index, itype, &tmps, true)
+		Es1.BulkSave(index, itype, &tmps, true)
 		arrEs = []map[string]interface{}{}
 	}
 	buyerEsLock.Unlock()

udpcreateindex/src/config.json

@@ -1,140 +1,107 @@
 {
   "udpport": ":1483",
   "msg_server": "10.171.112.160:7070",
-  "uname": "",
-  "upwd": "",
-  "mongodb": {
+  "mgo_bidding": {
     "addr": "192.168.3.207:27092",
-    "pool": 10,
-    "db": "wjh"
+    "size": 10,
+    "db": "wjh",
+    "uname": "",
+    "upwd": "",
+    "collect": "bidding"
   },
-  "savedb": {
+  "mgo_extract": {
     "addr": "192.168.3.207:27092",
     "size": 10,
-    "db": "wjh"
+    "db": "wjh",
+    "collect": "extract"
   },
-  "jkmail": {
-    "to": "zhangjinkun@topnet.net.cn",
-    "api": "http://10.171.112.160:19281/_send/_mail"
+  "mgo_qyxy": {
+    "addr": "192.168.3.207",
+    "pool": 10,
+    "db": "mixdata"
   },
-  "winner": {
+  "mgo_project": {
+    "addr": "192.168.3.207",
     "db": "qfw",
-    "collect": "winner",
-    "index": "winner",
-    "type": "winner"
-  },
-  "buyer": {
-    "db": "mxs",
-    "collect": "buyer_err",
-    "index": "buyer_v6",
-    "type": "buyer"
-  },
-  "biddingback": {
-    "db": "qfw_data",
-    "collect": "bidding",
-    "index": "bidding_v1",
-    "type": "bidding"
-  },
-  "bidding": {
-    "db": "qfw_data",
-    "collect": "bidding",
-    "index": "bidding_v1",
-    "type": "bidding",
-    "multiIndex": "",
-    "extractdb": "qfw_data",
-    "extractcollect": "extract",
-    "indexfields": [
-      "buyerzipcode", "winnertel", "winnerperson", "contractcode", "winneraddr", "agencyaddr", "buyeraddr", "signaturedate", "projectperiod", "projectaddr", "agencytel", "agencyperson",
-      "buyerperson", "agency", "projectscope", "projectcode", "bidopentime", "supervisorrate", "buyertel", "bidamount", "winner", "buyer", "budget", "projectname", "bidstatus", "buyerclass",
-      "topscopeclass", "s_topscopeclass", "s_subscopeclass", "area", "city", "district", "s_winner", "_id", "title", "detail", "site", "comeintime", "href", "infoformat", "publishtime",
-      "s_sha", "spidercode", "subtype", "toptype", "projectinfo", "purchasing", "purchasinglist", "filetext", "channel", "winnerorder", "project_scale", "project_duration", "project_timeunit",
-      "project_startdate", "project_completedate", "payway", "contract_guarantee", "bid_guarantee", "qualifies", "entidlist", "funds", "review_experts", "bidmethod", "bidendtime", "bidopenaddress",
-      "docamount", "agencyrate", "agencyfee", "getdocmethod", "china_bidding", "bidway", "purchasing_tag", "multipackage"
-    ],
-    "indexfieldsmap": {
-      "buyerzipcode": "string", "winnertel": "string", "winnerperson": "string", "contractcode": "string", "winneraddr": "string", "agencyaddr": "string", "buyeraddr": "string", "signaturedate": "int64",
-      "projectperiod": "string", "projectaddr": "string", "agencytel": "string", "agencyperson": "string", "buyerperson": "string", "agency": "string", "projectscope": "string", "projectcode": "string",
-      "bidopentime": "int64", "supervisorrate": "float64", "buyertel": "string", "bidamount": "float64", "winner": "string", "buyer": "string", "budget": "float64", "projectname": "string", "bidstatus": "string",
-      "buyerclass": "string", "topscopeclass": "", "s_topscopeclass": "string", "s_subscopeclass": "string", "area": "string", "city": "string", "district": "string", "s_winner": "string", "_id": "", "title": "string",
-      "detail": "string", "site": "string", "comeintime": "int64", "href": "string", "infoformat": "int32", "publishtime": "int64", "s_sha": "string", "spidercode": "string", "subtype": "string", "toptype": "string",
-      "projectinfo": "", "purchasing": "string", "purchasinglist": "", "filetext": "string", "channel": "string", "winnerorder": "", "project_scale": "string", "project_duration": "int32", "project_timeunit": "string",
-      "project_startdate": "int64", "project_completedate": "int64", "payway": "string", "contract_guarantee": "bool", "bid_guarantee": "bool", "qualifies": "", "entidlist": "[]string", "funds": "string",
-      "review_experts": "", "bidmethod": "string", "bidendtime": "int64", "bidopenaddress": "string", "docamount": "float64", "agencyrate": "float64", "agencyfee": "float64", "bidway": "string",
-      "getdocmethod": "string", "china_bidding": "string", "purchasing_tag": "string", "multipackage": "int32"
-    },
-    "fields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,s_topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass,dataging,winnerorder,project_scale,project_duration,project_timeunit,project_startdate,project_completedate, payway,contract_guarantee,bid_guarantee,qualifies,funds,review_experts,bidmethod,bidendtime,bidopenaddress,docamount,bidway,agencyrate,agencyfee,getdocmethod,purchasing_tag",
-    "projectinfo": "approvecode,approvecontent,approvestatus,approvetime,approvedept,approvenumber,projecttype,approvecity",
-    "projectinfomap": {
-      "approvecode": "string", "approvecontent": "string", "approvestatus": "string", "approvetime": "string", "approvedept": "string", "approvenumber": "string", "projecttype": "string", "approvecity": "string"
-    },
-    "purchasinglist": "itemname,brandname,model,unitname,number,unitprice,totalprice,projectname,buyer,item,projectscope,expurasingtime,reserved_amount",
-    "purchasinglistmap": {
-      "itemname": "string", "brandname": "string", "model": "string", "unitname": "string", "number": "float64", "unitprice": "float64", "totalprice": "float64", "projectname": "string", "buyer": "string", "item": "string",
-      "projectscope": "string", "expurasingtime": "string", "reserved_amount": "string"
-    },
-    "winnerorder": "sort,sortstr,entname",
-    "winnerordermap": {
-      "sort": "int", "sortstr": "string", "entname": "string"
-    }
-  },
-  "filelength": 50000,
-  "detaillength": 50000,
-  "project": {
-    "db": "qfw_data",
     "collect": "projectset",
     "index": "projectset",
     "type": "projectset"
   },
-  "project2": {
-    "addr": "192.168.3.207:27092",
-    "pool": 1,
-    "db": "mxs",
-    "collect": "huawei_project",
-    "index": "project_v2",
-    "type": "project"
-  },
-  "qyxy_ent": {
-    "addr": "192.168.3.207:27092",
-    "pool": 5,
-    "db": "mixdata"
-  },
   "standard": {
-    "addr": "192.168.3.207:27001",
+    "addr": "192.168.3.207:27092",
     "pool": 10,
     "db": "mixdata",
     "coll_area": "address_jy_2021",
-    "winnerent": {
+    "winner": {
       "collect1": "winner_enterprise",
       "collect2": "winner_err",
       "index": "winner_v1",
       "type": "winner"
     },
-    "buyerent": {
+    "buyer": {
       "collect1": "buyer_enterprise",
       "collect2": "buyer_err",
       "index": "buyer_v6",
       "type": "buyer"
     },
-    "agencyent": {
+    "agency": {
       "collect": "agency_enterprise",
       "index": "agencyent_v1",
       "type": "agencyent"
     }
   },
-  "elastic": {
-    "addr": "http://192.168.3.11:9800",
+  "elastic_1": {
+    "addr": "http://127.0.0.1:9800",
     "pool": 12,
-    "node": "4q7v7e6mQ5aeCwjUgM6HcA"
+    "label": "旧es集群库"
+  },
+  "elastic_2": {
+    "addr": "http://127.0.0.1:19800",
+    "pool": 12,
+    "label": "新es集群库"
+  },
+  "bidding_index": {
+    "index": "bidding_v2",
+    "type": "bidding",
+    "multiIndex": "",
+    "esfieldsmap": {
+      "buyerzipcode": "string", "winnertel": "string", "winnerperson": "string", "contractcode": "string", "winneraddr": "string", "agencyaddr": "string", "buyeraddr": "string", "signaturedate": "int64",
+      "projectperiod": "string", "projectaddr": "string", "agencytel": "string", "agencyperson": "string", "buyerperson": "string", "agency": "string", "projectscope": "string", "projectcode": "string",
+      "bidopentime": "int64", "supervisorrate": "float64", "buyertel": "string", "bidamount": "float64", "winner": "string", "buyer": "string", "budget": "float64", "projectname": "string", "bidstatus": "string",
+      "buyerclass": "string", "topscopeclass": "", "s_topscopeclass": "string", "s_subscopeclass": "string", "area": "string", "city": "string", "district": "string", "s_winner": "string", "_id": "", "title": "string",
+      "detail": "string", "site": "string", "comeintime": "int64", "href": "string", "infoformat": "int32", "publishtime": "int64", "s_sha": "string", "spidercode": "string", "subtype": "string", "toptype": "string",
+      "projectinfo": "", "purchasing": "string", "purchasinglist": "", "channel": "string", "winnerorder": "", "project_scale": "string", "project_duration": "int32", "project_timeunit": "string",
+      "project_startdate": "int64", "project_completedate": "int64", "payway": "string", "contract_guarantee": "bool", "bid_guarantee": "bool", "qualifies": "", "entidlist": "", "funds": "string",
+      "review_experts": "", "bidmethod": "string", "bidendtime": "int64", "bidopenaddress": "string", "docamount": "float64", "agencyrate": "float64", "agencyfee": "float64", "bidway": "string",
+      "getdocmethod": "string", "china_bidding": "string", "purchasing_tag": "string", "multipackage": "int32"
+    },
+    "mgofields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,s_topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass,dataging,winnerorder,project_scale,project_duration,project_timeunit,project_startdate,project_completedate, payway,contract_guarantee,bid_guarantee,qualifies,funds,review_experts,bidmethod,bidendtime,bidopenaddress,docamount,bidway,agencyrate,agencyfee,getdocmethod,purchasing_tag",
+    "projectinfomap": {
+      "approvecode": "string", "approvecontent": "string", "approvestatus": "string", "approvetime": "string", "approvedept": "string", "approvenumber": "string", "projecttype": "string", "approvecity": "string"
+    },
+    "purchasinglistmap": {
+      "itemname": "string", "item": "string", "brandname": "string", "model": "string", "unitname": "string", "number": "float64", "unitprice": "float64", "totalprice": "float64"
+    },
+    "procurementlistmap": {
+      "projectname": "string", "buyer": "string", "item": "string", "projectscope": "string", "expurasingtime": "string", "totalprice": "float64"
+    },
+    "winnerordermap": {
+      "sort": "int", "sortstr": "string", "entname": "string"
+    }
   },
   "jyfb_udp": {
     "addr": "127.0.0.1",
     "port": 11118
   },
+  "jkmail": {
+    "to": "zhangjinkun@topnet.net.cn",
+    "api": "http://172.17.145.179:19281/_send/_mail"
+  },
   "filter-keyword": ["(招标网|千里马|采招网|招标采购导航网|招标与采购网|中国招投标网|中国采购与招标网|中国采购与招标|优质采)[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}(qianlima|zhaobiao|okcis|zbytb|infobidding|bidcenter|youzhicai|chinabidding|Chinabidding｜CHINABIDDING)[a-z0-9.\\/\\/]{0,40}",
     "招标网[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}zhaobiao[a-z0-9.\\/\\/]{0,40}",
     "千里马[\\w\\W]{0,15}[a-z0-9:\\/\\/.]{0,20}qianlima[a-z0-9.\\/\\/]{0,10}",
     "[\\(（]?(网址)?[:：；;]?(http|https|htpps)*[:：]?(\\/\\/)?(www|jinan|WWW)?.(zhaobiao|chinabidding|Chinabidding|CHINABIDDING|infobidding|zbytb|okcis|qianlima|youzhicai).(com|cn|COM|CN)?(.cn|.CN)?\\/?[\\)）]?",
     "[\\(（]?(网址)?(:：)?(http|https|htpps)*(:|：)?\\/\\/www.bidcenter.com.cn\\/",
-      "千里马(平台|网站)+", "[“\"]?优质采(平台|电子交易平台|云采购平台|交易平台)?[”\"]?", "《?(中国采购与|中国)?招(投)?标(与采购|采购导航)?网》?",
+    "千里马(平台|网站)+", "[“\"]?优质采(平台|电子交易平台|云采购平台|交易平台)?[”\"]?", "《?(中国采购与|中国)?招(投)?标(与采购|采购导航)?网》?",
     "《?元博网(采购与招标网)?》?", "《?(中国)?招标采购导航网》?", "中\\W{0,3}国采\\W{0,3}招\\W{0,3}网\\W*[(（]?(bidcenter.com.cn)?[)）]?", "已方宝", "中国招标与采购"]
 }

udpcreateindex/src/datamonitor.go

@@ -42,7 +42,7 @@ func (is *InfoStatus) Save() {
 		save["endtime"] = is.Endtime
 		save["flag"] = "indexser"
 		is.Val = map[string]int{}
-		go extractmgo.Save("datamonitor", save)
+		go extractMgo.Save("datamonitor", save)
 	}
 	is.Starttime = is.Endtime
 	is.Lock.Unlock()

udpcreateindex/src/default.go

@@ -1,79 +0,0 @@
-package main
-
-import (
-	"go.mongodb.org/mongo-driver/bson"
-	"log"
-	"mongodb"
-	"qfw/util"
-	elastic "qfw/util/elastic"
-)
-
-func defaultFunc(data []byte, mapInfo map[string]interface{}) {
-	defer util.Catch()
-	tasktype, _ := mapInfo["stype"].(string)
-	if tasktype == "" {
-		return
-	}
-	q, _ := mapInfo["query"].(map[string]interface{})
-	if q == nil {
-		q = map[string]interface{}{
-			"_id": bson.M{
-				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
-				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
-			},
-		}
-	}
-	var Mgo *mongodb.MongodbSim
-	c, _ := mapInfo["c"].(string)
-	db, _ := mapInfo["d"].(string)
-	index, _ := mapInfo["index"].(string)
-	itype, _ := mapInfo["type"].(string)
-	if itype == "" {
-		itype = index
-	}
-	if mapInfo["mgoaddr"] != nil {
-		Mgo = &mongodb.MongodbSim{
-			MongodbAddr: mapInfo["mgoaddr"].(string),
-			Size:        5,
-			DbName:      db,
-		}
-		Mgo.InitPool()
-	} else {
-		Mgo = mgo
-	}
-	session := Mgo.GetMgoConn()
-	defer Mgo.DestoryMongoConn(session)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
-	savepool := make(chan bool, 10)
-	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
-	query := session.DB(db).C(c).Find(q).Iter()
-
-	arr := make([]map[string]interface{}, savesizei)
-	var n int
-	i := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
-		go IS.Add(tasktype)
-		arr[i] = tmp
-		n++
-		if i == savesizei-1 {
-			savepool <- true
-			tmps := arr
-			go func(tmpn *[]map[string]interface{}) {
-				defer func() {
-					<-savepool
-				}()
-				elastic.BulkSave(index, itype, tmpn, true)
-			}(&tmps)
-			i = 0
-			arr = make([]map[string]interface{}, savesizei)
-		}
-		if n%savesizei == 0 {
-			log.Println("当前:", n)
-		}
-		tmp = make(map[string]interface{})
-	}
-	if i > 0 {
-		elastic.BulkSave(index, itype, &arr, true)
-	}
-	log.Println(mapInfo, "create "+tasktype+" index...over", n)
-}

udpcreateindex/src/init.go

@@ -0,0 +1,258 @@
+package main
+
+import (
+	"fmt"
+	mu "mfw/util"
+	"mongodb"
+	"net"
+	"qfw/util"
+	elastic "qfw/util/elastic"
+	"strings"
+	"time"
+	u "util"
+)
+
+var (
+	Sysconfig map[string]interface{}
+
+	biddingMgo  *mongodb.MongodbSim
+	extractMgo  *mongodb.MongodbSim
+	qyxyMgo     *mongodb.MongodbSim
+	projectMgo  *mongodb.MongodbSim
+	standardMgo *mongodb.MongodbSim
+
+	Es1 *elastic.Elastic
+	Es2 *elastic.Elastic
+
+	currentColl string
+	udpclient   mu.UdpClient //udp对象
+	updport     string
+	multiIndex  []string
+
+	biddingMgoFields      []string
+	biddingEsFields       map[string]interface{}
+	projectinfoFields     map[string]interface{}
+	purchasinglistFields  map[string]interface{}
+	procurementlisFields  map[string]interface{}
+	winnerorderlistFields map[string]interface{}
+
+	updateBiddingPool chan []map[string]interface{}
+	updateExtractPool chan []map[string]interface{}
+	saveEsPool        chan map[string]interface{}
+	saveEsAllPool     chan map[string]interface{}
+	saveEsElsePool    chan map[string]interface{}
+	updateBiddingSp   chan bool
+	updateExtractSp   chan bool
+	saveEsSp          chan bool
+	saveEsAllSp       chan bool
+	saveEsElseSp      chan bool
+
+	JyUdpAddr *net.UDPAddr
+
+	esAddr string
+	esNode string
+
+	FilterKeyword []string              //正文竟品关键词过滤
+	ProvinceDict  map[string][]Province //省份-map
+	CityDict      map[string][]City     //城市-map
+	DistrictDict  map[string][]District //区县-map
+
+	bidding, extract, qyxy, project, standard, buyer, winner, biddingIndex map[string]interface{}
+
+	MgoBulkSize  = 200   // mgo批量保存大小
+	EsBulkSize   = 200   // es批量保存大小
+	detailLength = 50000 // es保存detail长度
+	fileLength   = 50000 // es保存附件文本长度
+	pscopeLength = 32766 // projectscope长度
+)
+
+var StopFlag = false // 程序生索引停止标志
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	updport, _ = Sysconfig["updport"].(string)
+
+	bidding, _ = Sysconfig["mgo_bidding"].(map[string]interface{})
+	extract, _ = Sysconfig["mgo_extract"].(map[string]interface{})
+	qyxy, _ = Sysconfig["mgo_qyxy"].(map[string]interface{})
+	project, _ = Sysconfig["mgo_project"].(map[string]interface{})
+	standard, _ = Sysconfig["standard"].(map[string]interface{})
+	buyer, _ = Sysconfig["buyer"].(map[string]interface{})
+	winner, _ = Sysconfig["winner"].(map[string]interface{})
+
+	currentColl = util.ObjToString(bidding["collect"])
+	biddingIndex, _ = Sysconfig["bidding_index"].(map[string]interface{})
+
+	biddingMgo = &mongodb.MongodbSim{
+		MongodbAddr: bidding["addr"].(string),
+		Size:        util.IntAllDef(bidding["pool"], 5),
+		DbName:      bidding["db"].(string),
+		UserName:    bidding["uname"].(string),
+		Password:    bidding["upwd"].(string),
+	}
+	biddingMgo.InitPool()
+	extractMgo = &mongodb.MongodbSim{
+		MongodbAddr: extract["addr"].(string),
+		Size:        util.IntAllDef(extract["pool"], 5),
+		DbName:      extract["db"].(string),
+	}
+	extractMgo.InitPool()
+	qyxyMgo = &mongodb.MongodbSim{
+		MongodbAddr: qyxy["addr"].(string),
+		Size:        util.IntAllDef(qyxy["pool"], 5),
+		DbName:      qyxy["db"].(string),
+	}
+	qyxyMgo.InitPool()
+	projectMgo = &mongodb.MongodbSim{
+		MongodbAddr: project["addr"].(string),
+		Size:        util.IntAllDef(project["pool"], 5),
+		DbName:      project["db"].(string),
+	}
+	projectMgo.InitPool()
+	standardMgo = &mongodb.MongodbSim{
+		MongodbAddr: standard["addr"].(string),
+		Size:        util.IntAllDef(standard["pool"], 5),
+		DbName:      standard["db"].(string),
+		UserName:    bidding["uname"].(string),
+		Password:    bidding["upwd"].(string),
+	}
+	standardMgo.InitPool()
+
+	econf1 := Sysconfig["elastic_1"].(map[string]interface{})
+	//esNode = econf1["node"].(string)
+	Es1 = &elastic.Elastic{
+		S_esurl: econf1["addr"].(string),
+		I_size:  util.IntAllDef(econf1["pool"], 5),
+	}
+	Es1.InitElasticSize()
+
+	econf2 := Sysconfig["elastic_2"].(map[string]interface{})
+	Es2 = &elastic.Elastic{
+		S_esurl: econf2["addr"].(string),
+		I_size:  util.IntAllDef(econf2["pool"], 5),
+	}
+	Es2.InitElasticSize()
+
+	if mi := util.ObjToString(biddingIndex["multiIndex"]); mi != "" {
+		multiIndex = strings.Split(mi, ",")
+	}
+	util.Debug(multiIndex)
+	biddingMgoFields = strings.Split(biddingIndex["mgofields"].(string), ",")
+	biddingEsFields = biddingIndex["esfieldsmap"].(map[string]interface{})             // bidding es字段
+	projectinfoFields = biddingIndex["projectinfomap"].(map[string]interface{})        // projectinfo
+	purchasinglistFields = biddingIndex["purchasinglistmap"].(map[string]interface{})  //采购清单
+	procurementlisFields = biddingIndex["procurementlistmap"].(map[string]interface{}) //采购意向
+	winnerorderlistFields = biddingIndex["winnerordermap"].(map[string]interface{})    //中标候选
+
+	FilterKeyword = util.ObjArrToStringArr(Sysconfig["filter-keyword"].([]interface{}))
+	initCheckCity()
+	//初始化oss
+	u.InitOss()
+
+	m := Sysconfig["jyfb_udp"].(map[string]interface{})
+	JyUdpAddr = &net.UDPAddr{
+		IP:   net.ParseIP(m["addr"].(string)),
+		Port: util.IntAll(m["port"]),
+	}
+
+	updateBiddingPool = make(chan []map[string]interface{}, 5000)
+	updateBiddingSp = make(chan bool, 5)
+	updateExtractPool = make(chan []map[string]interface{}, 5000)
+	updateExtractSp = make(chan bool, 5)
+	saveEsPool = make(chan map[string]interface{}, 5000)
+	saveEsSp = make(chan bool, 5)
+	saveEsAllPool = make(chan map[string]interface{}, 5000)
+	saveEsAllSp = make(chan bool, 5)
+	saveEsElsePool = make(chan map[string]interface{}, 5000)
+	saveEsElseSp = make(chan bool, 5)
+}
+
+func NewTk(m map[string]interface{}) *TaskInfo {
+	p := &TaskInfo{
+		stype:       m["stype"].(string),
+		currentTime: time.Now().Unix(),
+	}
+	return p
+}
+
+type TaskInfo struct {
+	stype         string
+	thread        int
+	currentTime   int64
+	bidding_count int
+	extract_count int
+	es_count      int
+}
+
+type Province struct {
+	P_Name string
+}
+type City struct {
+	P_Name string
+	C_Name string
+}
+type District struct {
+	P_Name string
+	C_Name string
+	D_Name string
+}
+
+//初始化城市
+func initCheckCity() {
+	//初始化-城市配置
+	ProvinceDict = make(map[string][]Province, 0)
+	CityDict = make(map[string][]City, 0)
+	DistrictDict = make(map[string][]District, 0)
+
+	q := map[string]interface{}{
+		"town_code": map[string]interface{}{
+			"$exists": 0,
+		},
+	}
+	sess := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(sess)
+	it := sess.DB(standardMgo.DbName).C(util.ObjToString(standard["coll_area"])).Find(&q).Iter()
+	total := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		district_code := util.IntAll(tmp["district_code"])
+		city_code := util.IntAll(tmp["city_code"])
+		if district_code > 0 {
+			province := util.ObjToString(tmp["province"])
+			city := util.ObjToString(tmp["city"])
+			district := util.ObjToString(tmp["district"])
+			data := District{province, city, district}
+			if DistrictDict[district] == nil {
+				DistrictDict[district] = []District{data}
+			} else {
+				arr := DistrictDict[district]
+				arr = append(arr, data)
+				DistrictDict[district] = arr
+			}
+		} else {
+			if city_code > 0 {
+				province := util.ObjToString(tmp["province"])
+				city := util.ObjToString(tmp["city"])
+				data := City{province, city}
+				if CityDict[city] == nil {
+					CityDict[city] = []City{data}
+				} else {
+					arr := CityDict[city]
+					arr = append(arr, data)
+					CityDict[city] = arr
+				}
+			} else {
+				province := util.ObjToString(tmp["province"])
+				data := Province{province}
+				if ProvinceDict[province] == nil {
+					ProvinceDict[province] = []Province{data}
+				} else {
+					arr := ProvinceDict[province]
+					arr = append(arr, data)
+					ProvinceDict[province] = arr
+				}
+			}
+		}
+		tmp = make(map[string]interface{})
+	}
+	util.Debug(fmt.Sprintf("城市配置加载完毕...省~%d 市~%d 区~%d", len(ProvinceDict), len(CityDict), len(DistrictDict)))
+}

udpcreateindex/src/main.go

@@ -2,242 +2,34 @@ package main
 
 import (
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"log"
 	mu "mfw/util"
-	"mongodb"
 	"net"
 	"net/http"
-	_ "net/http/pprof"
 	"qfw/util"
-	elastic "qfw/util/elastic"
-	"strings"
+	"qfw/util/redis"
 	"time"
-	u "util"
 )
 
-type Province struct {
-	P_Name string
-}
-type City struct {
-	P_Name string
-	C_Name string
-}
-type District struct {
-	P_Name string
-	C_Name string
-	D_Name string
-}
-
-var (
-	Sysconfig                map[string]interface{} //配置文件
-	mgo                      *mongodb.MongodbSim    //mongodb操作对象
-	extractmgo               *mongodb.MongodbSim    //mongodb操作对象
-	project2db               *mongodb.MongodbSim    //mongodb操作对象
-	mgostandard              *mongodb.MongodbSim    //mongodb操作对象
-	qyxydb                   *mongodb.MongodbSim    //mongodb操作对象
-	udpclient                mu.UdpClient           //udp对象
-	updport                  string
-	savesizei                = 500
-	biddingIndexFields       = []string{"_id", "buyerclass", "s_winner", "title", "detail", "detail_bak", "area", "areaval", "site", "type", "amount", "bidopendate", "bidopentime", "buyer", "channel", "city", "comeintime", "contenthtml", "descript", "description", "extracttype", "href", "infoformat", "keywords", "projectcode", "projectname", "publishtime", "s_sha", "spidercode", "subtype", "summary", "toptype", "urltop", "winner", "agency", "budget", "bidamount", "s_subscopeclass", "projectscope", "bidstatus"}
-	biddingIndexFieldsMap    = map[string]string{}
-	projectinfoFields        []string
-	projectinfoFieldsMap     = map[string]string{}
-	multiIndex               []string
-	purchasinglistFields     []string
-	winnerorderlistFields    []string
-	purchasinglistFieldsMap  = map[string]string{}
-	winnerorderlistFieldsMap = map[string]string{}
-	BulkSize                 = 400
-	detailLength             = 50000
-	fileLength               = 50000
-	//bidding_other连接信息
-	bidding_other_es *elastic.Elastic
-	other_index      string
-	other_itype      string
-
-	JyUdpAddr *net.UDPAddr
-
-	esAddr string
-	esNode string
-
-	FilterKeyword []string              //正文竟品关键词过滤
-	ProvinceDict  map[string][]Province //省份-map
-	CityDict      map[string][]City     //城市-map
-	DistrictDict  map[string][]District //区县-map
-
-	winner, bidding, biddingback, project, project2, buyer, standard, qyxy_ent map[string]interface{}
-)
-var UpdataMgoCache = make(chan []map[string]interface{}, 1000)
-var SP = make(chan bool, 5)
-var SaveLogChan = make(chan []map[string]interface{}, 1000)
-var SaveSp = make(chan bool, 5)
-
-var StopFlag = false // 程序生索引停止标志
-
-func init() {
-	util.ReadConfig(&Sysconfig)
+func main() {
 	// company_id
-	//redis.InitRedis1("qyxy_id=172.17.4.189:8379", 4)
-	//inits()
-	//go checkMapJob()
-	detailLength = util.IntAllDef(Sysconfig["detaillength"], 50000)
-	fileLength = util.IntAllDef(Sysconfig["filelength"], 50000)
-	updport, _ = Sysconfig["updport"].(string)
-
-	winner, _ = Sysconfig["winner"].(map[string]interface{})
-	standard, _ = Sysconfig["standard"].(map[string]interface{})
-	buyer, _ = Sysconfig["buyer"].(map[string]interface{})
-	bidding, _ = Sysconfig["bidding"].(map[string]interface{})
-	biddingback, _ = Sysconfig["biddingback"].(map[string]interface{})
-	project, _ = Sysconfig["project"].(map[string]interface{})
-	project2, _ = Sysconfig["project2"].(map[string]interface{})
-	qyxy_ent, _ = Sysconfig["qyxy_ent"].(map[string]interface{})
-	mconf, _ := Sysconfig["mongodb"].(map[string]interface{})
-
-	mgo = &mongodb.MongodbSim{ //mongodb为binding连接
-		MongodbAddr: mconf["addr"].(string),
-		Size:        util.IntAllDef(mconf["pool"], 5),
-		DbName:      mconf["db"].(string),
-		UserName:    Sysconfig["uname"].(string),
-		Password:    Sysconfig["upwd"].(string),
-	}
-	mgo.InitPool()
-	project2db = &mongodb.MongodbSim{
-		MongodbAddr: project2["addr"].(string),
-		Size:        util.IntAllDef(project2["pool"], 5),
-		DbName:      project2["db"].(string),
-	}
-
-	project2db.InitPool()
-
-	//企业数据
-	qyxydb = &mongodb.MongodbSim{
-		MongodbAddr: qyxy_ent["addr"].(string),
-		Size:        util.IntAllDef(qyxy_ent["pool"], 5),
-		DbName:      qyxy_ent["db"].(string),
-	}
-	qyxydb.InitPool()
-
-	savedb, _ := Sysconfig["savedb"].(map[string]interface{})
-	if savedb == nil {
-		log.Println("未设置保存数据库，默认使用招标库")
-		extractmgo = mgo
-	} else { //savedb为抽取连接
-		addr, _ := savedb["addr"].(string)
-		size := util.IntAllDef(savedb["size"], 5)
-		db, _ := savedb["db"].(string)
-		extractmgo = &mongodb.MongodbSim{
-			MongodbAddr: addr,
-			Size:        size,
-			DbName:      db,
-		}
-		extractmgo.InitPool()
-	}
-	mgostandard = &mongodb.MongodbSim{
-		MongodbAddr: standard["addr"].(string),
-		Size:        util.IntAllDef(standard["pool"], 5),
-		DbName:      standard["db"].(string),
-		UserName:    Sysconfig["uname"].(string),
-		Password:    Sysconfig["upwd"].(string),
-	}
-	mgostandard.InitPool()
-
-	//初始化es
-	//bidding
-	econf := Sysconfig["elastic"].(map[string]interface{})
-	esAddr = econf["addr"].(string)
-	esNode = econf["node"].(string)
-	elastic.InitElasticSize(econf["addr"].(string), util.IntAllDef(econf["pool"], 5))
-	//bidding_other
-	if Sysconfig["elastic_other"] != nil {
-		econf_other := Sysconfig["elastic_other"].(map[string]interface{})
-		other_index = econf_other["index"].(string)
-		other_itype = econf_other["type"].(string)
-		bidding_other_es = &elastic.Elastic{
-			S_esurl: econf_other["addr"].(string),
-			I_size:  util.IntAllDef(econf_other["pool"], 5),
-		}
-		bidding_other_es.InitElasticSize()
-	}
-
-	//
-	if bidding["indexfields"] != nil {
-		biddingIndexFields = util.ObjArrToStringArr(bidding["indexfields"].([]interface{}))
-	}
-	if bidding["projectinfo"] != nil {
-		pf := util.ObjToString(bidding["projectinfo"])
-		if pf != "" {
-			projectinfoFields = strings.Split(pf, ",")
-		}
-	}
-	if bidding["purchasinglist"] != nil {
-		pcl := util.ObjToString(bidding["purchasinglist"])
-		if pcl != "" {
-			purchasinglistFields = strings.Split(pcl, ",")
-		}
-	}
-	if bidding["winnerorder"] != nil {
-		winnerorder := util.ObjToString(bidding["winnerorder"])
-		if winnerorder != "" {
-			winnerorderlistFields = strings.Split(winnerorder, ",")
-		}
-	}
-	if bidding["multiIndex"] != nil {
-		mi := util.ObjToString(bidding["multiIndex"])
-		if mi != "" {
-			multiIndex = strings.Split(mi, ",")
-		}
-	}
-	//
-	if bidding["indexfieldsmap"] != nil {
-		for k, v := range bidding["indexfieldsmap"].(map[string]interface{}) {
-			biddingIndexFieldsMap[k] = util.ObjToString(v)
-		}
-		log.Println(biddingIndexFieldsMap)
-	}
-	if bidding["projectinfomap"] != nil {
-		for k, v := range bidding["projectinfomap"].(map[string]interface{}) {
-			projectinfoFieldsMap[k] = util.ObjToString(v)
-		}
-		log.Println(projectinfoFieldsMap)
-	}
-	if bidding["purchasinglistmap"] != nil {
-		for k, v := range bidding["purchasinglistmap"].(map[string]interface{}) {
-			purchasinglistFieldsMap[k] = util.ObjToString(v)
-		}
-		log.Println(purchasinglistFieldsMap)
-	}
-	if bidding["winnerordermap"] != nil {
-		for k, v := range bidding["winnerordermap"].(map[string]interface{}) {
-			winnerorderlistFieldsMap[k] = util.ObjToString(v)
-		}
-		log.Println(winnerorderlistFieldsMap)
-	}
-	log.Println(projectinfoFields)
-	log.Println(purchasinglistFields)
-
-	FilterKeyword = util.ObjArrToStringArr(Sysconfig["filter-keyword"].([]interface{}))
-	initCheckCity()
-	//初始化oss
-	u.InitOss()
-
-	m := Sysconfig["jyfb_udp"].(map[string]interface{})
-	JyUdpAddr = &net.UDPAddr{
-		IP:   net.ParseIP(m["addr"].(string)),
-		Port: util.IntAll(m["port"]),
-	}
-}
+	redis.InitRedis1("qyxy_id=172.17.4.189:8379", 4)
+	inits()
 
-func main() {
 	//go inspectQuery()
-	//go task_index()
-	//go UpdateExtract() //抽取表中新增entidlist字段
+	go checkMapJob()
+	go task_index()
+
+	go UpdateBidding()
+	go UpdateExtract()
+	go SaveEsMethod()
+	go SaveAllEsMethod()
+	go SaveElseEsMethod()
 	updport := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
 	udpclient.Listen(processUdpMsg)
-	log.Println("Udp服务监听", updport)
+	util.Debug("Udp服务监听", updport)
 
 	ch := make(chan bool, 1)
 	<-ch
@@ -297,7 +89,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 			}
 			go udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
 			tasktype, _ := mapInfo["stype"].(string)
-			log.Println("tasktype:", tasktype)
+			t := NewTk(mapInfo)
 			switch tasktype {
 			case "winner":
 				pool <- true
@@ -313,7 +105,8 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					defer func() {
 						<-pool
 					}()
-					biddingTask(data, mapInfo, tasktype)
+					t.thread = 1
+					t.biddingTask(data, mapInfo)
 				}()
 			case "bidding_history": //增量id段历史数据
 				pool <- true
@@ -321,7 +114,8 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					defer func() {
 						<-pool
 					}()
-					biddingTask(data, mapInfo, tasktype)
+					t.thread = 1
+					t.biddingTask(data, mapInfo)
 				}()
 			case "project":
 				pool <- true
@@ -337,7 +131,8 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					defer func() {
 						<-pool
 					}()
-					biddingBackTask(data, mapInfo)
+					t.thread = 30
+					t.biddingBackTask(data, mapInfo)
 				}()
 			case "biddingall": //合并并重新生成索引，不生成关键词
 				pool <- true
@@ -345,7 +140,8 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					defer func() {
 						<-pool
 					}()
-					biddingAllTask(data, mapInfo)
+					t.thread = 30
+					t.biddingAllTask(data, mapInfo)
 				}()
 			case "biddingdata": //bidding全量数据
 				pool <- true
@@ -353,7 +149,8 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					defer func() {
 						<-pool
 					}()
-					biddingDataTask(data, mapInfo)
+					t.thread = 30
+					t.biddingDataTask(data, mapInfo)
 				}()
 			case "biddingmerge": //重新合并但不生成索引，不生成关键词
 				pool <- true
@@ -417,7 +214,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					defer func() {
 						<-pool
 					}()
-					defaultFunc(data, mapInfo)
+					util.Debug("err ---", mapInfo)
 				}()
 			}
 		}
@@ -426,98 +223,184 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 	}
 }
 
-//初始化城市
-func initCheckCity() {
-	//初始化-城市配置
-	ProvinceDict = make(map[string][]Province, 0)
-	CityDict = make(map[string][]City, 0)
-	DistrictDict = make(map[string][]District, 0)
+func SaveEsMethod() {
+	arru := make([]map[string]interface{}, EsBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveEsPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				saveEsSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsSp
+					}()
+					Es1.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+					Es2.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+					if len(multiIndex) == 2 {
+						Es1.BulkSave(multiIndex[0], multiIndex[1], &arru, true)
+					}
+				}(arru)
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveEsSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsSp
+					}()
+					Es1.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+					Es2.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+					if len(multiIndex) == 2 {
+						Es1.BulkSave(multiIndex[0], multiIndex[1], &arru, true)
+					}
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		}
+	}
+}
 
-	q := map[string]interface{}{
-		"town_code": map[string]interface{}{
-			"$exists": 0,
-		},
+func SaveElseEsMethod() {
+	arru := make([]map[string]interface{}, EsBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveEsElsePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				saveEsElseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsElseSp
+					}()
+					Es2.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+				}(arru)
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveEsElseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsElseSp
+					}()
+					Es2.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		}
 	}
-	sess := mgostandard.GetMgoConn()
-	defer mgostandard.DestoryMongoConn(sess)
-	it := sess.DB("mixdata").C(util.ObjToString(standard["coll_area"])).Find(&q).Iter()
-	total := 0
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%1000 == 0 {
-			log.Println("当前数量:", total)
+}
+
+func SaveAllEsMethod() {
+	arru := make([]map[string]interface{}, EsBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveEsAllPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				saveEsAllSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsAllSp
+					}()
+					Es1.BulkSave("bidding_all", "bidding", &arru, true)
+				}(arru)
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveEsAllSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsAllSp
+					}()
+					Es1.BulkSave("bidding_all", "bidding", &arru, true)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
 		}
-		district_code := util.IntAll(tmp["district_code"])
-		city_code := util.IntAll(tmp["city_code"])
-		if district_code > 0 {
-			province := util.ObjToString(tmp["province"])
-			city := util.ObjToString(tmp["city"])
-			district := util.ObjToString(tmp["district"])
-			data := District{province, city, district}
-			if DistrictDict[district] == nil {
-				DistrictDict[district] = []District{data}
-			} else {
-				arr := DistrictDict[district]
-				arr = append(arr, data)
-				DistrictDict[district] = arr
+	}
+}
+
+func UpdateBidding() {
+	arru := make([][]map[string]interface{}, MgoBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updateBiddingPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == MgoBulkSize {
+				updateBiddingSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateBiddingSp
+					}()
+					biddingMgo.UpdateBulk(currentColl, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, MgoBulkSize)
+				indexu = 0
 			}
-		} else {
-			if city_code > 0 {
-				province := util.ObjToString(tmp["province"])
-				city := util.ObjToString(tmp["city"])
-				data := City{province, city}
-				if CityDict[city] == nil {
-					CityDict[city] = []City{data}
-				} else {
-					arr := CityDict[city]
-					arr = append(arr, data)
-					CityDict[city] = arr
-				}
-			} else {
-				province := util.ObjToString(tmp["province"])
-				data := Province{province}
-				if ProvinceDict[province] == nil {
-					ProvinceDict[province] = []Province{data}
-				} else {
-					arr := ProvinceDict[province]
-					arr = append(arr, data)
-					ProvinceDict[province] = arr
-				}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateBiddingSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateBiddingSp
+					}()
+					biddingMgo.UpdateBulk(currentColl, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, MgoBulkSize)
+				indexu = 0
 			}
 		}
-		tmp = make(map[string]interface{})
 	}
-	util.Debug(fmt.Sprintf("城市配置加载完毕...省~%d 市~%d 区~%d", len(ProvinceDict), len(CityDict), len(DistrictDict)))
 }
 
-func saveLog() {
-	arru := make([][]map[string]interface{}, 200)
+func UpdateExtract() {
+	extract := util.ObjToString(extract["collect"])
+	arru := make([][]map[string]interface{}, MgoBulkSize)
 	indexu := 0
 	for {
 		select {
-		case v := <-SaveLogChan:
+		case v := <-updateExtractPool:
 			arru[indexu] = v
 			indexu++
-			if indexu == 200 {
-				SaveSp <- true
+			if indexu == MgoBulkSize {
+				updateExtractSp <- true
 				go func(arru [][]map[string]interface{}) {
 					defer func() {
-						<-SaveSp
+						<-updateExtractSp
 					}()
-					extractmgo.UpSertBulk("createIndex_log", arru...)
+					extractMgo.UpdateBulk(extract, arru...)
 				}(arru)
-				arru = make([][]map[string]interface{}, 200)
+				arru = make([][]map[string]interface{}, MgoBulkSize)
 				indexu = 0
 			}
 		case <-time.After(1000 * time.Millisecond):
 			if indexu > 0 {
-				SaveSp <- true
+				updateExtractSp <- true
 				go func(arru [][]map[string]interface{}) {
 					defer func() {
-						<-SaveSp
+						<-updateExtractSp
 					}()
-					extractmgo.UpSertBulk("createIndex_log", arru...)
+					extractMgo.UpdateBulk(extract, arru...)
 				}(arru[:indexu])
-				arru = make([][]map[string]interface{}, 200)
+				arru = make([][]map[string]interface{}, MgoBulkSize)
 				indexu = 0
 			}
 		}

udpcreateindex/src/projectindex.go

@@ -5,7 +5,6 @@ import (
 	"math"
 	mgov "mongodb"
 	"qfw/util"
-	"qfw/util/elastic"
 	"strconv"
 	"time"
 )
@@ -35,24 +34,17 @@ func projectTask(data []byte, project, mapInfo map[string]interface{}) {
 		}
 	}
 
-	var session *mgov.MgoSess
-	if project["addr"] != nil {
-		session = project2db.GetMgoConn()
-		defer project2db.DestoryMongoConn(session)
-	} else {
-		session = extractmgo.GetMgoConn()
-		defer extractmgo.DestoryMongoConn(session)
-	}
+	conn := projectMgo.GetMgoConn()
+	defer projectMgo.DestoryMongoConn(conn)
 	c, _ := project["collect"].(string)
-	db, _ := project["db"].(string)
 	index, _ := project["index"].(string)
 	itype, _ := project["type"].(string)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
+	count, _ := conn.DB(projectMgo.DbName).C(c).Find(&q).Count()
 	savepool := make(chan bool, 10)
 
-	log.Println(db, c, "查询语句:", q, "同步总数:", count, "elastic库:", index, q["pici"])
-	query := session.DB(db).C(c).Find(q).Iter()
-	arr := make([]map[string]interface{}, savesizei)
+	log.Println(c, "查询语句:", q, "同步总数:", count, "elastic库:", index, q["pici"])
+	query := conn.DB(projectMgo.DbName).C(c).Find(q).Iter()
+	arr := make([]map[string]interface{}, MgoBulkSize)
 	var n int
 	i := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); i++ {
@@ -190,7 +182,7 @@ func projectTask(data []byte, project, mapInfo map[string]interface{}) {
 		tmp["s_projectname"] = tmp["projectname"]
 		arr[i] = tmp
 		n++
-		if i == savesizei-1 {
+		if i == MgoBulkSize-1 {
 			savepool <- true
 			tmps := arr
 			go func(tmpn *[]map[string]interface{}) {
@@ -201,24 +193,19 @@ func projectTask(data []byte, project, mapInfo map[string]interface{}) {
 					util.Debug("es队列紧张，暂停10s执行")
 					time.Sleep(time.Second * 10)
 				}
-				elastic.BulkSave(index, itype, tmpn, true)
+				Es1.BulkSave(index, itype, tmpn, true)
 			}(&tmps)
 			i = 0
-			arr = make([]map[string]interface{}, savesizei)
+			arr = make([]map[string]interface{}, MgoBulkSize)
 		}
-		if n%savesizei == 1000 {
+		if n%MgoBulkSize == 1000 {
 			log.Println("当前:", n)
 		}
 		tmp = make(map[string]interface{})
 	}
 
 	if i > 0 {
-		//util.Debug(arr)
-		if StopFlag {
-			util.Debug("es队列紧张，暂停10s执行")
-			time.Sleep(time.Second * 10)
-		}
-		elastic.BulkSave(index, itype, &arr, true)
+		Es1.BulkSave(index, itype, &arr, true)
 	}
 	log.Println(mapInfo, "create project index...over", n)
 }

udpcreateindex/src/qyxyindex.go

@@ -1,441 +0,0 @@
-package main
-
-import (
-	"log"
-	"qfw/util"
-	elastic "qfw/util/elastic"
-	"regexp"
-
-	//"sync"
-	"time"
-)
-
-var (
-	timeReg = regexp.MustCompile("[\\d]{4}-[\\d]{2}-[\\d]{2}")
-)
-
-func qyxyTask1(q map[string]interface{}) {
-	defer util.Catch()
-	//	savelock := sync.Mutex{}
-	//连接
-	session := qyxydb.GetMgoConn()
-	defer qyxydb.DestoryMongoConn(session)
-	//
-	c, _ := qyxy_ent["collect"].(string)
-	db, _ := qyxy_ent["db"].(string)
-	index, _ := qyxy_ent["index"].(string)
-	itype, _ := qyxy_ent["type"].(string)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
-	savepool := make(chan bool, 10)
-
-	log.Println("企业信用索引	查询语句:", q, "同步总数:", count, "elastic库:", index)
-	query := session.DB(db).C(c).Find(q).Iter()
-
-	arr := make([]map[string]interface{}, savesizei)
-	var n int
-	i := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
-		delete(tmp, "_id")
-		tmp["_id"] = tmp["company_id"]
-		delete(tmp, "cancels")
-		delete(tmp, "cancel_date")
-		delete(tmp, "intellectuals")
-		delete(tmp, "chattels")
-		delete(tmp, "checks")
-		delete(tmp, "revoke_date")
-		delete(tmp, "changes")
-		delete(tmp, "partners")
-
-		if tmp["establish_date"] != nil {
-			establish_date_time, ok := tmp["establish_date"].(time.Time)
-			if ok {
-				tmp["establish_date"] = establish_date_time.Unix()
-			} else {
-				tmp["establish_date"] = 0
-				util.Debug(tmp["company_id"], "establish_date")
-			}
-		}
-
-		if tmp["lastupdatetime"] != nil {
-			lastupdatetime_time, ok := tmp["lastupdatetime"].(time.Time)
-			if ok {
-				tmp["lastupdatetime"] = lastupdatetime_time.Unix()
-			} else {
-				tmp["lastupdatetime"] = 0
-				util.Debug(tmp["company_id"], "lastupdatetime")
-			}
-		}
-
-		if tmp["issue_date"] != nil {
-			issue_date_time, ok := tmp["issue_date"].(time.Time)
-			if ok {
-				tmp["issue_date"] = issue_date_time.Unix()
-			} else {
-				tmp["issue_date"] = 0
-				util.Debug(tmp["company_id"], "issue_date")
-			}
-		}
-
-		if operation_startdate, ok := tmp["operation_startdate"].(string); operation_startdate != "" && ok {
-			operation_startdate = timeReg.FindString(operation_startdate)
-			tmp["operation_startdate"] = operation_startdate + " 00:00:00"
-		}
-
-		if operation_enddate, ok := tmp["operation_enddate"].(string); operation_enddate != "" && ok {
-			operation_enddate = timeReg.FindString(operation_enddate)
-			tmp["operation_enddate"] = operation_enddate + " 00:00:00"
-		}
-
-		// if revoke_date, ok := tmp["revoke_date"].(string); revoke_date != "" && ok {
-		// 	revoke_date = timeReg.FindString(revoke_date)
-		// 	tmp["revoke_date"] = revoke_date + " 00:00:00"
-		// }
-		// if changes, ok := tmp["changes"].([]interface{}); ok && len(changes) > 0 {
-		// 	for _, change := range changes {
-		// 		if tmp1, ok := change.(map[string]interface{}); tmp1 != nil && ok && len(tmp1) > 0 {
-		// 			if change_date, ok := tmp1["change_date"].(string); ok && change_date != "" {
-		// 				change_date = timeReg.FindString(change_date)
-		// 				tmp1["change_date"] = change_date + " 00:00:00"
-		// 			}
-		// 		}
-		// 	}
-		// }
-		//operations
-		if operations, ok := tmp["operations"].([]interface{}); ok && len(operations) > 0 {
-			for _, operation := range operations {
-				if tmp1, ok := operation.(map[string]interface{}); tmp1 != nil && ok && len(tmp1) > 0 {
-					if included_time, ok := tmp1["included_time"].(string); ok && included_time != "" {
-						included_time = timeReg.FindString(included_time)
-						tmp1["included_time"] = included_time + " 00:00:00"
-					}
-					if removed_time, ok := tmp1["removed_time"].(string); ok && removed_time != "" {
-						removed_time = timeReg.FindString(removed_time)
-						tmp1["removed_time"] = removed_time + " 00:00:00"
-					}
-				}
-			}
-		}
-		//punishes
-		if punishes, ok := tmp["punishes"].([]interface{}); ok && len(punishes) > 0 {
-			for _, punishe := range punishes {
-				if tmp1, ok := punishe.(map[string]interface{}); tmp1 != nil && ok && len(tmp1) > 0 {
-					if public_date, ok := tmp1["public_date"].(string); ok && public_date != "" {
-						public_date = timeReg.FindString(public_date)
-						tmp1["public_date"] = public_date + " 00:00:00"
-					}
-					if punish_date, ok := tmp1["punish_date"].(string); ok && punish_date != "" {
-						punish_date = timeReg.FindString(punish_date)
-						tmp1["punish_date"] = punish_date + " 00:00:00"
-					}
-				}
-			}
-		}
-		//annual_reports
-		if annual_reports, ok := tmp["annual_reports"].([]interface{}); ok && len(annual_reports) > 0 {
-			for _, annual_report := range annual_reports {
-				if tmp1, ok := annual_report.(map[string]interface{}); tmp1 != nil && ok && len(tmp1) > 0 {
-					if report_changes, ok := tmp1["report_changes"].([]interface{}); ok && len(report_changes) > 0 {
-						for _, report_change := range report_changes {
-							if tmp2, ok := report_change.(map[string]interface{}); tmp2 != nil && ok && len(tmp2) > 0 {
-								if change_date, ok := tmp2["change_date"].(string); ok && change_date != "" {
-									change_date = timeReg.FindString(change_date)
-									tmp2["change_date"] = change_date + " 00:00:00"
-								}
-							}
-						}
-					}
-					if report_partners, ok := tmp1["report_partners"].([]interface{}); ok && len(report_partners) > 0 {
-						for _, report_partner := range report_partners {
-							if tmp2, ok := report_partner.(map[string]interface{}); tmp2 != nil && ok && len(tmp2) > 0 {
-								if stock_realdate, ok := tmp2["stock_realdate"].(string); ok && stock_realdate != "" {
-									stock_realdate = timeReg.FindString(stock_realdate)
-									tmp2["stock_realdate"] = stock_realdate + " 00:00:00"
-								}
-								if stock_date, ok := tmp2["stock_date"].(string); ok && stock_date != "" {
-									stock_date = timeReg.FindString(stock_date)
-									tmp2["stock_date"] = stock_date + " 00:00:00"
-								}
-							}
-						}
-					}
-					if report_equity_changes, ok := tmp1["report_equity_changes"].([]interface{}); ok && len(report_equity_changes) > 0 {
-						for _, report_equity_change := range report_equity_changes {
-							if tmp2, ok := report_equity_change.(map[string]interface{}); tmp2 != nil && ok && len(tmp2) > 0 {
-								if change_date, ok := tmp2["change_date"].(string); ok && change_date != "" {
-									change_date = timeReg.FindString(change_date)
-									tmp2["change_date"] = change_date + " 00:00:00"
-								}
-							}
-						}
-					}
-					if report_out_guarantees, ok := tmp1["report_out_guarantees"].([]interface{}); ok && len(report_out_guarantees) > 0 {
-						for _, report_out_guarantee := range report_out_guarantees {
-							if tmp2, ok := report_out_guarantee.(map[string]interface{}); tmp2 != nil && ok && len(tmp2) > 0 {
-								if perform_time, ok := tmp2["perform_time"].(string); ok && perform_time != "" {
-									perform_time = timeReg.FindString(perform_time)
-									tmp2["perform_time"] = perform_time + " 00:00:00"
-								}
-								if guarantee_time, ok := tmp2["guarantee_time"].(string); ok && guarantee_time != "" {
-									guarantee_time = timeReg.FindString(guarantee_time)
-									tmp2["guarantee_time"] = guarantee_time + " 00:00:00"
-								}
-							}
-						}
-					}
-
-				}
-			}
-		}
-		arr[i] = tmp
-		n++
-		if i == savesizei-1 {
-			savepool <- true
-			tmps := arr
-			go func(tmpn *[]map[string]interface{}) {
-				defer func() {
-					<-savepool
-				}()
-				elastic.BulkSave(index, itype, tmpn, true)
-			}(&tmps)
-			i = 0
-			arr = make([]map[string]interface{}, savesizei)
-		}
-		if n%savesizei == 0 {
-			log.Println("当前:", n)
-		}
-
-		// n++
-		// savelock.Lock()
-		// arr = append(arr, tmp)
-		// //生索引
-		// if len(arr) >= savesizei-1 {
-		// 	tmps := arr
-		// 	elastic.BulkSave(index, itype, &tmps, true)
-		// 	time.Sleep(1 * time.Second)
-		// 	arr = []map[string]interface{}{}
-		// }
-		// savelock.Unlock()
-		// //计数
-		// if n%savesizei == 0 {
-		// 	log.Println("当前:", n)
-		// }
-		tmp = make(map[string]interface{})
-
-	}
-	// savelock.Lock()
-	// if len(arr) > 0 {
-	// 	tmps := arr
-	// 	elastic.BulkSave(index, itype, &tmps, true)
-	// }
-	// savelock.Unlock()
-	if i > 0 {
-		elastic.BulkSave(index, itype, &arr, true)
-	}
-	log.Println("create qyxy index...over", n)
-}
-
-func qyxyTask(q map[string]interface{}) {
-	defer util.Catch()
-	//	savelock := sync.Mutex{}
-	//连接
-	session := qyxydb.GetMgoConn()
-	defer qyxydb.DestoryMongoConn(session)
-	//
-	c, _ := qyxy_ent["collect"].(string)
-	db, _ := qyxy_ent["db"].(string)
-	index, _ := qyxy_ent["index"].(string)
-	itype, _ := qyxy_ent["type"].(string)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
-	savepool := make(chan bool, 10)
-
-	log.Println("企业信用索引	查询语句:", q, "同步总数:", count, "elastic库:", index)
-	query := session.DB(db).C(c).Find(q).Iter()
-
-	arr := make([]map[string]interface{}, savesizei)
-	var n int
-	i := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
-		//delete(tmp, "_id")
-		tmp["_id"] = tmp["company_id"]
-		// delete(tmp, "cancels")
-		// delete(tmp, "cancel_date")
-		// delete(tmp, "intellectuals")
-		// delete(tmp, "chattels")
-		// delete(tmp, "checks")
-		// delete(tmp, "revoke_date")
-		delete(tmp, "changes")
-		// delete(tmp, "partners")
-
-		// if tmp["establish_date"] != nil {
-		// 	establish_date_time, ok := tmp["establish_date"].(time.Time)
-		// 	if ok {
-		// 		tmp["establish_date"] = establish_date_time.Unix()
-		// 	} else {
-		// 		tmp["establish_date"] = 0
-		// 		util.Debug(tmp["company_id"], "establish_date")
-		// 	}
-		// }
-
-		// if tmp["lastupdatetime"] != nil {
-		// 	lastupdatetime_time, ok := tmp["lastupdatetime"].(time.Time)
-		// 	if ok {
-		// 		tmp["lastupdatetime"] = lastupdatetime_time.Unix()
-		// 	} else {
-		// 		tmp["lastupdatetime"] = 0
-		// 		util.Debug(tmp["company_id"], "lastupdatetime")
-		// 	}
-		// }
-
-		// if tmp["issue_date"] != nil {
-		// 	issue_date_time, ok := tmp["issue_date"].(time.Time)
-		// 	if ok {
-		// 		tmp["issue_date"] = issue_date_time.Unix()
-		// 	} else {
-		// 		tmp["issue_date"] = 0
-		// 		util.Debug(tmp["company_id"], "issue_date")
-		// 	}
-		// }
-
-		// if operation_startdate, ok := tmp["operation_startdate"].(string); operation_startdate != "" && ok {
-		// 	operation_startdate = timeReg.FindString(operation_startdate)
-		// 	tmp["operation_startdate"] = operation_startdate + " 00:00:00"
-		// }
-
-		// if operation_enddate, ok := tmp["operation_enddate"].(string); operation_enddate != "" && ok {
-		// 	operation_enddate = timeReg.FindString(operation_enddate)
-		// 	tmp["operation_enddate"] = operation_enddate + " 00:00:00"
-		// }
-
-		// //operations
-		// if operations, ok := tmp["operations"].([]interface{}); ok && len(operations) > 0 {
-		// 	for _, operation := range operations {
-		// 		if tmp1, ok := operation.(map[string]interface{}); tmp1 != nil && ok && len(tmp1) > 0 {
-		// 			if included_time, ok := tmp1["included_time"].(string); ok && included_time != "" {
-		// 				included_time = timeReg.FindString(included_time)
-		// 				tmp1["included_time"] = included_time + " 00:00:00"
-		// 			}
-		// 			if removed_time, ok := tmp1["removed_time"].(string); ok && removed_time != "" {
-		// 				removed_time = timeReg.FindString(removed_time)
-		// 				tmp1["removed_time"] = removed_time + " 00:00:00"
-		// 			}
-		// 		}
-		// 	}
-		// }
-		// //punishes
-		// if punishes, ok := tmp["punishes"].([]interface{}); ok && len(punishes) > 0 {
-		// 	for _, punishe := range punishes {
-		// 		if tmp1, ok := punishe.(map[string]interface{}); tmp1 != nil && ok && len(tmp1) > 0 {
-		// 			if public_date, ok := tmp1["public_date"].(string); ok && public_date != "" {
-		// 				public_date = timeReg.FindString(public_date)
-		// 				tmp1["public_date"] = public_date + " 00:00:00"
-		// 			}
-		// 			if punish_date, ok := tmp1["punish_date"].(string); ok && punish_date != "" {
-		// 				punish_date = timeReg.FindString(punish_date)
-		// 				tmp1["punish_date"] = punish_date + " 00:00:00"
-		// 			}
-		// 		}
-		// 	}
-		// }
-		// //annual_reports
-		// if annual_reports, ok := tmp["annual_reports"].([]interface{}); ok && len(annual_reports) > 0 {
-		// 	for _, annual_report := range annual_reports {
-		// 		if tmp1, ok := annual_report.(map[string]interface{}); tmp1 != nil && ok && len(tmp1) > 0 {
-		// 			if report_changes, ok := tmp1["report_changes"].([]interface{}); ok && len(report_changes) > 0 {
-		// 				for _, report_change := range report_changes {
-		// 					if tmp2, ok := report_change.(map[string]interface{}); tmp2 != nil && ok && len(tmp2) > 0 {
-		// 						if change_date, ok := tmp2["change_date"].(string); ok && change_date != "" {
-		// 							change_date = timeReg.FindString(change_date)
-		// 							tmp2["change_date"] = change_date + " 00:00:00"
-		// 						}
-		// 					}
-		// 				}
-		// 			}
-		// 			if report_partners, ok := tmp1["report_partners"].([]interface{}); ok && len(report_partners) > 0 {
-		// 				for _, report_partner := range report_partners {
-		// 					if tmp2, ok := report_partner.(map[string]interface{}); tmp2 != nil && ok && len(tmp2) > 0 {
-		// 						if stock_realdate, ok := tmp2["stock_realdate"].(string); ok && stock_realdate != "" {
-		// 							stock_realdate = timeReg.FindString(stock_realdate)
-		// 							tmp2["stock_realdate"] = stock_realdate + " 00:00:00"
-		// 						}
-		// 						if stock_date, ok := tmp2["stock_date"].(string); ok && stock_date != "" {
-		// 							stock_date = timeReg.FindString(stock_date)
-		// 							tmp2["stock_date"] = stock_date + " 00:00:00"
-		// 						}
-		// 					}
-		// 				}
-		// 			}
-		// 			if report_equity_changes, ok := tmp1["report_equity_changes"].([]interface{}); ok && len(report_equity_changes) > 0 {
-		// 				for _, report_equity_change := range report_equity_changes {
-		// 					if tmp2, ok := report_equity_change.(map[string]interface{}); tmp2 != nil && ok && len(tmp2) > 0 {
-		// 						if change_date, ok := tmp2["change_date"].(string); ok && change_date != "" {
-		// 							change_date = timeReg.FindString(change_date)
-		// 							tmp2["change_date"] = change_date + " 00:00:00"
-		// 						}
-		// 					}
-		// 				}
-		// 			}
-		// 			if report_out_guarantees, ok := tmp1["report_out_guarantees"].([]interface{}); ok && len(report_out_guarantees) > 0 {
-		// 				for _, report_out_guarantee := range report_out_guarantees {
-		// 					if tmp2, ok := report_out_guarantee.(map[string]interface{}); tmp2 != nil && ok && len(tmp2) > 0 {
-		// 						if perform_time, ok := tmp2["perform_time"].(string); ok && perform_time != "" {
-		// 							perform_time = timeReg.FindString(perform_time)
-		// 							tmp2["perform_time"] = perform_time + " 00:00:00"
-		// 						}
-		// 						if guarantee_time, ok := tmp2["guarantee_time"].(string); ok && guarantee_time != "" {
-		// 							guarantee_time = timeReg.FindString(guarantee_time)
-		// 							tmp2["guarantee_time"] = guarantee_time + " 00:00:00"
-		// 						}
-		// 					}
-		// 				}
-		// 			}
-
-		// 		}
-		// 	}
-		// }
-		arr[i] = tmp
-		n++
-		if i == savesizei-1 {
-			savepool <- true
-			tmps := arr
-			go func(tmpn *[]map[string]interface{}) {
-				defer func() {
-					<-savepool
-				}()
-				elastic.BulkSave(index, itype, tmpn, true)
-			}(&tmps)
-			i = 0
-			arr = make([]map[string]interface{}, savesizei)
-		}
-		if n%savesizei == 0 {
-			log.Println("当前:", n)
-		}
-
-		// n++
-		// savelock.Lock()
-		// arr = append(arr, tmp)
-		// //生索引
-		// if len(arr) >= savesizei-1 {
-		// 	tmps := arr
-		// 	elastic.BulkSave(index, itype, &tmps, true)
-		// 	time.Sleep(1 * time.Second)
-		// 	arr = []map[string]interface{}{}
-		// }
-		// savelock.Unlock()
-		// //计数
-		// if n%savesizei == 0 {
-		// 	log.Println("当前:", n)
-		// }
-		tmp = make(map[string]interface{})
-
-	}
-	// savelock.Lock()
-	// if len(arr) > 0 {
-	// 	tmps := arr
-	// 	elastic.BulkSave(index, itype, &tmps, true)
-	// }
-	// savelock.Unlock()
-	if i > 0 {
-		elastic.BulkSave(index, itype, &arr, true)
-	}
-	log.Println("create qyxy index...over", n)
-}

udpcreateindex/src/standardata.go

@@ -6,7 +6,6 @@ import (
 	mgov "mongodb"
 	"qfw/util"
 	elastic "qfw/util/elastic"
-
 )
 
 func standardTask(stype string, mapInfo map[string]interface{}) {
@@ -32,20 +31,19 @@ func standardTask(stype string, mapInfo map[string]interface{}) {
 
 //winnerent
 func winnerEnt(q map[string]interface{}) {
-	session := mgostandard.GetMgoConn()
-	defer mgostandard.DestoryMongoConn(session)
-	db, _ := standard["db"].(string)
+	session := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(session)
 	winnerent, _ := standard["winnerent"].(map[string]interface{})
 	c, _ := winnerent["collect"].(string)
 	index, _ := winnerent["index"].(string)
 	itype, _ := winnerent["type"].(string)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
+	count, _ := session.DB(standardMgo.DbName).C(c).Find(&q).Count()
 	savepool := make(chan bool, 10)
 
-	log.Println(db, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
+	log.Println(standardMgo.DbName, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
 
-	query := session.DB(db).C(c).Find(q).Iter()
-	arr := make([]map[string]interface{}, savesizei)
+	query := session.DB(standardMgo.DbName).C(c).Find(q).Iter()
+	arr := make([]map[string]interface{}, EsBulkSize)
 	var n int
 	i := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
@@ -56,7 +54,7 @@ func winnerEnt(q map[string]interface{}) {
 		tmp["company"] = tmp["company_name"]
 		arr[i] = tmp
 		n++
-		if i == savesizei-1 {
+		if i == EsBulkSize-1 {
 			savepool <- true
 			tmps := arr
 			go func(tmpn *[]map[string]interface{}) {
@@ -66,9 +64,9 @@ func winnerEnt(q map[string]interface{}) {
 				elastic.BulkSave(index, itype, tmpn, true)
 			}(&tmps)
 			i = 0
-			arr = make([]map[string]interface{}, savesizei)
+			arr = make([]map[string]interface{}, EsBulkSize)
 		}
-		if n%savesizei == 0 {
+		if n%EsBulkSize == 0 {
 			log.Println("当前:", n)
 		}
 		tmp = make(map[string]interface{})
@@ -81,20 +79,19 @@ func winnerEnt(q map[string]interface{}) {
 
 //buyerent
 func buyerEnt(q map[string]interface{}) {
-	session := mgostandard.GetMgoConn()
-	defer mgostandard.DestoryMongoConn(session)
-	db, _ := standard["db"].(string)
+	session := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(session)
 	buyerent, _ := standard["buyerent"].(map[string]interface{})
 	c, _ := buyerent["collect"].(string)
 	index, _ := buyerent["index"].(string)
 	itype, _ := buyerent["type"].(string)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
+	count, _ := session.DB(standardMgo.DbName).C(c).Find(&q).Count()
 	savepool := make(chan bool, 10)
 
-	log.Println(db, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
+	log.Println(standardMgo.DbName, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
 
-	query := session.DB(db).C(c).Find(q).Iter()
-	arr := make([]map[string]interface{}, savesizei)
+	query := session.DB(standardMgo.DbName).C(c).Find(q).Iter()
+	arr := make([]map[string]interface{}, EsBulkSize)
 	var n int
 	i := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
@@ -105,7 +102,7 @@ func buyerEnt(q map[string]interface{}) {
 		tmp["buyer"] = tmp["buyer_name"]
 		arr[i] = tmp
 		n++
-		if i == savesizei-1 {
+		if i == EsBulkSize-1 {
 			savepool <- true
 			tmps := arr
 			go func(tmpn *[]map[string]interface{}) {
@@ -115,9 +112,9 @@ func buyerEnt(q map[string]interface{}) {
 				elastic.BulkSave(index, itype, tmpn, true)
 			}(&tmps)
 			i = 0
-			arr = make([]map[string]interface{}, savesizei)
+			arr = make([]map[string]interface{}, EsBulkSize)
 		}
-		if n%savesizei == 0 {
+		if n%EsBulkSize == 0 {
 			log.Println("当前:", n)
 		}
 		tmp = make(map[string]interface{})
@@ -130,20 +127,19 @@ func buyerEnt(q map[string]interface{}) {
 
 //agencyent
 func agencyEnt(q map[string]interface{}) {
-	session := mgostandard.GetMgoConn()
-	defer mgostandard.DestoryMongoConn(session)
-	db, _ := standard["db"].(string)
+	session := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(session)
 	agencyent, _ := standard["agencyent"].(map[string]interface{})
 	c, _ := agencyent["collect"].(string)
 	index, _ := agencyent["index"].(string)
 	itype, _ := agencyent["type"].(string)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
+	count, _ := session.DB(standardMgo.DbName).C(c).Find(&q).Count()
 	savepool := make(chan bool, 10)
 
-	log.Println(db, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
+	log.Println(standardMgo.DbName, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
 
-	query := session.DB(db).C(c).Find(q).Iter()
-	arr := make([]map[string]interface{}, savesizei)
+	query := session.DB(standardMgo.DbName).C(c).Find(q).Iter()
+	arr := make([]map[string]interface{}, EsBulkSize)
 	var n int
 	i := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
@@ -155,7 +151,7 @@ func agencyEnt(q map[string]interface{}) {
 		tmp["agency"] = tmp["agency_name"]
 		arr[i] = tmp
 		n++
-		if i == savesizei-1 {
+		if i == EsBulkSize-1 {
 			savepool <- true
 			tmps := arr
 			go func(tmpn *[]map[string]interface{}) {
@@ -165,9 +161,9 @@ func agencyEnt(q map[string]interface{}) {
 				elastic.BulkSave(index, itype, tmpn, true)
 			}(&tmps)
 			i = 0
-			arr = make([]map[string]interface{}, savesizei)
+			arr = make([]map[string]interface{}, EsBulkSize)
 		}
-		if n%savesizei == 0 {
+		if n%EsBulkSize == 0 {
 			log.Println("当前:", n)
 		}
 		tmp = make(map[string]interface{})

udpcreateindex/src/task.go

@@ -2,13 +2,8 @@
 package main
 
 import (
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"log"
-	"mongodb"
-	qutil "qfw/util"
-	"time"
-
 	"github.com/cron"
+	qutil "qfw/util"
 )
 
 func task_index() {
@@ -31,64 +26,14 @@ func task_buyeres() {
 	buyerEsTaskOnce()
 }
 
-func task_biddingAll() {
-	qutil.Debug("定时任务，bidding_all")
-	t := time.Now()
-	currenttime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
-	endtime := time.Unix(currenttime.Unix(), 0).AddDate(0, 0, -1)
-
-	sid := primitive.NewObjectIDFromTimestamp(endtime)
-	eid := primitive.NewObjectIDFromTimestamp(currenttime)
-
-	biddingDataTask(nil, map[string]interface{}{"gtid": mongodb.BsonIdToSId(sid), "lteid": mongodb.BsonIdToSId(eid)})
-}
-
-//招标附件、标的物,临时用
-func task_biddingfile() {
-	defer qutil.Catch()
-	q := map[string]interface{}{
-		"comeintime": map[string]interface{}{
-			"$gte": time.Now().Unix() - 5400,
-			"$lte": time.Now().Unix() - 3600,
-		},
-		"attach_text": map[string]interface{}{
-			"$exists": true,
-		},
-	}
-	biddingPurchaingTask(q)
-}
-
-//project2项目索引
-func task_projects() {
-	t := time.Now()
-	pici := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local).Unix()
-	log.Println(pici)
-	mapInfo := map[string]interface{}{
-		"query": map[string]interface{}{
-			"pici": map[string]interface{}{
-				"$gte": pici - 86400,
-				"$lte": pici,
-			},
-		},
-	}
-	projectTask([]byte{}, project2, mapInfo)
-}
-
-//企业信用信息
-func task_qyxyindex() {
-	defer qutil.Catch()
-	q := map[string]interface{}{}
-	qyxyTask(q)
-}
-
-//定时任务site：中国招标投标公共服务平台
-func crontab() {
-	defer qutil.Catch()
-	q := map[string]interface{}{
-		"_id": map[string]interface{}{
-			"$gte": mongodb.StringTOBsonId("5e344f000000000000000000"), //2020-02-01 5e344f000000000000000000
-			"$lte": mongodb.StringTOBsonId("5ea1bb800000000000000000"), //2020-04-23 5ea1bb800000000000000000
-		},
-	}
-	site_attach_text(q)
-}
+//func task_biddingAll() {
+//	qutil.Debug("定时任务，bidding_all")
+//	t := time.Now()
+//	currenttime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
+//	endtime := time.Unix(currenttime.Unix(), 0).AddDate(0, 0, -1)
+//
+//	sid := primitive.NewObjectIDFromTimestamp(endtime)
+//	eid := primitive.NewObjectIDFromTimestamp(currenttime)
+//
+//	//biddingDataTask(nil, map[string]interface{}{"gtid": mongodb.BsonIdToSId(sid), "lteid": mongodb.BsonIdToSId(eid)})
+//}

udpcreateindex/src/winnerindex.go

@@ -6,7 +6,6 @@ import (
 	"mongodb"
 	"qfw/util"
 	elastic "qfw/util/elastic"
-
 )
 
 func winnerTask(data []byte, mapInfo map[string]interface{}) {
@@ -20,26 +19,25 @@ func winnerTask(data []byte, mapInfo map[string]interface{}) {
 			},
 		}
 	}
-	session := extractmgo.GetMgoConn()
-	defer extractmgo.DestoryMongoConn(session)
+	session := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(session)
 	c, _ := winner["collect"].(string)
-	db, _ := winner["db"].(string)
 	index, _ := winner["index"].(string)
 	itype, _ := winner["type"].(string)
-	count, _ := session.DB(db).C(c).Find(&q).Count()
+	count, _ := session.DB(standardMgo.DbName).C(c).Find(&q).Count()
 	savepool := make(chan bool, 10)
 
 	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
-	query := session.DB(db).C(c).Find(q).Select(bson.M{"pici": 0}).Iter()
+	query := session.DB(standardMgo.DbName).C(c).Find(q).Select(bson.M{"pici": 0}).Iter()
 
-	arr := make([]map[string]interface{}, savesizei)
+	arr := make([]map[string]interface{}, EsBulkSize)
 	var n int
 	i := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
 		go IS.Add("winner")
 		arr[i] = tmp
 		n++
-		if i == savesizei-1 {
+		if i == EsBulkSize-1 {
 			savepool <- true
 			tmps := arr
 			go func(tmpn *[]map[string]interface{}) {
@@ -49,9 +47,9 @@ func winnerTask(data []byte, mapInfo map[string]interface{}) {
 				elastic.BulkSave(index, itype, tmpn, true)
 			}(&tmps)
 			i = 0
-			arr = make([]map[string]interface{}, savesizei)
+			arr = make([]map[string]interface{}, EsBulkSize)
 		}
-		if n%savesizei == 0 {
+		if n%EsBulkSize == 0 {
 			log.Println("当前:", n)
 		}
 		tmp = make(map[string]interface{})

udpcreateindex/src/winnertask.go

@@ -5,7 +5,6 @@ import (
 	"log"
 	"mongodb"
 	qu "qfw/util"
-	elastic "qfw/util/elastic"
 	"sync"
 	"time"
 )
@@ -39,11 +38,11 @@ func winnerEsTaskOnce() {
 	index, _ := winnerent["index"].(string)
 	itype, _ := winnerent["type"].(string)
 	//mongo
-	sess := mgostandard.GetMgoConn()
-	defer mgostandard.DestoryMongoConn(sess)
+	sess := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(sess)
 
-	log.Println("q:", q, "db:", mgostandard.DbName, "coll:", win_ent)
-	it_1 := sess.DB(mgostandard.DbName).C(win_ent).Find(&q).Sort("_id").Iter()
+	log.Println("q:", q, "db:", standardMgo.DbName, "coll:", win_ent)
+	it_1 := sess.DB(standardMgo.DbName).C(win_ent).Find(&q).Sort("_id").Iter()
 	num_1 := 0
 	for tmp := make(map[string]interface{}); it_1.Next(&tmp); num_1++ {
 		if num_1%100 == 0 && num_1 > 0 {
@@ -74,9 +73,9 @@ func winnerEsTaskOnce() {
 
 			winerEsLock.Lock()
 			arrEs = append(arrEs, savetmp)
-			if len(arrEs) >= BulkSize {
+			if len(arrEs) >= EsBulkSize {
 				tmps := arrEs
-				elastic.BulkSave(index, itype, &tmps, true)
+				Es1.BulkSave(index, itype, &tmps, true)
 				arrEs = []map[string]interface{}{}
 			}
 			winerEsLock.Unlock()
@@ -120,7 +119,7 @@ func winnerEsTaskOnce() {
 	winerEsLock.Lock()
 	if len(arrEs) > 0 {
 		tmps := arrEs
-		elastic.BulkSave(index, itype, &tmps, true)
+		Es1.BulkSave(index, itype, &tmps, true)
 		arrEs = []map[string]interface{}{}
 	}
 	winerEsLock.Unlock()