no message

forecast/es/config.json

@@ -0,0 +1,39 @@
+{
+  "mgodb": "192.168.3.207:27092",
+  "dbsize": 12,
+  "dbname": "wjh",
+  "dbcoll": "project_forecast",
+  "uname": "",
+  "upwd": "",
+  "tasktime": 0,
+  "updatetime": 0,
+  "elastic": {
+    "addr": "http://192.168.3.206:9800",
+    "index": "forecast_v1",
+    "itype": "forecast",
+    "pool": 12,
+    "esfields": [
+      "_id",
+      "area",
+      "city",
+      "nature",
+      "buyer",
+      "buyerclass",
+      "title",
+      "projectname",
+      "projectcode",
+      "spidercode",
+      "stage",
+      "main_project",
+      "top_category",
+      "sub_category",
+      "approvestatus",
+      "href",
+      "jyhref",
+      "publishtime",
+      "yucetime",
+      "yucestarttime",
+      "yuceendtime"
+    ]
+  }
+}

forecast/es/main.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"mongodb"
+	qu "qfw/util"
+	es "qfw/util/elastic"
+)
+
+var (
+	Sysconfig        map[string]interface{} //配置文件
+	Mgo              *mongodb.MongodbSim
+	Dbname           string
+	Dbcoll           string
+	Es               *es.Elastic
+	Index            string
+	Itype            string
+	EsFields         []string
+	TaskTime         int
+	Updatetime       int64
+)
+var EsSaveCache = make(chan map[string]interface{}, 5000)
+var SP = make(chan bool, 5)
+
+func init() {
+	qu.ReadConfig(&Sysconfig)
+	Dbname = Sysconfig["dbname"].(string)
+	Dbcoll = Sysconfig["dbcoll"].(string)
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: Sysconfig["mgodb"].(string),
+		Size:        qu.IntAllDef(Sysconfig["dbsize"], 5),
+		DbName:      Dbname,
+		UserName: 	 Sysconfig["uname"].(string),
+		Password: 	 Sysconfig["upwd"].(string),
+	}
+	Mgo.InitPool()
+	//es
+	econf := Sysconfig["elastic"].(map[string]interface{})
+	Index = econf["index"].(string)
+	Itype = econf["itype"].(string)
+	Es = &es.Elastic{
+		S_esurl: econf["addr"].(string),
+		I_size:  qu.IntAllDef(econf["pool"], 12),
+	}
+	Es.InitElasticSize()
+	EsFields = qu.ObjArrToStringArr(econf["esfields"].([]interface{}))
+	//TaskTime = qu.IntAll(Sysconfig["tasktime"])
+	Updatetime = qu.Int64All(Sysconfig["updatetime"])
+
+}
+
+func main() {
+	go SaveEs()
+
+	//go TimeTask()
+	go SaveAll()
+	//go SaveAdd()
+	ch := make(chan bool, 1)
+	<-ch
+}

forecast/es/task.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	"github.com/cron"
+	"go.mongodb.org/mongo-driver/bson"
+	"log"
+	"qfw/util"
+	"reflect"
+	"sync"
+	"time"
+)
+
+//定时任务
+func TimeTask() {
+	c := cron.New()
+	cronstr := "0 0 15 ? * Tue" //每周二15点执行
+	//cronstr := "0 */" + fmt.Sprint(TaskTime) + " * * * ?" //每TaskTime小时执行一次
+	err := c.AddFunc(cronstr, func() { SaveAdd() })
+	if err != nil {
+		util.Debug(err)
+		return 
+	}
+	c.Start()
+}
+
+// SaveAdd 增量数据
+func SaveAdd() {
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
+	//q := bson.M{"_id": "affe29f8d061f3faa4170cafba41f316"}
+	q := bson.M{"updatetime": bson.M{"$gt": Updatetime}}
+	util.Debug(q)
+	it := sess.DB(Dbname).C(Dbcoll).Find(q).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%5000 == 0 {
+			log.Println("current:", count)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			esMap := map[string]interface{}{}
+			//生索引字段处理
+			for _, field := range EsFields {
+				if tmp[field] == nil {
+					continue
+				}
+
+			}
+
+			EsSaveCache <- esMap		// 保存es
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("Run Over...Count:", count)
+}
+
+// SaveAll 存量数据生es
+func SaveAll() {
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+	//q := bson.M{"_id": mongodb.StringTOBsonId("6194a3c105180be8dae19cbb")}
+	it := sess.DB(Dbname).C(Dbcoll).Find(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%20000 == 0 {
+			log.Println("current:", count, tmp["_id"])
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			esMap := map[string]interface{}{}
+			//生索引字段处理
+			for _, field := range EsFields {
+				if tmp[field] == nil {
+					continue
+				}
+				if field == "buyerclass" {
+					if reflect.TypeOf(tmp["buyerclass"]).String() == "[]interface {}" {
+						esMap["buyerclass"] = util.ObjArrToStringArr(tmp["buyerclass"].([]interface{}))[0]
+					}else {
+						esMap["buyerclass"] = tmp["buyerclass"]
+					}
+				}else {
+					esMap[field] = tmp[field]
+				}
+			}
+			// 处理result
+			if mp, ok := tmp["results"].([]interface{}); ok {
+				var mpArr []map[string]interface{}
+				for _, v := range mp{
+					v1 := v.(map[string]interface{})
+					if v1["purchasing"] != nil {
+						mpArr = append(mpArr, map[string]interface{}{"purchasing": v1["purchasing"]})
+					}
+				}
+				if len(mpArr) > 0 {
+					esMap["results"] = mpArr
+				}
+			}
+			EsSaveCache <- esMap
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("Run Over...Count:", count)
+}
+
+// SaveEs 过滤后数据存库
+func SaveEs() {
+	log.Println("Es Save...")
+	arru := make([]map[string]interface{}, 100)
+	indexu := 0
+	for {
+		select {
+		case v := <-EsSaveCache:
+			arru[indexu] = v
+			indexu++
+			if indexu == 100 {
+				SP <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-SP
+					}()
+					Es.BulkSave(Index, Itype, &arru, true)
+				}(arru)
+				arru = make([]map[string]interface{}, 100)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				SP <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-SP
+					}()
+					Es.BulkSave(Index, Itype, &arru, true)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, 100)
+				indexu = 0
+			}
+		}
+	}
+}

forecast/project-yece/main.go

@@ -0,0 +1,205 @@
+package main
+
+import (
+	"mongodb"
+	es "qfw/util/elastic"
+	"time"
+)
+
+var (
+	MongoTool			*mongodb.MongodbSim
+	Es              	*es.Elastic
+
+	updatePool			chan []map[string]interface{}
+	updateSp			chan bool
+	updatePool1			chan []map[string]interface{}
+	updateSp1			chan bool
+	saveSize			int
+	savePool			chan map[string]interface{}
+	saveSp				chan bool
+	savePool1			chan map[string]interface{}
+	saveSp1				chan bool
+
+)
+
+func init() {
+	MongoTool = &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",				// 172.17.4.187:27082,172.17.145.163:27083
+		Size:        10,
+		DbName:      "mixdata",
+		UserName:	 "SJZY_RWESBid_Other",
+		Password: 	 "SJZY@O17t8herB3B",
+	}
+	MongoTool.InitPool()
+
+	Es = &es.Elastic{
+		S_esurl: "http://172.17.145.170:9800",									//http://172.17.145.170:9800
+		I_size:  10,
+	}
+	Es.InitElasticSize()
+
+	saveSize = 200
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp = make(chan bool, 5)
+	updatePool1 = make(chan []map[string]interface{}, 5000)
+	updateSp1 = make(chan bool, 5)
+	savePool = make(chan map[string]interface{}, 5000)
+	saveSp = make(chan bool, 5)
+	savePool1 = make(chan map[string]interface{}, 5000)
+	saveSp1 = make(chan bool, 5)
+}
+
+func main() {
+	go saveMethod()
+	go saveMethod1()
+	go updateMethod()
+	//go updateMethod1()
+
+	//go findEs()
+	//go fcResult()
+	go TimeTask()
+
+	ch := make(chan bool, 1)
+	<-ch
+}
+
+
+func saveMethod() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-savePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MongoTool.SaveBulk("project_forecast_yece_tmp", arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MongoTool.SaveBulk("project_forecast_yece_tmp", arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func saveMethod1() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-savePool1:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveSp1 <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp1
+					}()
+					MongoTool.SaveBulk("project_forecast", arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveSp1 <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp1
+					}()
+					MongoTool.SaveBulk("project_forecast", arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func updateMethod() {
+	arru := make([][]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoTool.UpSertBulk("project_forecast_yece_tmp", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoTool.UpSertBulk("project_forecast_yece_tmp", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func updateMethod1() {
+	arru := make([][]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool1:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				updateSp1 <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp1
+					}()
+					MongoTool.UpSertBulk("project_forecast", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp1 <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp1
+					}()
+					MongoTool.UpSertBulk("project_forecast", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

forecast/project-yece/task.go

@@ -0,0 +1,331 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/cron"
+	"go.mongodb.org/mongo-driver/bson"
+	es "gopkg.in/olivere/elastic.v1"
+	"mongodb"
+	"qfw/util"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+)
+
+var date1 = regexp.MustCompile("20[0-2][0-9][年|\\-\\/|.][0-9]{1,2}[月|\\-|\\/|.][0-9]{1,2}[日]?")
+
+func TimeTask() {
+	c := cron.New()
+	cronstr := "0 0 2 * * ?"  //每天3点执行		临时表数据
+	cronstr1 := "0 0 3 * * ?" //每天4点执行		结果表数据
+	_ = c.AddFunc(cronstr, func() {
+		findEs()
+	})
+	_ = c.AddFunc(cronstr1, func() {
+		fcResult()
+	})
+
+	c.Start()
+}
+
+func findEs() {
+	util.Debug("预测结果时间-------临时表保存数据----------")
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+
+	ch := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
+	//esquery := `{
+	//"query": {
+	//	"bool": {
+	//	  "must": [
+	//		{
+	//		  "terms": {
+	//			"subtype": [
+	//			  "成交",
+	//			  "合同"
+	//			]
+	//		  }
+	//		},
+	//		{
+	//		  "terms": {
+	//			"s_subscopeclass": [
+	//			  "服务采购_法律咨询",
+	//			  "服务采购_会计",
+	//			  "服务采购_物业",
+	//			  "服务采购_审计",
+	//			  "服务采购_安保",
+	//			  "服务采购_仓储物流",
+	//			  "服务采购_广告宣传印刷"
+	//			]
+	//		  }
+	//		}
+	//	  ]
+	//	}
+	//},
+	//"_source": [
+	//	"_id",
+	//	"title",
+	//	"buyer",
+	//	"buyerclass",
+	//	"s_subscopeclass",
+	//	"yuceendtime",
+	//	"area",
+	//	"city",
+	//	"subtype",
+	//	"projectname",
+	//	"purchasing",
+	//	"href",
+	//	"projectcode",
+	//	"publishtime",
+	//	"buyerperson",
+	//	"buyertel"]
+	//}`
+
+	//esquery := `{"query":{"bool":{"must":[{"term":{"id":"6168359c06a9d911e598d573"}}]}}}`
+	currenttime := time.Now().Unix()
+	stime := time.Unix(currenttime, 0).AddDate(0, 0, -1).Unix()
+	esquery := `{"query":{"bool":{"must":[{"range":{"comeintime":{"from":` + fmt.Sprint(stime) + `,"to":` + fmt.Sprint(currenttime) + `}}}],"must_not":[{"constant_score":{"filter":{"missing":{"field":"yuceendtime"}}}}]}}}`
+	util.Debug(esquery)
+	fieldArr := []string{"_id", "title", "buyer", "buyerclass", "s_subscopeclass", "yuceendtime", "area", "city", "subtype",
+		"projectname", "purchasing", "href", "projectcode", "publishtime", "buyerperson", "buyertel", "projectperiod",
+		"project_duration", "project_timeunit", "signaturedate"}
+	escount := Es.Count("bidding", "bidding", esquery)
+	util.Debug("查询总数:", escount)
+	//查询条件类型转换
+	var q es.Query
+	tmpQuery := es.BoolQuery{
+		QueryStrings: esquery,
+	}
+	q = tmpQuery
+	numDocs := 0
+
+	//游标查询，index不支持别名，只能写索引库的名称
+	res, err := client.Scroll("bidding_v1").Query(q).Size(500).Do() //查询一条获取游标
+	if err == nil {
+		scrollId := res.ScrollId
+		for {
+			if scrollId == "" {
+				util.Debug("ScrollId Is Error")
+				break
+			}
+			searchResult, err := client.Scroll("bidding_v1").Size(500).ScrollId(scrollId).Do() //查询
+			if err != nil {
+				if err.Error() == "EOS" { //迭代完毕
+					util.Debug("Es Search Data Over：", err)
+				} else {
+					util.Debug("Es Search Data Error：", err)
+				}
+				break
+			}
+			for _, hit := range searchResult.Hits.Hits {
+				//开始处理数据
+				wg.Add(1)
+				ch <- true
+				go func(tmpHit *es.SearchHit) {
+					defer func() {
+						<-ch
+						wg.Done()
+					}()
+					tmp := make(map[string]interface{})
+					if json.Unmarshal(*tmpHit.Source, &tmp) == nil {
+						save := make(map[string]interface{})
+						for _, v := range fieldArr {
+							if tmp[v] != nil {
+								save[v] = tmp[v]
+							}
+						}
+						//istart, iend := YcTime(save)
+						//if istart > 0 && iend > 0 {
+						//	save["yucestarttime"] = istart
+						//	save["yuceendtime"] = iend
+						//
+						//	upStr := fmt.Sprintf("ctx._source.yuceendtime=%d", iend)
+						//	if tmp["yeceendtime"] != nil {
+						//		upStr += fmt.Sprintf(";ctx._source.yeceendtime=%d", 0)
+						//	}
+						//	updateEs := map[string]string{
+						//		"id": util.ObjToString(tmp["_id"]),
+						//		"updateStr": upStr,
+						//	}
+						//	Es.UpdateOne("bidding", "bidding", updateEs)
+						//}
+						savePool <- save
+						//update := []map[string]interface{}{{
+						//	"_id": save["_id"],
+						//},
+						//	{"$set": save},
+						//}
+						//updatePool <- update
+					}
+				}(hit)
+				numDocs += 1
+				if numDocs%100 == 0 {
+					util.Debug("Current：", numDocs)
+				}
+			}
+			scrollId = searchResult.ScrollId
+		}
+		wg.Wait()
+		util.Debug("over---", numDocs)
+		client.ClearScroll().ScrollId(scrollId).Do() //清理游标
+	}
+}
+
+func YcTime(tmp map[string]interface{}) (int64, int64) {
+	// yucestarttime、yuceendtime
+	yucestarttime := int64(0)
+	yuceendtime := int64(0)
+	// 项目周期中
+	if util.ObjToString(tmp["projectperiod"]) != "" {
+		dateStr := date1.FindStringSubmatch(util.ObjToString(tmp["projectperiod"]))
+		if len(dateStr) == 2 {
+			sdate := FormatDateStr(dateStr[0])
+			edate := FormatDateStr(dateStr[1])
+			if sdate < edate && sdate != 0 && edate != 0 {
+				yucestarttime = sdate
+				yuceendtime = edate
+			}
+		}
+	}
+	if yucestarttime > 0 && yuceendtime > yucestarttime {
+		return yucestarttime, yuceendtime
+	}
+	// 预测开始时间 合同签订日期
+	if util.IntAll(tmp["signaturedate"]) <= 0 {
+		if util.IntAll(tmp["publishtime"]) <= 0 {
+			return 0, 0
+		} else {
+			yucestarttime = util.Int64All(tmp["publishtime"])
+		}
+	} else {
+		yucestarttime = util.Int64All(tmp["signaturedate"])
+	}
+
+	// 预测结束时间
+	if yucestarttime > 0 {
+		if util.IntAll(tmp["project_duration"]) > 0 && util.ObjToString(tmp["project_timeunit"]) != "" {
+			yuceendtime = YcEndTime(yucestarttime, util.IntAll(tmp["project_duration"]), util.ObjToString(tmp["project_timeunit"]))
+			return yucestarttime, yuceendtime
+		}
+	}
+	return 0, 0
+}
+
+func fcResult() {
+	util.Debug("预测结果时间-------结果表迁移数据----------")
+	currenttime := time.Now().Unix()
+	endtime := time.Unix(currenttime, 0).AddDate(0, 3, 0).Unix()
+	q := bson.M{"yuceendtime": bson.M{"$gte": currenttime, "$lt": endtime}, "move": nil}
+	field := bson.M{"project_duration": 0, "project_timeunit": 0, "projectperiod": 0, "s_subscopeclass": 0}
+	util.Debug(q)
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+	result := sess.DB("mixdata").C("project_forecast_yece_tmp").Find(q).Select(field).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); result.Next(&tmp); count++ {
+		if count%200 == 0 {
+			util.Debug("count---", count)
+		}
+		update := []map[string]interface{}{{
+			"_id": tmp["_id"],
+		},
+			{"$set": bson.M{"move": true}},
+		}
+		updatePool <- update
+
+		id := mongodb.BsonIdToSId(tmp["_id"])
+		tmp["infoid"] = id
+		delete(tmp, "_id")
+		tmp["yucetime"] = currenttime
+		tmp["jyhref"] = `/jyapp/article/content/` + util.CommonEncodeArticle("content", id) + `.html`
+		if tmp["buyer"] == nil || tmp["buyerperson"] == nil || tmp["buyertel"] == nil {
+			esq := `{"query":{"bool":{"must":[{"term":{"ids":"` + id + `"}}]}}}`
+			info := Es.Get("project", "project", esq)
+			if len(*info) > 0 {
+				if (*info)[0]["buyer"] != nil {
+					tmp["buyer"] = (*info)[0]["buyer"]
+				}
+				if (*info)[0]["buyerperson"] != nil {
+					tmp["buyerperson"] = (*info)[0]["buyerperson"]
+				}
+				if (*info)[0]["buyertel"] != nil {
+					tmp["buyertel"] = (*info)[0]["buyertel"]
+				}
+			}
+		}
+
+		tpmp := make(map[string]interface{})
+		tpmp["p_rate"] = "60%"
+		if tmp["purchasing"] != nil {
+			tpmp["purchasing"] = tmp["purchasing"]
+			tpmp["purchasing"] = util.ObjToString(tpmp["purchasing"]) + "," + util.ObjToString(tmp["projectname"])
+		} else {
+			tpmp["purchasing"] = tmp["projectname"]
+		}
+		var arr []map[string]interface{}
+		for _, v := range strings.Split(util.ObjToString(tpmp["purchasing"]), ",") {
+			p := make(map[string]interface{})
+			p["p_purchasing"] = v
+			p["p_id"] = id
+			p["p_orther"] = tmp["projectname"]
+			if tmp["buyerperson"] != nil {
+				p["p_person"] = tmp["buyerperson"]
+			}
+			if tmp["buyertel"] != nil {
+				p["p_phone"] = tmp["buyertel"]
+			}
+			arr = append(arr, p)
+		}
+		tpmp["p_projects"] = arr
+		delete(tmp, "buyerperson")
+		delete(tmp, "buyertel")
+		tmp["results"] = append([]map[string]interface{}{}, tpmp)
+
+		savePool1 <- tmp
+		//update1 := []map[string]interface{}{{
+		//	"infoid": id,
+		//},
+		//	{"$set": tmp},
+		//}
+		//updatePool1 <- update1
+	}
+
+	util.Debug("over ---", count)
+}
+
+func YcEndTime(starttime int64, num int, unit string) int64 {
+	yuceendtime := int64(0)
+	if unit == "日历天" || unit == "天" || unit == "日" {
+		yuceendtime = starttime + int64(num*86400)
+	} else if unit == "周" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num*7).Unix()
+	} else if unit == "月" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, num, 0).Unix()
+	} else if unit == "年" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(num, 0, 0).Unix()
+	} else if unit == "工作日" {
+		n := num / 7 * 2
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num+n).Unix()
+	}
+	return yuceendtime
+}
+
+func FormatDateStr(ds string) int64 {
+	ds = strings.Replace(ds, "年", "-", -1)
+	ds = strings.Replace(ds, "月", "-", -1)
+	ds = strings.Replace(ds, "日", "", -1)
+	ds = strings.Replace(ds, "/", "-", -1)
+	ds = strings.Replace(ds, ".", "-", -1)
+	util.Debug(ds)
+
+	location, err := time.ParseInLocation(util.Date_Short_Layout, ds, time.Local)
+	if err != nil {
+		util.Debug(err)
+		return 0
+	} else {
+		return location.Unix()
+	}
+}

fullproject/src_v1/config.json

@@ -6,8 +6,8 @@
     "mongodbPoolSize": 10,
     "mongodbName": "wjh",
 	"hints":"publishtime_1",
-    "extractColl": "bidding",
-    "extractColl1": "bidding",
+    "extractColl": "extract",
+    "extractColl1": "extract",
     "projectColl": "projectset",
     "backupFlag": true,
     "siteColl": "site",

fullproject/src_v1/main.go

@@ -70,7 +70,7 @@ func DealSign() {
 	}
 }
 
-func main() {
+func mainT() {
 	//udp跑增量  id段   project
 	//udp跑全量			qlT
 	//udp跑历史数据  信息id1,id2/或id段  ls
@@ -90,9 +90,9 @@ func main() {
 }
 
 //测试组人员使用
-func mainT() {
-	sid = "6062826adeed5af79ca930c9"
-	eid = "6062826adeed5af79ca930ca"
+func main() {
+	sid = "60fa9ef71a75b8f446125761"
+	eid = "6193756145a326c6c3ff2833"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()

fullproject/src_v1/project.go

@@ -99,10 +99,11 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 	code := strings.ReplaceAll(qu.ObjToString(tmp["spidercode"]), " ", "")
 	p.mapSpiderLock.Lock()
 	isflow := p.mapSpider[code]
+	qu.Debug(code, isflow)
 	p.mapSpiderLock.Unlock()
 	if isflow == 0 {
 		p.NewProject(tmp, info)
-		//qu.Debug("直接新建项目，", "project id", id)
+		qu.Debug("直接新建项目，", "project id")
 		return
 	}
 
@@ -135,7 +136,6 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 		//问题出地LastTime!!!!!
 		diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime)))
 		if diffTime <= p.validTime {
-
 			//代理机构完全不相同，直接新建项目
 			if CheckContain(compareProject.Agency, info.Agency) == 3 {
 				continue
@@ -241,6 +241,7 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 		//	qu.Debug("舍弃数据---", info.Id)
 		//	return
 		//}
+		qu.Debug("直接新建项目----")
 		id, p1 := p.NewProject(tmp, info)
 		p.AllIdsMapLock.Lock()
 		p.AllIdsMap[id] = &ID{Id: id, P: p1}
@@ -499,8 +500,8 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	set["ids"] = []string{thisinfo.Id}
 	if thisinfo.TopType == "招标" {
 		//if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
-			set["zbtime"] = tmp["publishtime"]
-			p1.Zbtime = qu.Int64All(tmp["publishtime"])
+		set["zbtime"] = tmp["publishtime"]
+		p1.Zbtime = qu.Int64All(tmp["publishtime"])
 		//}
 	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
 		set["jgtime"] = tmp["publishtime"]
@@ -526,7 +527,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	}
 	//标的物
 	if thisinfo.Purchasing != "" {
-		list := Duplicate(strings.Split(thisinfo.Purchasing, ","))		//标的物 去重 03/03
+		list := Duplicate(strings.Split(thisinfo.Purchasing, ",")) //标的物 去重 03/03
 		p := strings.Join(qu.ObjArrToStringArr(list), ",")
 		set["purchasing"] = p
 		p1.Purchasing = p
@@ -585,7 +586,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	// 资质条件
 	if len(thisinfo.Qualifies) > 0 {
 		var str []string
-		for _, v := range thisinfo.Qualifies{
+		for _, v := range thisinfo.Qualifies {
 			if len(qu.ObjToString(v["key"])) > 0 {
 				if BinarySearch(str, qu.ObjToString(v["key"])) == -1 {
 					str = append(str, qu.ObjToString(v["key"]))
@@ -872,7 +873,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	if thisinfo.ContractCode != "" {
 		if pInfo.ContractCode == "" {
 			set["contractcode"] = thisinfo.ContractCode
-		}else {
+		} else {
 			list := strings.Split(pInfo.ContractCode, ",")
 			if BinarySearch(list, thisinfo.ContractCode) == -1 {
 				list = append(list, thisinfo.ContractCode)
@@ -906,7 +907,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	}
 	if thisinfo.Purchasing != "" {
 		if pInfo.Purchasing == "" {
-			list := Duplicate(strings.Split(thisinfo.Purchasing, ","))		//标的物 去重 03/03
+			list := Duplicate(strings.Split(thisinfo.Purchasing, ",")) //标的物 去重 03/03
 			p := strings.Join(qu.ObjArrToStringArr(list), ",")
 			pInfo.Purchasing = p
 			set["purchasing"] = p
@@ -967,6 +968,16 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		}
 		set["s_winner"] = strings.Join(pInfo.Winners, ",")
 	}
+
+	if len(thisinfo.EntIdList) > 0 {
+		for _, v := range thisinfo.EntIdList {
+			if BinarySearch(pInfo.EntIdList, v) == -1 {
+				pInfo.EntIdList = append(pInfo.EntIdList, v)
+			}
+		}
+		set["entidlist"] = pInfo.EntIdList
+	}
+
 	//项目规模
 	if len(thisinfo.ProjectScale) > 0 {
 		pInfo.ProjectScale = thisinfo.ProjectScale
@@ -985,11 +996,11 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	//开工日期
 	if thisinfo.ProjectStartDate > 0 {
 		if pInfo.ProjectStartDate > 0 {
-			if pInfo.ProjectStartDate < thisinfo.ProjectStartDate  {
+			if pInfo.ProjectStartDate < thisinfo.ProjectStartDate {
 				pInfo.ProjectStartDate = thisinfo.ProjectStartDate
 				set["project_startdate"] = thisinfo.ProjectStartDate
 			}
-		}else {
+		} else {
 			pInfo.ProjectStartDate = thisinfo.ProjectStartDate
 			set["project_startdate"] = thisinfo.ProjectStartDate
 		}
@@ -1001,7 +1012,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 				pInfo.ProjctCompleteDate = thisinfo.ProjectCompleteDate
 				set["project_completedate"] = thisinfo.ProjectCompleteDate
 			}
-		}else {
+		} else {
 			pInfo.ProjctCompleteDate = thisinfo.ProjectCompleteDate
 			set["project_completedate"] = thisinfo.ProjectCompleteDate
 		}
@@ -1027,7 +1038,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		if len(pInfo.Qualifies) > 0 {
 			str = append(str, strings.Split(pInfo.Qualifies, ",")...)
 		}
-		for _, v := range thisinfo.Qualifies{
+		for _, v := range thisinfo.Qualifies {
 			if len(qu.ObjToString(v["key"])) > 0 {
 				if BinarySearch(str, qu.ObjToString(v["key"])) == -1 {
 					str = append(str, qu.ObjToString(v["key"]))
@@ -1039,14 +1050,6 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 			set["qualifies"] = strings.Join(str, ",")
 		}
 	}
-	if len(thisinfo.EntIdList) > 0 {
-		for _, v := range thisinfo.EntIdList{
-			if BinarySearch(pInfo.EntIdList, v) == -1 {
-				pInfo.EntIdList = append(pInfo.EntIdList, v)
-			}
-		}
-		set["entidlist"] = pInfo.EntIdList
-	}
 
 	if thisinfo.HasPackage { //多包处理
 		set["multipackage"] = 1
@@ -1418,9 +1421,9 @@ func IsCreatePro(info *Info) (bol bool) {
 		info.SubType == "结果变更" || info.SubType == "其它" {
 		if info.ProjectName == "" && info.ProjectCode == "" {
 			bol = false
-		}else if info.ProjectName == "" && info.Buyer == "" {
+		} else if info.ProjectName == "" && info.Buyer == "" {
 			bol = false
-		}else if info.ProjectCode == "" && info.Buyer == "" {
+		} else if info.ProjectCode == "" && info.Buyer == "" {
 			bol = false
 		}
 	}

qyxy/qyxy_change/main.go

@@ -25,8 +25,8 @@ func init() {
 		MongodbAddr: util.ObjToString(Sysconfig["dbServer"]),
 		Size:        util.IntAll(Sysconfig["dbSize"]),
 		DbName:      dbname,
-		UserName: 	 util.ObjToString(Sysconfig["uname"]),
-		Password: 	 util.ObjToString(Sysconfig["upwd"]),
+		UserName:    util.ObjToString(Sysconfig["uname"]),
+		Password:    util.ObjToString(Sysconfig["upwd"]),
 	}
 	MgoMix.InitPool()
 	Mgo = &mongodb.MongodbSim{
@@ -65,8 +65,9 @@ func main() {
 	//go updateMethod()
 
 	//go TimeTask()
-	//go GetData()
-	go TaskAll()
+	go GetData()
+	//go TaskAll()
+
 	ch := make(chan bool, 1)
 	<-ch
 }

qyxy/qyxy_change/task.go

@@ -21,7 +21,7 @@ var updateSp = make(chan bool, 5)
 func TimeTask() {
 	c := cron.New()
 	//cronstr := "0 0 " + fmt.Sprint(TaskTime) + " * * ?"			//每天TaskTime跑一次
-	cronstrPa := "0 0 15 ? * WED"								//凭安增量数据每周二跑一次
+	cronstrPa := "0 0 15 ? * WED" //凭安增量数据每周二跑一次
 	_ = c.AddFunc(cronstrPa, func() { GetData() })
 	c.Start()
 }
@@ -35,14 +35,15 @@ func GetData() {
 	wg := &sync.WaitGroup{}
 
 	q := bson.M{"_id": bson.M{"$gt": lastId}}
-	//q := bson.M{"_id": 262454280}
+	//q := bson.M{"_id": 368454367}
+	var zid int
 	it := sess.DB("mixdata").C("company_change").Find(q).Select(nil).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
-		lastId = util.Int64All(tmp["_id"])
 		if count%20000 == 0 {
 			log.Println("current:", count)
 		}
+		zid = util.IntAll(tmp["_id"])
 		pool <- true
 		wg.Add(1)
 		go func(tmp map[string]interface{}) {
@@ -60,17 +61,17 @@ func GetData() {
 				item["content_before"] = tmp["content_before"]
 				item["content_after"] = tmp["content_after"]
 				item["change_date"] = tmp["change_date"]
-				setMark(item)					//change_name_new
+				setMark(item) //change_name_new
 				//update["changes"] = changes
 				update["update_time"] = currentTime
 				saveInfo := []map[string]interface{}{
-					{"company_id": tmp["company_id"]},
-					{"$set": update, "$push": item},
+					{"_id": (*info)["_id"]},
+					{"$set": update, "$push": map[string]interface{}{"changes": item}},
 				}
 				updatePool <- saveInfo
-			}else {
-				query := bson.M{"company_id": tmp["company_id"]}
-				qyxy, b1 := MgoMix.FindOneByField("qyxy_std", query, bson.M{"company_name": 1})
+			} else {
+				query := bson.M{"_id": tmp["company_id"]}
+				qyxy, b1 := MgoMix.FindOne("qyxy_std", query)
 				if b1 && len(*qyxy) > 0 {
 					save := make(map[string]interface{})
 					var changes []map[string]interface{}
@@ -79,16 +80,17 @@ func GetData() {
 					item["content_before"] = tmp["content_before"]
 					item["content_after"] = tmp["content_after"]
 					item["change_date"] = tmp["change_date"]
-					setMark(item)					//change_name_new
+					setMark(item) //change_name_new
 					changes = append(changes, item)
 
 					save["company_name"] = (*qyxy)["company_name"]
+					save["company_id"] = (*qyxy)["_id"]
 					save["changes"] = changes
 					save["create_time"] = currentTime
 					save["update_time"] = currentTime
 					save["_id"] = primitive.NewObjectID()
 					saveInfo := []map[string]interface{}{
-						{"company_id": tmp["company_id"]},
+						{"_id": save["_id"]},
 						{"$set": save},
 					}
 					updatePool <- saveInfo
@@ -96,7 +98,7 @@ func GetData() {
 			}
 		}(tmp)
 	}
-	util.Debug("over---", count)
+	util.Debug("over---", count, zid)
 }
 
 func setMark(tmp map[string]interface{}) {
@@ -116,17 +118,17 @@ func setMark(tmp map[string]interface{}) {
 func clearRepeat(list []interface{}) []interface{} {
 	var tmp []interface{}
 	if len(list) > 1 {
-		for k, v := range list{
+		for k, v := range list {
 			if k < len(list)-1 {
 				if fmt.Sprint(list[k]) != fmt.Sprint(list[k+1]) {
 					tmp = append(tmp, v)
 				}
-			}else {
+			} else {
 				tmp = append(tmp, v)
 			}
 		}
 		return tmp
-	}else {
+	} else {
 		return list
 	}
 }
@@ -170,13 +172,13 @@ func TaskAll() {
 func disposeFuc(maps []map[string]interface{}, tmp map[string]interface{}) {
 	var changes []map[string]interface{}
 	currentTime := time.Now().Unix()
-	for _, v := range maps{
+	for _, v := range maps {
 		item := make(map[string]interface{})
 		item["change_field"] = v["change_field"]
 		item["content_before"] = v["content_before"]
 		item["content_after"] = v["content_after"]
 		item["change_date"] = v["change_date"]
-		setMark(item)					//change_name_new
+		setMark(item) //change_name_new
 		changes = append(changes, item)
 	}
 	tmp["changes"] = changes
@@ -222,7 +224,6 @@ func SaveData() {
 	}
 }
 
-
 func updateMethod() {
 	arru := make([][]map[string]interface{}, 200)
 	indexu := 0
@@ -257,6 +258,3 @@ func updateMethod() {
 		}
 	}
 }
-
-
-

qyxy/qyxy_es/config.json

@@ -1,21 +1,22 @@
 {
-  "mgodb": "172.17.4.187:27082,172.17.145.163:27083",
+  "mgodb": "192.168.3.207:27092",
   "dbsize": 12,
-  "dbname": "mixdata",
-  "dbcoll": "qyxy_std",
+  "dbname": "wjh",
+  "dbcoll": "qyxy_oprd",
   "uname": "SJZY_RWMIX_Other",
   "upwd": "SJZY@M34I6x7D9ata",
   "tasktime": 0,
-  "updatetime": 1631771072,
+  "updatetime": 0,
   "elastic": {
-    "addr": "http://172.17.145.170:9800",
-    "index": "qyxy_v1",
+    "addr": "http://192.168.3.206:9800",
+    "index": "oprd_qyxy_v1",
     "itype": "qyxy",
     "pool": 12,
     "esfields": [
       "_id",
       "company_name",
       "history_name",
+      "credit_no",
       "org_code",
       "tax_code",
       "company_code",

qyxy/qyxy_es/main.go

@@ -7,16 +7,16 @@ import (
 )
 
 var (
-	Sysconfig        map[string]interface{} //配置文件
-	Mgo              *mongodb.MongodbSim
-	Dbname           string
-	Dbcoll           string
-	Es               *es.Elastic
-	Index            string
-	Itype            string
-	EsFields         []string
-	TaskTime         int
-	Updatetime       int64
+	Sysconfig  map[string]interface{} //配置文件
+	Mgo        *mongodb.MongodbSim
+	Dbname     string
+	Dbcoll     string
+	Es         *es.Elastic
+	Index      string
+	Itype      string
+	EsFields   []string
+	TaskTime   int
+	Updatetime int64
 )
 var EsSaveCache = make(chan map[string]interface{}, 5000)
 var SP = make(chan bool, 5)
@@ -29,8 +29,8 @@ func init() {
 		MongodbAddr: Sysconfig["mgodb"].(string),
 		Size:        qu.IntAllDef(Sysconfig["dbsize"], 5),
 		DbName:      Dbname,
-		UserName: 	 Sysconfig["uname"].(string),
-		Password: 	 Sysconfig["upwd"].(string),
+		UserName:    Sysconfig["uname"].(string),
+		Password:    Sysconfig["upwd"].(string),
 	}
 	Mgo.InitPool()
 	//es
@@ -52,8 +52,8 @@ func main() {
 	go SaveEs()
 
 	//go TimeTask()
-	//go StdAll()
-	go StdAdd()
+	go StdAll()
+	//go StdAdd()
 	ch := make(chan bool, 1)
 	<-ch
 }

qyxy/qyxy_es/task.go

@@ -13,20 +13,20 @@ import (
 )
 
 var (
-	partner = []string{"identify_no", "stock_type", "stock_name", "identify_type", "stock_capital", "stock_realcapital"}
+	partner  = []string{"identify_no", "stock_type", "stock_name", "identify_type", "stock_capital", "stock_realcapital"}
 	employee = []string{"employee_name", "position"}
 
 	TypeMap = map[string]string{
 		"采购单位": "1",
 		"投标企业": "2",
 		"代理机构": "3",
-		"厂商": "4",
+		"厂商":   "4",
 	}
 	TypeMap1 = map[string]string{
 		"固定电话": "1",
-		"手机号": "2",
-		"邮箱": "3",
-		"不存在": "4",
+		"手机号":  "2",
+		"邮箱":   "3",
+		"不存在":  "4",
 	}
 )
 
@@ -38,7 +38,7 @@ func TimeTask() {
 	err := c.AddFunc(cronstr, func() { StdAdd() })
 	if err != nil {
 		util.Debug(err)
-		return 
+		return
 	}
 	c.Start()
 }
@@ -53,6 +53,7 @@ func StdAdd() {
 	wg := &sync.WaitGroup{}
 	//q := bson.M{"_id": "affe29f8d061f3faa4170cafba41f316"}
 	q := bson.M{"updatetime": bson.M{"$gt": Updatetime}}
+	util.Debug(q)
 	it := sess.DB(Dbname).C(Dbcoll).Find(q).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
@@ -78,57 +79,56 @@ func StdAdd() {
 				if field == "company_name" {
 					esMap[field] = tmp["company_name"]
 					esMap["name"] = tmp["company_name"]
-				}else if field == "history_name" {
+				} else if field == "history_name" {
 					var nameArr []string
-					for _, v := range strings.Split(util.ObjToString(esMap["history_name"]), "；") {
-						if v != "" {
-							nameArr = append(nameArr, v)
-						}
+					names := util.ObjToString(tmp["history_name"])
+					if strings.Contains(names, ",") {
+						nameArr = append(nameArr, strings.Split(names, ",")...)
 					}
 					if len(nameArr) > 0 {
 						esMap["history_name"] = nameArr
 					}
-				}else if field == "establish_date" {
+				} else if field == "establish_date" {
 					// 成立日期修改成时间戳
 					location, err := time.ParseInLocation(util.Date_Short_Layout, util.ObjToString(tmp["establish_date"]), time.Local)
 					if err != nil {
 						util.Debug(err)
-					}else {
+					} else {
 						esMap["establish_date"] = location.Unix()
 					}
-				}else if field == "lastupdatetime" {
+				} else if field == "lastupdatetime" {
 					esMap["lastupdatetime"] = tmp["update_time_msql"]
 				} else if field == "bid_projectname" {
 					if pname, ok := tmp["bid_projectname"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(pname)
 						esMap["bid_projectname"] = strings.Join(p1, ",")
 					}
-				}else if field == "bid_purchasing" {
+				} else if field == "bid_purchasing" {
 					if pur, ok := tmp["bid_purchasing"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(pur)
 						esMap["bid_purchasing"] = strings.Join(p1, ",")
 					}
-				}else if field == "bid_area" {
+				} else if field == "bid_area" {
 					if areas, ok := tmp["bid_area"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(areas)
 						esMap["bid_area"] = strings.Join(p1, ",")
 					}
-				}else if field == "partners" {
+				} else if field == "partners" {
 					if ps, ok := tmp["partners"].([]interface{}); ok {
 						var parr []map[string]interface{}
-						for _, v := range ps{
+						for _, v := range ps {
 							p := make(map[string]interface{})
 							v1 := v.(map[string]interface{})
-							for _, field := range partner{
+							for _, field := range partner {
 								if v1[field] == nil {
 									continue
 								}
 								if field == "stock_capital" || field == "stock_realcapital" {
 									if v, err := strconv.ParseFloat(util.ObjToString(v1[field]), 64); err == nil {
-										v, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", v), 64)	//保留小数点两位
+										v, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", v), 64) //保留小数点两位
 										p[field] = v
 									}
-								}else {
+								} else {
 									p[field] = v1[field]
 								}
 							}
@@ -140,16 +140,16 @@ func StdAdd() {
 							esMap[field] = parr
 						}
 					}
-				}else if field == "employees" {
+				} else if field == "employees" {
 					if ps, ok := tmp["employees"].([]interface{}); ok {
 						var parr []map[string]interface{}
-						for _, v := range ps{
+						for _, v := range ps {
 							p := make(map[string]interface{})
 							v1 := v.(map[string]interface{})
-							for _, field := range employee{
+							for _, field := range employee {
 								if v1[field] == nil {
 									continue
-								}else {
+								} else {
 									p[field] = v1[field]
 								}
 							}
@@ -161,23 +161,23 @@ func StdAdd() {
 							esMap[field] = parr
 						}
 					}
-				}else if field == "bid_unittype" {
+				} else if field == "bid_unittype" {
 					if t2, ok := tmp["bid_unittype"].([]interface{}); ok {
 						var arr []string
-						for _, v := range util.ObjArrToStringArr(t2){
+						for _, v := range util.ObjArrToStringArr(t2) {
 							arr = append(arr, TypeMap[v])
 						}
 						esMap["bid_unittype"] = strings.Join(arr, ",")
 					}
-				}else if field == "bid_contracttype" {
+				} else if field == "bid_contracttype" {
 					if t2, ok := tmp["bid_contracttype"].([]interface{}); ok {
 						var arr []string
-						for _, v := range util.ObjArrToStringArr(t2){
+						for _, v := range util.ObjArrToStringArr(t2) {
 							arr = append(arr, TypeMap1[v])
 						}
 						esMap["bid_contracttype"] = strings.Join(arr, ",")
 					}
-				}else {
+				} else {
 					esMap[field] = tmp[field]
 				}
 			}
@@ -186,25 +186,25 @@ func StdAdd() {
 			if company_type == "个体工商户" {
 				if len([]rune(company_name)) >= 5 {
 					esMap["company_type_int"] = 31
-				}else {
+				} else {
 					esMap["company_type_int"] = 32
 				}
-			}else if company_type == "其他" || company_type == "" {
+			} else if company_type == "其他" || company_type == "" {
 				if len([]rune(company_name)) >= 4 {
 					esMap["company_type_int"] = 21
-				}else {
+				} else {
 					esMap["company_type_int"] = 22
 				}
-			}else {
+			} else {
 				if company_type == "内资分公司" {
 					esMap["company_type_int"] = 12
-				}else if len([]rune(company_name)) >= 4 {
+				} else if len([]rune(company_name)) >= 4 {
 					esMap["company_type_int"] = 11
-				}else {
+				} else {
 					esMap["company_type_int"] = 13
 				}
 			}
-			EsSaveCache <- esMap		// 保存es
+			EsSaveCache <- esMap // 保存es
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
@@ -243,9 +243,9 @@ func StdAll() {
 				if field == "company_name" {
 					esMap[field] = tmp["company_name"]
 					esMap["name"] = tmp["company_name"]
-				}else if field == "history_name" {
+				} else if field == "history_name" {
 					var nameArr []string
-					for _, v := range strings.Split(util.ObjToString(esMap["history_name"]), "；") {
+					for _, v := range strings.Split(util.ObjToString(tmp["history_name"]), "；") {
 						if v != "" {
 							nameArr = append(nameArr, v)
 						}
@@ -253,47 +253,51 @@ func StdAll() {
 					if len(nameArr) > 0 {
 						esMap["history_name"] = nameArr
 					}
-				}else if field == "establish_date" {
+				} else if field == "establish_date" {
 					// 成立日期修改成时间戳
 					location, err := time.ParseInLocation(util.Date_Short_Layout, util.ObjToString(tmp["establish_date"]), time.Local)
 					if err != nil {
 						util.Debug(err)
-					}else {
+					} else {
 						esMap["establish_date"] = location.Unix()
 					}
-				}else if field == "lastupdatetime" {
+				} else if field == "lastupdatetime" {
 					esMap["lastupdatetime"] = tmp["update_time_msql"]
 				} else if field == "bid_projectname" {
 					if pname, ok := tmp["bid_projectname"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(pname)
 						esMap["bid_projectname"] = strings.Join(p1, ",")
 					}
-				}else if field == "bid_purchasing" {
+				} else if field == "bid_purchasing" {
 					if pur, ok := tmp["bid_purchasing"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(pur)
 						esMap["bid_purchasing"] = strings.Join(p1, ",")
 					}
-				}else if field == "bid_area" {
+				} else if field == "bid_area" {
 					if areas, ok := tmp["bid_area"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(areas)
 						esMap["bid_area"] = strings.Join(p1, ",")
 					}
-				}else if field == "partners" {
+				} else if field == "partners" {
 					if ps, ok := tmp["partners"].([]interface{}); ok {
 						var parr []map[string]interface{}
-						for _, v := range ps{
+						for _, v := range ps {
 							p := make(map[string]interface{})
 							v1 := v.(map[string]interface{})
-							for _, field := range partner{
+							for _, field := range partner {
 								if v1[field] == nil {
 									continue
 								}
 								if field == "stock_capital" || field == "stock_realcapital" {
-									if v, err := strconv.ParseFloat(util.ObjToString(v1[field]), 64); err == nil {
-										v, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", v), 64)	//保留小数点两位
+									text := util.ObjToString(v1[field])
+									if strings.Contains(text, "万元") {
+										text = strings.Replace(text, "万元", "", -1)
+									}
+									if v, err := strconv.ParseFloat(text, 64); err == nil {
+										v, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", v), 64) //保留小数点两位
 										p[field] = v
 									}
-								}else {
+								} else {
 									p[field] = v1[field]
 								}
 							}
@@ -305,16 +309,16 @@ func StdAll() {
 							esMap[field] = parr
 						}
 					}
-				}else if field == "employees" {
+				} else if field == "employees" {
 					if ps, ok := tmp["employees"].([]interface{}); ok {
 						var parr []map[string]interface{}
-						for _, v := range ps{
+						for _, v := range ps {
 							p := make(map[string]interface{})
 							v1 := v.(map[string]interface{})
-							for _, field := range employee{
+							for _, field := range employee {
 								if v1[field] == nil {
 									continue
-								}else {
+								} else {
 									p[field] = v1[field]
 								}
 							}
@@ -326,23 +330,23 @@ func StdAll() {
 							esMap[field] = parr
 						}
 					}
-				}else if field == "bid_unittype" {
+				} else if field == "bid_unittype" {
 					if t2, ok := tmp["bid_unittype"].([]interface{}); ok {
 						var arr []string
-						for _, v := range util.ObjArrToStringArr(t2){
+						for _, v := range util.ObjArrToStringArr(t2) {
 							arr = append(arr, TypeMap[v])
 						}
 						esMap["bid_unittype"] = strings.Join(arr, ",")
 					}
-				}else if field == "bid_contracttype" {
+				} else if field == "bid_contracttype" {
 					if t2, ok := tmp["bid_contracttype"].([]interface{}); ok {
 						var arr []string
-						for _, v := range util.ObjArrToStringArr(t2){
+						for _, v := range util.ObjArrToStringArr(t2) {
 							arr = append(arr, TypeMap1[v])
 						}
 						esMap["bid_contracttype"] = strings.Join(arr, ",")
 					}
-				}else {
+				} else {
 					esMap[field] = tmp[field]
 				}
 			}
@@ -351,25 +355,25 @@ func StdAll() {
 			if company_type == "个体工商户" {
 				if len([]rune(company_name)) >= 5 {
 					esMap["company_type_int"] = 31
-				}else {
+				} else {
 					esMap["company_type_int"] = 32
 				}
-			}else if company_type == "其他" || company_type == "" {
+			} else if company_type == "其他" || company_type == "" {
 				if len([]rune(company_name)) >= 4 {
 					esMap["company_type_int"] = 21
-				}else {
+				} else {
 					esMap["company_type_int"] = 22
 				}
-			}else {
+			} else {
 				if company_type == "内资分公司" {
 					esMap["company_type_int"] = 12
-				}else if len([]rune(company_name)) >= 4 {
+				} else if len([]rune(company_name)) >= 4 {
 					esMap["company_type_int"] = 11
-				}else {
+				} else {
 					esMap["company_type_int"] = 13
 				}
 			}
-			EsSaveCache <- esMap		// 保存es
+			EsSaveCache <- esMap // 保存es
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
@@ -412,4 +416,4 @@ func SaveEs() {
 			}
 		}
 	}
-}
+}

qyxy/qyxy_inc_tmp/main.go

@@ -13,29 +13,30 @@ import (
 	"mongodb"
 	"os"
 	"qfw/util"
+	"strings"
 	"time"
 )
 
 var (
-	MongoTool			*mongodb.MongodbSim
+	MongoTool *mongodb.MongodbSim
 
-	updatePool			chan []map[string]interface{}
-	updateSp			chan bool
-	savePool			chan map[string]interface{}
-	saveSp				chan bool
-	saveSize			int
-
-	CollArr				[]string
-	CurrentColl			string
+	updatePool chan []map[string]interface{}
+	updateSp   chan bool
+	savePool   chan map[string]interface{}
+	saveSp     chan bool
+	saveSize   int
 
+	saveArr [][]map[string]interface{}
 
+	CollArr     []string
+	CurrentColl string
 )
 
 func init() {
 	MongoTool = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.4.181:27001",
+		MongodbAddr: "192.168.3.207:29098",
 		Size:        10,
-		DbName:      "mixdata",
+		DbName:      "qfw",
 	}
 	MongoTool.InitPool()
 
@@ -49,7 +50,7 @@ func init() {
 
 }
 
-func main() {
+func main1() {
 
 	var path string
 	flag.StringVar(&path, "p", "", "路径 /nas/qyxy/ftp/ftpuser/upload/20210811/")
@@ -68,12 +69,68 @@ func main() {
 	<-c
 }
 
+func main() {
+	go updateMethod()
+
+	var path string
+	flag.StringVar(&path, "p", "", "路径 /")
+	flag.Parse()
+	if path == "" {
+		flag.PrintDefaults()
+		log.Fatal("参数错误.")
+	}
+	subFiles, _ := ioutil.ReadDir(path)
+	for _, s := range subFiles {
+		count := 0
+		CurrentColl = strings.Split(s.Name(), ".")[0]
+		file := path + s.Name()
+		util.Debug("current date：  ", file)
+		// 打开本地gz格式压缩包
+		fr, err := os.Open(file)
+		if err != nil {
+			panic(err)
+		} else {
+			println("open file success!")
+		}
+		// defer: 在函数退出时,执行关闭文件
+		defer fr.Close()
+		// 创建gzip文件读取对象
+		gr, err := gzip.NewReader(fr)
+		if err != nil {
+			panic(err)
+		}
+		// defer: 在函数退出时,执行关闭gzip对象
+		defer gr.Close()
+		bfRd := bufio.NewReader(gr)
+		for {
+			line, err := bfRd.ReadBytes('\n')
+			count = hookfn(line, count)
+			if err != nil {
+				if err == io.EOF {
+					fmt.Println("read gzip data finish! ")
+					if len(saveArr) > 0 {
+						tmps := saveArr
+						MongoTool.UpSertBulk(CurrentColl, tmps...)
+						saveArr = [][]map[string]interface{}{}
+					}
+					break
+				} else {
+					fmt.Println("[read gzip data err]: ", err)
+				}
+			}
+			if count%5000 == 0 {
+				util.Debug("current exc---", file, count)
+			}
+		}
+	}
+}
+
 func task(path string) {
 	for _, c := range CollArr {
 		CurrentColl = c
 		subPath := path + c + "/"
 		subFiles, _ := ioutil.ReadDir(subPath)
-		for _, s := range subFiles{
+		for _, s := range subFiles {
 			if s.IsDir() {
 				taskinfo(subPath + s.Name())
 			}
@@ -105,29 +162,28 @@ func taskinfo(path string) {
 	bfRd := bufio.NewReader(gr)
 	for {
 		line, err := bfRd.ReadBytes('\n')
-		count  = hookfn(line, count)
+		count = hookfn(line, count)
 		if err != nil {
 			if err == io.EOF {
 				fmt.Println("read gzip data finish! ")
 				break
-			}else {
+			} else {
 				fmt.Println("[read gzip data err]: ", err)
 			}
 		}
-		if count % 5000 == 0 {
+		if count%5000 == 0 {
 			util.Debug("current exc---", file, count)
 		}
 	}
 }
 
-
-func hookfn(line []byte, count int) int {
+func hookfn1(line []byte, count int) int {
 	tmp := make(map[string]interface{})
-	err:= json.Unmarshal(line, &tmp)
+	err := json.Unmarshal(line, &tmp)
 	if err != nil {
 		util.Debug("err---", err)
 	}
-	count ++
+	count++
 
 	save := make(map[string]interface{})
 	save["$push"] = map[string]interface{}{
@@ -143,6 +199,32 @@ func hookfn(line []byte, count int) int {
 	return count
 }
 
+func hookfn(line []byte, count int) int {
+	tmp := make(map[string]interface{})
+	err := json.Unmarshal(line, &tmp)
+	if err != nil {
+		util.Debug("err---", err)
+	}
+	count++
+
+	if len(tmp) > 0 {
+		id := tmp["_id"].(map[string]interface{})["$oid"]
+		delete(tmp, "_id")
+		saveInfo := []map[string]interface{}{
+			{"_id": mongodb.StringTOBsonId(util.ObjToString(id))},
+			tmp,
+		}
+		saveArr = append(saveArr, saveInfo)
+		if len(saveArr) > 500 {
+			tmps := saveArr
+			MongoTool.UpSertBulk(CurrentColl, tmps...)
+			saveArr = [][]map[string]interface{}{}
+		}
+	}
+
+	return count
+}
+
 func updateMethod() {
 	arru := make([][]map[string]interface{}, saveSize)
 	indexu := 0
@@ -157,7 +239,7 @@ func updateMethod() {
 					defer func() {
 						<-updateSp
 					}()
-					MongoTool.UpSertBulk("qyxy_tmp", arru...)
+					MongoTool.UpSertBulk("jy_tmp", arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, saveSize)
 				indexu = 0
@@ -169,11 +251,11 @@ func updateMethod() {
 					defer func() {
 						<-updateSp
 					}()
-					MongoTool.UpSertBulk("qyxy_tmp", arru...)
+					MongoTool.UpSertBulk("jy_tmp", arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, saveSize)
 				indexu = 0
 			}
 		}
 	}
-}
+}

qyxy/qyxy_std/config.json

@@ -1,8 +1,8 @@
 {
-  "dbServer": "172.17.4.187:27082,172.17.145.163:27083",
-  "dbName": "mixdata",
+  "dbServer": "192.168.3.207:27092",
+  "dbName": "wjh",
   "dbSize": 10,
-  "dbSave": "qyxy_std_1",
+  "dbSave": "qyxy_std",
   "uname": "SJZY_RWMIX_Other",
   "upwd": "SJZY@M34I6x7D9ata",
   "lastId": 261511180

qyxy/qyxy_std/init.go

@@ -6,28 +6,29 @@ import (
 	"log"
 	"mongodb"
 	"qfw/util"
+	"qfw/util/elastic"
 	"qfw/util/redis"
 )
 
 var (
-	Sysconfig        	map[string]interface{}
-	MongoTool			*mongodb.MongodbSim
-	MongoTool1			*mongodb.MongodbSim
-	MysqlTool			*Mysql
-	DbSave				string
-
-	updatePool			chan []map[string]interface{}
-	updateSp			chan bool
-	saveSize			int
-	savePool			chan map[string]interface{}
-	saveSp				chan bool
-
-	QyStypeMap       	map[string]string
-	CompanyStatusMap 	map[string]string
-	AddressMap       	map[string]*City
-	AddressOldMap    	map[string]*City
-
-	lastId				int64
+	Sysconfig  map[string]interface{}
+	MongoTool  *mongodb.MongodbSim
+	MongoTool1 *mongodb.MongodbSim
+	MysqlTool  *Mysql
+	DbSave     string
+
+	updatePool chan []map[string]interface{}
+	updateSp   chan bool
+	saveSize   int
+	savePool   chan map[string]interface{}
+	saveSp     chan bool
+
+	QyStypeMap       map[string]string
+	CompanyStatusMap map[string]string
+	AddressMap       map[string]*City
+	AddressOldMap    map[string]*City
+
+	Es *elastic.Elastic
 )
 
 func init() {
@@ -38,13 +39,13 @@ func init() {
 		MongodbAddr: util.ObjToString(Sysconfig["dbServer"]),
 		Size:        util.IntAll(Sysconfig["dbSize"]),
 		DbName:      util.ObjToString(Sysconfig["dbName"]),
-		UserName: 	 util.ObjToString(Sysconfig["uname"]),
-		Password: 	 util.ObjToString(Sysconfig["upwd"]),
+		UserName:    util.ObjToString(Sysconfig["uname"]),
+		Password:    util.ObjToString(Sysconfig["upwd"]),
 	}
 	MongoTool.InitPool()
 
 	MongoTool1 = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.4.181:27001",
+		MongodbAddr: "172.17.4.181:27001", // 172.17.4.181:27001
 		Size:        10,
 		DbName:      "mixdata",
 	}
@@ -58,14 +59,18 @@ func init() {
 	//}
 	//MysqlTool.Init()
 
+	Es = &elastic.Elastic{
+		S_esurl: "http://172.17.145.170:9800", //http://172.17.145.170:9800
+		I_size:  10,
+	}
+	Es.InitElasticSize()
+
 	saveSize = 200
 	updatePool = make(chan []map[string]interface{}, 5000)
 	updateSp = make(chan bool, 5)
 	savePool = make(chan map[string]interface{}, 5000)
 	saveSp = make(chan bool, 5)
 
-	lastId = 0
-
 	InitQyStype()
 	InitCompanyStatus()
 	InitAddress()
@@ -125,7 +130,7 @@ func InitAddress() {
 	result := sess.DB("mixdata").C("address_new_2020").Find(nil).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); result.Next(&tmp); count++ {
-		if count % 50000 == 0 {
+		if count%50000 == 0 {
 			util.Debug("current---", count)
 		}
 		code := util.ObjToString(tmp["code"])
@@ -149,4 +154,4 @@ func InitAddress() {
 		}
 	}
 	util.Debug("Init Address end...", len(AddressMap), len(AddressOldMap))
-}
+}

qyxy/qyxy_std/task.go

@@ -15,11 +15,11 @@ import (
 )
 
 var (
-	AreaFiled				 = []string{"credit_no", "company_code"}
-	WordsArr 				= []string{"研发", "研制", "开发", "生产", "制造", "制作", "加工", "种植"}
+	AreaFiled = []string{"credit_no", "company_code"}
+	WordsArr  = []string{"研发", "研制", "开发", "生产", "制造", "制作", "加工", "种植"}
 
-	seg 					jiebago.Segmenter
-	regPre, _ 				= regexp.Compile(`^(.+[省|市|区|县|州])?(.+)`)
+	seg       jiebago.Segmenter
+	regPre, _ = regexp.Compile(`^(.+[省|市|区|县|州])?(.+)`)
 )
 
 //	企业基本信息
@@ -30,7 +30,7 @@ var company_base = []string{"company_name", "company_code", "credit_no", "org_co
 }
 
 var province_map = map[string]string{
-	"BJ": "北京", "TJ": "天津", "SH": "上海", "CQ": "重庆", "HB": "河北", "SX": "山西", "NMG": "内蒙", "LN": "辽宁", "JL": "吉林",
+	"BJ": "北京", "TJ": "天津", "SH": "上海", "CQ": "重庆", "HB": "河北", "SX": "山西", "NMG": "内蒙古", "LN": "辽宁", "JL": "吉林",
 	"HLJ": "黑龙江", "JS": "江苏", "ZJ": "浙江", "AH": "安徽", "FJ": "福建", "JX": "江西", "SD": "山东", "HEN": "河南", "HUB": "湖北",
 	"HUN": "湖南", "GD": "广东", "GX": "广西", "HAIN": "海南", "SC": "四川", "GZ": "贵州", "YN": "云南", "XZ": "西藏", "SAX": "陕西",
 	"GS": "甘肃", "QH": "青海", "NX": "宁夏", "XJ": "新疆",
@@ -39,7 +39,7 @@ var province_map = map[string]string{
 func TimeTask() {
 	c := cron.New()
 	//cronstr := "0 0 " + fmt.Sprint(TaskTime) + " * * ?"			//每天TaskTime跑一次
-	cronstrPa := "0 0 15 ? * WED"								//凭安增量数据每周二跑一次
+	cronstrPa := "0 0 15 ? * WED" //凭安增量数据每周二跑一次
 	_ = c.AddFunc(cronstrPa, func() {
 		TaskFun()
 	})
@@ -51,16 +51,20 @@ func TaskFun() {
 	sess := MongoTool1.GetMgoConn()
 	defer MongoTool1.DestoryMongoConn(sess)
 
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+
 	pool := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
 
 	//q := bson.M{"_id": bson.M{"$gt": lastId}}
 	//q := bson.M{"_id": 262454280}
+	//q := map[string]interface{}{"_id": "3fd1352a95f552bda9bac1248eb24f03"}
+	//q := map[string]interface{}{"company_name": nil}
 	it := sess.DB("mixdata").C("qyxy_tmp").Find(nil).Select(nil).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
-		lastId = util.Int64All(tmp["_id"])
-		if count%20000 == 0 {
+		if count%2000 == 0 {
 			util.Debug("current:", count)
 		}
 		pool <- true
@@ -71,6 +75,8 @@ func TaskFun() {
 				wg.Done()
 			}()
 			IncStd(tmp)
+			//taskinfo(tmp)
+
 		}(tmp)
 	}
 
@@ -86,7 +92,7 @@ func IncStd(tmp map[string]interface{}) {
 	if tmp["company_base"] != nil {
 		companyArr := util.ObjArrToMapArr(tmp["company_base"].([]interface{}))
 		company := companyArr[len(companyArr)-1]
-		for _, v := range company_base{
+		for _, v := range company_base {
 			if company[v] == nil {
 				continue
 			}
@@ -94,7 +100,7 @@ func IncStd(tmp map[string]interface{}) {
 			if v == "company_type" {
 				save["company_type_old"] = company[v]
 				if text := util.ObjToString(company["company_type"]); text != "" {
-					if strings.Contains(text, "个体") || strings.Contains(text, "非公司")  {
+					if strings.Contains(text, "个体") || strings.Contains(text, "非公司") {
 						save["company_type"] = "个体工商户"
 					} else {
 						text = strings.ReplaceAll(text, "（", "(")
@@ -107,7 +113,7 @@ func IncStd(tmp map[string]interface{}) {
 					}
 				}
 				// company_status
-			}else if v == "company_status" {
+			} else if v == "company_status" {
 				save["company_status_old"] = company[v]
 				if text := util.ObjToString(company["company_status"]); text != "" {
 					text = strings.ReplaceAll(text, "（", "(")
@@ -118,7 +124,7 @@ func IncStd(tmp map[string]interface{}) {
 						save["company_status"] = "其他"
 					}
 				}
-			}else if v == "capital" {
+			} else if v == "capital" {
 				// capital/currency
 				text := util.ObjToString(company[v])
 				if currency := GetCurrency(text); currency != "" {
@@ -129,9 +135,9 @@ func IncStd(tmp map[string]interface{}) {
 				if capital != 0 {
 					save[v] = capital
 				}
-			}else if v == "use_flag" {
+			} else if v == "use_flag" {
 				save[v] = util.IntAll(company[v])
-			}else {
+			} else {
 				save[v] = company[v]
 			}
 		}
@@ -152,7 +158,7 @@ func IncStd(tmp map[string]interface{}) {
 					code = code[:6]
 				}
 				if city := AddressMap[code]; city != nil { //未作废中取
-					if city.Province != "" && city.Province == save["company_area"] {
+					if city.Province != "" && city.Province == util.ObjToString(save["company_area"]) {
 						if city.City != "" {
 							save["company_city"] = city.City //市
 						}
@@ -163,7 +169,7 @@ func IncStd(tmp map[string]interface{}) {
 					}
 				} else { //作废中取
 					if city := AddressOldMap[code]; city != nil {
-						if city.Province != "" && city.Province == save["company_area"] {
+						if city.Province != "" && city.Province == util.ObjToString(save["company_area"]) {
 							if city.City != "" {
 								save["company_city"] = city.City //市
 							}
@@ -184,76 +190,191 @@ func IncStd(tmp map[string]interface{}) {
 		if tmp["company_history_name"] != nil {
 			history_name := util.ObjArrToMapArr(tmp["company_history_name"].([]interface{}))
 			var names []string
-			for _, h := range history_name{
+			for _, h := range history_name {
 				names = append(names, util.ObjToString(h["history_name"]))
 			}
-			save["history_name"] =  strings.Join(names, ",")
+			save["history_name"] = strings.Join(names, ",")
+		}
+
+		// company_employee、company_partner、annual_report_base涉及存量更新动作
+		oldTmp := &map[string]interface{}{}
+		if tmp["company_employee"] != nil || tmp["company_partner"] != nil || tmp["annual_report_base"] != nil {
+			fields := map[string]interface{}{"employees": 1, "partners": 1, "annual_reports": 1}
+			oldTmp, _ = MongoTool.FindOneByField("qyxy_std", map[string]interface{}{"_id": tmp["_id"]}, fields)
 		}
+
 		// company_employee
 		if tmp["company_employee"] != nil {
 			employees := util.ObjArrToMapArr(tmp["company_employee"].([]interface{}))
 			var names []string
 			var arr []map[string]interface{}
-			for _, v := range employees{
-				tmp := make(map[string]interface{})
-				tmp["employee_name"] = v["employee_name"]
-				tmp["position"] = v["position"]
-				tmp["is_history"] = v["is_history"]
-				arr = append(arr, tmp)
-				names = append(names, util.ObjToString(v["employee_name"]))
-			}
-			//save["employees"] = arr
-			push["employees"] = bson.M{"$each": arr}
-			save["employee_name"] =  strings.Join(names, ",")
+			if (*oldTmp)["employees"] != nil {
+				arr = util.ObjArrToMapArr((*oldTmp)["employees"].([]interface{}))
+			} else {
+				arr = make([]map[string]interface{}, 0)
+			}
+			//arr := (*oldTmp)["employees"].([]map[string]interface{})
+			for _, v := range employees {
+				ep := make(map[string]interface{})
+				if util.ObjToString(v["_operation_type"]) == "insert" {
+					ep["employee_name"] = v["employee_name"]
+					ep["position"] = v["position"]
+					ep["is_history"] = v["is_history"]
+					ep["_id"] = util.IntAll(v["id"])
+					arr = append(arr, ep)
+					names = append(names, util.ObjToString(v["employee_name"]))
+				} else {
+					eq_flag := true
+					for _, m := range arr {
+						if util.IntAll(v["id"]) == util.IntAll(m["_id"]) {
+							eq_flag = false
+							m["employee_name"] = v["employee_name"]
+							m["position"] = v["position"]
+							m["is_history"] = v["is_history"]
+							break
+						}
+					}
+					if eq_flag {
+						ep := make(map[string]interface{})
+						ep["employee_name"] = v["employee_name"]
+						ep["position"] = v["position"]
+						ep["is_history"] = v["is_history"]
+						ep["_id"] = util.IntAll(v["id"])
+						arr = append(arr, ep)
+						names = append(names, util.ObjToString(v["employee_name"]))
+					}
+				}
+
+			}
+			save["employees"] = arr
+			//push["employees"] = bson.M{"$each": arr}
+			save["employee_name"] = strings.Join(names, ",")
 		}
 		// company_partner
 		if tmp["company_partner"] != nil {
 			partners := util.ObjArrToMapArr(tmp["company_partner"].([]interface{}))
 			var names []string
 			var arr []map[string]interface{}
-			tmp["partners"] =  partners
-			for _, v := range partners{
-				if util.IntAll(tmp["is_history"]) == 0 {
-					tmp := make(map[string]interface{})
-					tmp["stock_capital"] = v["stock_capital"]
-					tmp["stock_name"] = v["stock_name"]
-					tmp["identify_no"] = v["identify_no"]
-					tmp["stock_realcapital"] = v["stock_realcapital"]
-					tmp["is_history"] = v["is_history"]
-					tmp["is_personal"] = v["is_personal"]
-					tmp["stock_type"] = v["stock_type"]
-					tmp["identify_type"] = v["identify_type"]
-					arr = append(arr, tmp)
+			if (*oldTmp)["partners"] != nil {
+				arr = util.ObjArrToMapArr((*oldTmp)["partners"].([]interface{}))
+			} else {
+				arr = make([]map[string]interface{}, 0)
+			}
+			for _, v := range partners {
+				if util.ObjToString(v["_operation_type"]) == "insert" {
+					exp := make(map[string]interface{})
+					exp["stock_capital"] = v["stock_capital"]
+					exp["stock_name"] = v["stock_name"]
+					exp["identify_no"] = v["identify_no"]
+					exp["stock_realcapital"] = v["stock_realcapital"]
+					exp["is_history"] = v["is_history"]
+					exp["is_personal"] = v["is_personal"]
+					exp["stock_type"] = v["stock_type"]
+					exp["identify_type"] = v["identify_type"]
+					exp["_id"] = util.IntAll(v["id"])
+					arr = append(arr, exp)
 					names = append(names, util.ObjToString(v["stock_name"]))
+				} else {
+					eqFlag := true
+					for _, m := range arr {
+						if util.IntAll(v["id"]) == util.IntAll(m["_id"]) {
+							eqFlag = false
+							m["stock_capital"] = v["stock_capital"]
+							m["stock_name"] = v["stock_name"]
+							m["identify_no"] = v["identify_no"]
+							m["stock_realcapital"] = v["stock_realcapital"]
+							m["is_history"] = v["is_history"]
+							m["is_personal"] = v["is_personal"]
+							m["stock_type"] = v["stock_type"]
+							m["identify_type"] = v["identify_type"]
+							break
+						}
+					}
+					if eqFlag {
+						exp := make(map[string]interface{})
+						exp["stock_capital"] = v["stock_capital"]
+						exp["stock_name"] = v["stock_name"]
+						exp["identify_no"] = v["identify_no"]
+						exp["stock_realcapital"] = v["stock_realcapital"]
+						exp["is_history"] = v["is_history"]
+						exp["is_personal"] = v["is_personal"]
+						exp["stock_type"] = v["stock_type"]
+						exp["identify_type"] = v["identify_type"]
+						exp["_id"] = util.IntAll(v["id"])
+						arr = append(arr, exp)
+						names = append(names, util.ObjToString(v["stock_name"]))
+					}
 				}
+
 			}
-			//save["partners"] =  arr
-			push["partners"] = bson.M{"$each": arr}
-			save["stock_name"] =  strings.Join(names, ",")
+			save["partners"] = arr
+			//push["partners"] = bson.M{"$each": arr}
+			save["stock_name"] = strings.Join(names, ",")
 		}
 		// annual_report_base
 		if tmp["annual_report_base"] != nil {
 			reports := util.ObjArrToMapArr(tmp["annual_report_base"].([]interface{}))
 			var arr []map[string]interface{}
+			if (*oldTmp)["annual_reports"] != nil {
+				arr = util.ObjArrToMapArr((*oldTmp)["annual_reports"].([]interface{}))
+			} else {
+				arr = make([]map[string]interface{}, 0)
+			}
 			year := 0
 			phone, email := "", ""
-			for _, v := range reports{
-				tmp := make(map[string]interface{})
-				tmp["operator_name"] = v["operator_name"]
-				tmp["report_year"] = v["report_year"]
-				tmp["zip_code"] = v["zip_code"]
-				tmp["employee_no"] = v["employee_no"]
-				tmp["company_phone"] = v["company_phone"]
-				tmp["company_email"] = v["company_email"]
-				arr = append(arr, tmp)
-				if year < util.IntAll(v["report_year"]) {
-					year = util.IntAll(v["report_year"])
-					phone = util.ObjToString(v["company_phone"])
-					email = util.ObjToString(v["company_email"])
+			for _, v := range reports {
+				if util.ObjToString(v["_operation_type"]) == "insert" {
+					exp := make(map[string]interface{})
+					exp["operator_name"] = v["operator_name"]
+					exp["report_year"] = v["report_year"]
+					exp["zip_code"] = v["zip_code"]
+					exp["employee_no"] = v["employee_no"]
+					exp["company_phone"] = v["company_phone"]
+					exp["company_email"] = v["company_email"]
+					exp["_id"] = util.IntAll(v["id"])
+					arr = append(arr, exp)
+					if year < util.IntAll(v["report_year"]) {
+						year = util.IntAll(v["report_year"])
+						phone = util.ObjToString(v["company_phone"])
+						email = util.ObjToString(v["company_email"])
+					}
+				} else {
+					eqFlag := true
+					for _, m := range arr {
+						if util.IntAll(v["id"]) == util.IntAll(m["_id"]) {
+							eqFlag = false
+							m["operator_name"] = v["operator_name"]
+							m["report_year"] = v["report_year"]
+							m["zip_code"] = v["zip_code"]
+							m["employee_no"] = v["employee_no"]
+							m["company_phone"] = v["company_phone"]
+							m["company_email"] = v["company_email"]
+							break
+						}
+					}
+					if eqFlag {
+						exp := make(map[string]interface{})
+						exp["stock_capital"] = v["stock_capital"]
+						exp["stock_name"] = v["stock_name"]
+						exp["identify_no"] = v["identify_no"]
+						exp["stock_realcapital"] = v["stock_realcapital"]
+						exp["is_history"] = v["is_history"]
+						exp["is_personal"] = v["is_personal"]
+						exp["stock_type"] = v["stock_type"]
+						exp["identify_type"] = v["identify_type"]
+						exp["_id"] = util.IntAll(v["id"])
+						arr = append(arr, exp)
+						if year < util.IntAll(v["report_year"]) {
+							year = util.IntAll(v["report_year"])
+							phone = util.ObjToString(v["company_phone"])
+							email = util.ObjToString(v["company_email"])
+						}
+					}
 				}
+
 			}
-			//save["annual_reports"] =  arr
-			push["annual_reports"] = bson.M{"$each": arr}
+			save["annual_reports"] = arr
+			//push["annual_reports"] = bson.M{"$each": arr}
 			if year != 0 {
 				save["company_phone"] = phone
 				save["company_email"] = email
@@ -264,14 +385,14 @@ func IncStd(tmp map[string]interface{}) {
 			webs := util.ObjArrToMapArr(tmp["annual_report_website"].([]interface{}))
 			year := 0
 			web := ""
-			for _, v := range webs{
+			for _, v := range webs {
 				if year < util.IntAll(v["report_year"]) && util.IntAll(v["is_history"]) == 0 {
 					year = util.IntAll(v["report_year"])
 					web = util.ObjToString(v["website_url"])
 				}
 			}
 			if year != 0 {
-				save["website_url"] =  web
+				save["website_url"] = web
 			}
 		}
 		// bid_contracttype
@@ -279,7 +400,7 @@ func IncStd(tmp map[string]interface{}) {
 		if phone := util.ObjToString(save["company_phone"]); phone != "" {
 			if len(phone) == 11 {
 				types = append(types, "手机号")
-			}else {
+			} else {
 				types = append(types, "固定电话")
 			}
 		}
@@ -295,12 +416,12 @@ func IncStd(tmp map[string]interface{}) {
 			err1 := json.Unmarshal([]byte(text), &maps)
 			if err1 != nil {
 				util.Debug(companyName, "winner-----map解析异常")
-			}else {
-				for k := range maps{
+			} else {
+				for k := range maps {
 					if k == "bid_contracttype" {
 						t1 := util.ObjArrToStringArr(maps[k].([]interface{}))
 						types = append(types, t1...)
-					}else {
+					} else {
 						save[k] = maps[k]
 					}
 				}
@@ -312,7 +433,7 @@ func IncStd(tmp map[string]interface{}) {
 		save["bid_contracttype"] = types
 		// bid_unittype
 		flag := false
-		for _, v := range WordsArr{
+		for _, v := range WordsArr {
 			if strings.Contains(util.ObjToString(save["business_scope"]), v) {
 				flag = true
 				break
@@ -328,12 +449,12 @@ func IncStd(tmp map[string]interface{}) {
 				name := util.ObjToString(save["company_name"])
 				if strings.Contains(t1, "有限合伙") {
 					save["search_type"] = "有限合伙"
-				}else if strings.Contains(t1, "合伙") {
+				} else if strings.Contains(t1, "合伙") {
 					save["search_type"] = "普通合伙"
-				}else if strings.Contains(name, "股份") ||
+				} else if strings.Contains(name, "股份") ||
 					(strings.Contains(t1, "上市") && !strings.Contains(t1, "非上市")) {
 					save["search_type"] = "股份有限公司"
-				}else {
+				} else {
 					save["search_type"] = "有限责任公司"
 				}
 			}
@@ -356,9 +477,9 @@ func IncStd(tmp map[string]interface{}) {
 }
 
 // InfoStd 存量全量数据
-func InfoStd(tmp map[string]interface{})  {
+func InfoStd(tmp map[string]interface{}) {
 	save := make(map[string]interface{})
-	for _, v := range company_base{
+	for _, v := range company_base {
 		if tmp[v] == nil {
 			continue
 		}
@@ -366,7 +487,7 @@ func InfoStd(tmp map[string]interface{})  {
 		if v == "company_type" {
 			save["company_type_old"] = tmp[v]
 			if text := util.ObjToString(tmp["company_type"]); text != "" {
-				if strings.Contains(text, "个体") || strings.Contains(text, "非公司")  {
+				if strings.Contains(text, "个体") || strings.Contains(text, "非公司") {
 					save["company_type"] = "个体工商户"
 				} else {
 					text = strings.ReplaceAll(text, "（", "(")
@@ -378,8 +499,8 @@ func InfoStd(tmp map[string]interface{})  {
 					}
 				}
 			}
-		// company_status
-		}else if v == "company_status" {
+			// company_status
+		} else if v == "company_status" {
 			save["company_status_old"] = tmp[v]
 			if text := util.ObjToString(tmp["company_status"]); text != "" {
 				text = strings.ReplaceAll(text, "（", "(")
@@ -390,7 +511,7 @@ func InfoStd(tmp map[string]interface{})  {
 					save["company_status"] = "其他"
 				}
 			}
-		}else if v == "capital" {
+		} else if v == "capital" {
 			// capital/currency
 			text := util.ObjToString(tmp[v])
 			if currency := GetCurrency(text); currency != "" {
@@ -401,9 +522,9 @@ func InfoStd(tmp map[string]interface{})  {
 			if capital != 0 {
 				save[v] = capital
 			}
-		}else if v == "use_flag" {
+		} else if v == "use_flag" {
 			save[v] = util.IntAll(tmp[v])
-		}else {
+		} else {
 			save[v] = tmp[v]
 		}
 	}
@@ -427,7 +548,7 @@ func InfoStd(tmp map[string]interface{})  {
 				code = code[:6]
 			}
 			if city := AddressMap[code]; city != nil { //未作废中取
-				if city.Province != "" && city.Province == save["company_area"] {
+				if city.Province != "" && strings.Contains(city.Province, util.ObjToString(save["company_area"])) {
 					if city.City != "" {
 						save["company_city"] = city.City //市
 					}
@@ -467,7 +588,7 @@ func InfoStd(tmp map[string]interface{})  {
 		if phone := util.ObjToString(save["company_phone"]); phone != "" {
 			if len(phone) == 11 {
 				types = append(types, "手机号")
-			}else {
+			} else {
 				types = append(types, "固定电话")
 			}
 		}
@@ -483,12 +604,12 @@ func InfoStd(tmp map[string]interface{})  {
 			err1 := json.Unmarshal([]byte(text), &maps)
 			if err1 != nil {
 				util.Debug(companyName, "winner-----map解析异常")
-			}else {
-				for k := range maps{
+			} else {
+				for k := range maps {
 					if k == "bid_contracttype" {
 						t1 := util.ObjArrToStringArr(maps[k].([]interface{}))
 						types = append(types, t1...)
-					}else {
+					} else {
 						save[k] = maps[k]
 					}
 				}
@@ -500,7 +621,7 @@ func InfoStd(tmp map[string]interface{})  {
 		save["bid_contracttype"] = types
 		// bid_unittype
 		flag := false
-		for _, v := range WordsArr{
+		for _, v := range WordsArr {
 			if strings.Contains(util.ObjToString(tmp["business_scope"]), v) {
 				flag = true
 				break
@@ -516,12 +637,12 @@ func InfoStd(tmp map[string]interface{})  {
 				name := util.ObjToString(save["company_name"])
 				if strings.Contains(t1, "有限合伙") {
 					save["search_type"] = "有限合伙"
-				}else if strings.Contains(t1, "合伙") {
+				} else if strings.Contains(t1, "合伙") {
 					save["search_type"] = "普通合伙"
-				}else if strings.Contains(name, "股份") ||
+				} else if strings.Contains(name, "股份") ||
 					(strings.Contains(t1, "上市") && !strings.Contains(t1, "非上市")) {
 					save["search_type"] = "股份有限公司"
-				}else {
+				} else {
 					save["search_type"] = "有限责任公司"
 				}
 			}
@@ -546,10 +667,10 @@ func historynamefunMysql(tmp map[string]interface{}) {
 	info := MysqlTool.SelectBySql(query, util.ObjToString(tmp["_id"]))
 	if len(*info) > 0 {
 		var names []string
-		for _, v := range *info{
+		for _, v := range *info {
 			names = append(names, util.ObjToString(v["history_name"]))
 		}
-		tmp["history_name"] =  strings.Join(names, ",")
+		tmp["history_name"] = strings.Join(names, ",")
 	}
 }
 func historyNameFun(tmp map[string]interface{}) {
@@ -557,36 +678,36 @@ func historyNameFun(tmp map[string]interface{}) {
 	info, b := MongoTool1.Find("company_history_name", bson.M{"company_id": tmp["_id"]}, nil, field, false, -1, -1)
 	if b && len(*info) > 0 {
 		var names []string
-		for _, v := range *info{
+		for _, v := range *info {
 			names = append(names, util.ObjToString(v["history_name"]))
 		}
-		tmp["history_name"] =  strings.Join(names, ",")
+		tmp["history_name"] = strings.Join(names, ",")
 	}
 }
 
 // company_employee 高管
-func employeeFunMySql(tmp map[string]interface{})  {
+func employeeFunMySql(tmp map[string]interface{}) {
 	query := "SELECT employee_name, position, is_history FROM company_employee WHERE company_id=?"
 	info := MysqlTool.SelectBySql(query, util.ObjToString(tmp["_id"]))
 	if len(*info) > 0 {
 		var names []string
-		tmp["employees"] =  *info
-		for _, v := range *info{
+		tmp["employees"] = *info
+		for _, v := range *info {
 			names = append(names, util.ObjToString(v["employee_name"]))
 		}
-		tmp["employee_name"] =  strings.Join(names, ",")
+		tmp["employee_name"] = strings.Join(names, ",")
 	}
 }
-func employeeFun(tmp map[string]interface{})  {
+func employeeFun(tmp map[string]interface{}) {
 	field := bson.M{"employee_name": 1, "position": 1, "is_history": 1}
 	info, b := MongoTool1.Find("company_employee", bson.M{"company_id": tmp["_id"]}, nil, field, false, -1, -1)
 	if b && len(*info) > 0 {
 		var names []string
-		tmp["employees"] =  *info
-		for _, v := range *info{
+		tmp["employees"] = *info
+		for _, v := range *info {
 			names = append(names, util.ObjToString(v["employee_name"]))
 		}
-		tmp["employee_name"] =  strings.Join(names, ",")
+		tmp["employee_name"] = strings.Join(names, ",")
 	}
 }
 
@@ -596,13 +717,13 @@ func partnerFunMySql(tmp map[string]interface{}) {
 	info := MysqlTool.SelectBySql(query, util.ObjToString(tmp["_id"]))
 	if len(*info) > 0 {
 		var names []string
-		tmp["partners"] =  *info
-		for _, v := range *info{
+		tmp["partners"] = *info
+		for _, v := range *info {
 			if util.IntAll(tmp["is_history"]) == 0 {
 				names = append(names, util.ObjToString(v["stock_name"]))
 			}
 		}
-		tmp["stock_name"] =  strings.Join(names, ",")
+		tmp["stock_name"] = strings.Join(names, ",")
 	}
 }
 func partnerFun(tmp map[string]interface{}) {
@@ -610,13 +731,13 @@ func partnerFun(tmp map[string]interface{}) {
 	info, b := MongoTool1.Find("company_partner", bson.M{"company_id": tmp["_id"]}, nil, field, false, -1, -1)
 	if b && len(*info) > 0 {
 		var names []string
-		tmp["partners"] =  *info
-		for _, v := range *info{
+		tmp["partners"] = *info
+		for _, v := range *info {
 			if util.IntAll(tmp["is_history"]) == 0 {
 				names = append(names, util.ObjToString(v["stock_name"]))
 			}
 		}
-		tmp["stock_name"] =  strings.Join(names, ",")
+		tmp["stock_name"] = strings.Join(names, ",")
 	}
 }
 
@@ -625,10 +746,10 @@ func reportFunMysql(tmp map[string]interface{}) {
 	query := "SELECT report_year, company_phone, company_email, zip_code, employee_no, operator_name FROM annual_report_base WHERE company_id=?"
 	info := MysqlTool.SelectBySql(query, util.ObjToString(tmp["_id"]))
 	if len(*info) > 0 {
-		tmp["annual_reports"] =  *info
+		tmp["annual_reports"] = *info
 		year := 0
 		phone, email := "", ""
-		for _, v := range *info{
+		for _, v := range *info {
 			if year < util.IntAll(v["report_year"]) {
 				year = util.IntAll(v["report_year"])
 				phone = util.ObjToString(v["company_phone"])
@@ -645,10 +766,10 @@ func reportFun(tmp map[string]interface{}) {
 	field := bson.M{"report_year": 1, "company_phone": 1, "company_email": 1, "zip_code": 1, "employee_no": 1, "operator_name": 1}
 	info, b := MongoTool1.Find("annual_report_base", bson.M{"company_id": tmp["_id"]}, nil, field, false, -1, -1)
 	if b && len(*info) > 0 {
-		tmp["annual_reports"] =  *info
+		tmp["annual_reports"] = *info
 		year := 0
 		phone, email := "", ""
-		for _, v := range *info{
+		for _, v := range *info {
 			if year < util.IntAll(v["report_year"]) {
 				year = util.IntAll(v["report_year"])
 				phone = util.ObjToString(v["company_phone"])
@@ -669,14 +790,14 @@ func websiteFunMySql(tmp map[string]interface{}) {
 	if len(*info) > 0 {
 		year := 0
 		web := ""
-		for _, v := range *info{
+		for _, v := range *info {
 			if year < util.IntAll(v["report_year"]) && util.IntAll(tmp["is_history"]) == 0 {
 				year = util.IntAll(v["report_year"])
 				web = util.ObjToString(v["website_url"])
 			}
 		}
 		if year != 0 {
-			tmp["website_url"] =  web
+			tmp["website_url"] = web
 		}
 	}
 }
@@ -686,14 +807,14 @@ func websiteFun(tmp map[string]interface{}) {
 	if b && len(*info) > 0 {
 		year := 0
 		web := ""
-		for _, v := range *info{
+		for _, v := range *info {
 			if year < util.IntAll(v["report_year"]) && util.IntAll(tmp["is_history"]) == 0 {
 				year = util.IntAll(v["report_year"])
 				web = util.ObjToString(v["website_url"])
 			}
 		}
 		if year != 0 {
-			tmp["website_url"] =  web
+			tmp["website_url"] = web
 		}
 	}
 }
@@ -722,4 +843,58 @@ func getStName(name string) string {
 	return name2
 }
 
+func taskinfo(tmp map[string]interface{}) {
+	q := map[string]interface{}{"company_id": tmp["_id"]}
+	info, _ := MongoTool1.FindOne("company_base", q)
+	if len(*info) > 0 {
+		InfoStd(*info)
+	}
+}
 
+func taskinfo1() {
+	//updataMap := make(map[string]interface{})
+	//updataMap["company_area"] = "内蒙古"
+	//for i, field := range AreaFiled {
+	//	if tmp[field] == nil {
+	//		continue
+	//	}
+	//	if code := fmt.Sprint(tmp[field]); code != "" {
+	//		if i == 0 && len(code) >= 8 { //credit_no企业信用代码
+	//			code = code[2:8]
+	//		} else if i == 1 && len(code) >= 6 { //company_code注册号
+	//			code = code[:6]
+	//		}
+	//		if city := AddressMap[code]; city != nil {
+	//			if city.Province != "" && city.Province == util.ObjToString(updataMap["company_area"]) {
+	//				if city.City != "" {
+	//					updataMap["company_city"] = city.City //市
+	//				}
+	//				if city.District != "" {
+	//					updataMap["company_district"] = city.District //县
+	//				}
+	//				break
+	//			}
+	//		} else { //作废中取
+	//			if city := AddressOldMap[code]; city != nil {
+	//				if city.Province != "" && city.Province == util.ObjToString(updataMap["company_area"]) {
+	//					if city.City != "" {
+	//						updataMap["company_city"] = city.City //市
+	//					}
+	//					if city.District != "" {
+	//						updataMap["company_district"] = city.District //县
+	//					}
+	//				}
+	//				break
+	//			}
+	//		}
+	//	}
+	//}
+	//
+	//_, _ = client.Update().Index("qyxy_v2").Type("qyxy").Id(util.ObjToString(tmp["_id"])).
+	//	Doc(updataMap).Refresh(true).Do()
+	//updataInfo := []map[string]interface{}{
+	//	{"_id": tmp["_id"]},
+	//	{"$set": updataMap},
+	//}
+	//updatePool <- updataInfo
+}

qyxy/save_es/main.go

@@ -12,10 +12,10 @@ import (
 )
 
 var (
-	MongoTool			*mongodb.MongodbSim
-	Es              	*es.Elastic
-	EsSaveCache			chan map[string]interface{}
-	SP					chan bool
+	MongoTool   *mongodb.MongodbSim
+	Es          *es.Elastic
+	EsSaveCache chan map[string]interface{}
+	SP          chan bool
 
 	EsField = []string{"_id", "company_name", "history_name", "updatetime", "legal_person", "legal_person_certno", "credit_no", "org_code", "tax_code",
 		"company_code", "area_code", "company_area", "company_city", "company_district", "currency", "capital", "company_type", "company_status",
@@ -26,15 +26,14 @@ var (
 		"采购单位": "1",
 		"投标企业": "2",
 		"代理机构": "3",
-		"厂商": "4",
+		"厂商":   "4",
 	}
 	TypeMap1 = map[string]string{
 		"固定电话": "1",
-		"手机号": "2",
-		"邮箱": "3",
-		"不存在": "4",
+		"手机号":  "2",
+		"邮箱":   "3",
+		"不存在":  "4",
 	}
-
 )
 
 func init() {
@@ -42,15 +41,15 @@ func init() {
 		MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
 		Size:        10,
 		DbName:      "mixdata",
-		UserName: 	 "SJZY_RWESBid_Other",
-		Password:	 "SJZY@O17t8herB3B",
+		UserName:    "SJZY_RWESBid_Other",
+		Password:    "SJZY@O17t8herB3B",
 		//MongodbAddr: "192.168.3.207:27092",
 		//Size:        10,
 		//DbName:      "wjh",
 	}
 	MongoTool.InitPool()
 	Es = &es.Elastic{
-		S_esurl: "http://172.17.145.170:9800",				// http://172.17.145.170:9800
+		S_esurl: "http://172.17.145.170:9800", // http://172.17.145.170:9800
 		I_size:  10,
 	}
 	Es.InitElasticSize()
@@ -81,7 +80,7 @@ func main() {
 				<-ch
 				wg.Done()
 			}()
-			if  p1, ok := tmp["partners"].([]interface{}); ok {
+			if p1, ok := tmp["partners"].([]interface{}); ok {
 				p2 := p1[0].(map[string]interface{})
 				if p2["stock_capital"] != "" {
 					taskinfo(tmp)
@@ -97,9 +96,9 @@ func main() {
 	<-c
 }
 
-func taskinfo(tmp map[string]interface{})  {
+func taskinfo(tmp map[string]interface{}) {
 	esMap := make(map[string]interface{})
-	for _, v := range EsField{
+	for _, v := range EsField {
 		if tmp[v] != nil {
 			esMap[v] = tmp[v]
 		}
@@ -110,21 +109,21 @@ func taskinfo(tmp map[string]interface{})  {
 	if company_type == "个体工商户" {
 		if len([]rune(company_name)) >= 5 {
 			esMap["company_type_int"] = 31
-		}else {
+		} else {
 			esMap["company_type_int"] = 32
 		}
-	}else if company_type == "其他" || company_type == "" {
+	} else if company_type == "其他" || company_type == "" {
 		if len([]rune(company_name)) >= 4 {
 			esMap["company_type_int"] = 21
-		}else {
+		} else {
 			esMap["company_type_int"] = 22
 		}
-	}else {
+	} else {
 		if company_type == "内资分公司" {
 			esMap["company_type_int"] = 12
-		}else if len([]rune(company_name)) >= 4 {
+		} else if len([]rune(company_name)) >= 4 {
 			esMap["company_type_int"] = 11
-		}else {
+		} else {
 			esMap["company_type_int"] = 13
 		}
 	}
@@ -142,34 +141,34 @@ func taskinfo(tmp map[string]interface{})  {
 	}
 	if t1, ok := tmp["bid_unittype"].([]interface{}); ok {
 		var arr []string
-		for _, v := range util.ObjArrToStringArr(t1){
+		for _, v := range util.ObjArrToStringArr(t1) {
 			arr = append(arr, TypeMap[v])
 		}
 		esMap["bid_unittype"] = strings.Join(arr, ",")
 	}
 	if t2, ok := tmp["bid_contracttype"].([]interface{}); ok {
 		var arr []string
-		for _, v := range util.ObjArrToStringArr(t2){
+		for _, v := range util.ObjArrToStringArr(t2) {
 			arr = append(arr, TypeMap1[v])
 		}
 		esMap["bid_contracttype"] = strings.Join(arr, ",")
 	}
 	if p1, ok := tmp["partners"].([]interface{}); ok {
-		for _, v := range p1{
+		for _, v := range p1 {
 			v1 := v.(map[string]interface{})
 			if text := util.ObjToString(v1["stock_capital"]); text != "" {
 				if c1, err := strconv.ParseFloat(text, 64); err == nil {
-					c1, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", c1), 64)	//保留小数点两位
+					c1, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", c1), 64) //保留小数点两位
 					v1["stock_capital"] = c1
-				}else {
+				} else {
 					delete(v1, "stock_capital")
 				}
 			}
 			if text := util.ObjToString(v1["stock_realcapital"]); text != "" {
 				if c2, err := strconv.ParseFloat(text, 64); err == nil {
-					c2, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", c2), 64)	//保留小数点两位
+					c2, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", c2), 64) //保留小数点两位
 					v1["stock_realcapital"] = c2
-				}else {
+				} else {
 					delete(v1, "stock_realcapital")
 				}
 			}

qyxy/winner_tag/main.go

@@ -0,0 +1,169 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"mongodb"
+	"qfw/util"
+	"qfw/util/elastic"
+	"strings"
+	"time"
+)
+
+var (
+	Mgo						*mongodb.MongodbSim
+	Es						*elastic.Elastic
+
+	updatePool            	chan []map[string]interface{}
+	updateSp              	chan bool
+)
+
+func init() {
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",
+		Size:        10,
+		DbName:      "mixdata",
+		UserName:	 "SJZY_RWESBid_Other",
+		Password: 	 "SJZY@O17t8herB3B",
+	}
+	Mgo.InitPool()
+	Es = &elastic.Elastic{
+		S_esurl: "http://172.17.145.170:9800",									//http://172.17.145.170:9800
+		I_size:  10,
+	}
+	Es.InitElasticSize()
+
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp = make(chan bool, 5)
+}
+
+func main() {
+	go updateMethod()
+
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+
+	//q := map[string]interface{}{"_id": mongodb.StringTOBsonId("5e659c142c27dc56292df860")}
+	//q := bson.M{"extracttype": -1}
+	filed := map[string]interface{}{"company_name": 1}
+	query := sess.DB("mixdata").C("winner_enterprise").Find(nil).Select(filed).Iter()
+
+	es := `{
+	  "query": {
+		"bool": {
+		  "must": [],
+		  "must_not": [],
+		  "should": [
+			{
+			  "multi_match": {
+				"query": %s,
+				"type": "phrase",
+				"fields": [
+				  "s_winner.mwinner"
+				]
+			  }
+			}
+		  ]
+		}
+	  },
+	  "aggs": {
+		"my_name": {
+		  "terms": {
+			"field": "s_subscopeclass",
+			"size": 100
+		  }
+		}
+	  },
+	  "size": 0,
+	  "sort": [],
+	  "facets": {}
+	}`
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%200 == 0 {
+			util.Debug("current ---", count)
+		}
+		func(tmp map[string]interface{}) {
+
+			name := util.ObjToString(tmp["company_name"])
+			esquery := fmt.Sprintf(es, `"` + name + `"`)
+
+			searchResult, _ := client.Search().Index("bidding").Type("bidding").Source(esquery).Do()
+			var result map[string]interface{}
+			mapCount := make(map[string]int)
+			b, _ := json.Marshal(searchResult.Aggregations)
+			_ = json.Unmarshal(b, &result)
+			bk, _ := result["my_name"].(map[string]interface{})
+			if arr, ok := bk["buckets"].([]interface{}); ok {
+				for _, v := range arr{
+					v1 := v.(map[string]interface{})
+					key := strings.Split(util.ObjToString(v1["key"]), "_")[0]
+					mapCount[key] += util.IntAll(v1["doc_count"])
+				}
+			}
+			maxCount := 0
+			maxStr := ""
+			for k, v := range mapCount{
+				if v > maxCount {
+					maxCount = v
+					maxStr = k
+				}else if v == maxCount {
+					if maxStr == "" {
+						maxStr = k
+					}else {
+						maxStr = maxStr + "," + k
+					}
+				}
+			}
+			if maxCount != 0 {
+				updatePool <- []map[string]interface{}{
+					{"_id": tmp["_id"]},
+					{"$set": map[string]interface{}{"tag_business": maxStr}},
+				}
+			}
+		}(tmp)
+		tmp = map[string]interface{}{}
+	}
+	util.Debug("over ---", count)
+
+	c := make(chan bool, 1)
+	<-c
+}
+
+func updateMethod() {
+	arru := make([][]map[string]interface{}, 200)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					Mgo.UpSertBulk("winner_enterprise", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					Mgo.UpSertBulk("winner_enterprise", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		}
+	}
+}

qyxy/winner_tag/tools.go

@@ -0,0 +1 @@
+package main

udpcreateindex/src/biddingall.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
 	"mongodb"
 	"qfw/util/elastic"
 	"qfw/util/redis"
@@ -14,7 +16,6 @@ import (
 	"strings"
 	"sync"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 //对字段处理 bidamount  budget
@@ -173,7 +174,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 			defer func() {
 				<-mpool
 			}()
-			if !bnil && compare != nil {
+			if !bnil && compare != nil && len(compare) > 0 {
 				subscopeclass, _ := compare["subscopeclass"].([]interface{}) //subscopeclass
 				if subscopeclass != nil {
 					m1 := map[string]bool{}
@@ -229,6 +230,18 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				//					}
 				//					update["s_winner"] = strings.Join(winnerarr, ",")
 				//				}
+			}else {
+				area := qutil.ObjToString(tmp["area"])
+				city := qutil.ObjToString(tmp["city"])
+				district := qutil.ObjToString(tmp["district"])
+				UpdatesLock.Lock()
+				rdata := standardCheckCity(area, city, district)
+				UpdatesLock.Unlock()
+				if len(rdata) > 0 {
+					for k, v := range rdata {
+						update[k] = v
+					}
+				}
 			}
 			//------------------对比结束
 			//同时保存到elastic
@@ -242,15 +255,13 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 					if w != "" {
 						id := redis.GetStr("qyxy_id", w)
 						if id == "" {
-							ents, _ := mgostandard.Find("qyxy_std", bson.M{"company_name": w}, bson.M{"updatetime": -1}, nil, false, -1, -1)
+							ents, _ := mgostandard.Find("qyxy_std", map[string]interface{}{"company_name": w}, map[string]interface{}{"updatetime": -1}, nil, false, -1, -1)
 							if len(*ents) > 0 {
 								id = qutil.ObjToString((*ents)[0]["_id"])
-								redis.PutCKV("qyxy_id", w, id)
 							}else {
-								ent, _ := qyxydb.FindOne("company_history_name", bson.M{"company_name": w})
+								ent, _ := qyxydb.FindOne("company_history_name", map[string]interface{}{"history_name": w})
 								if len(*ent) > 0 {
 									id = qutil.ObjToString((*ent)["company_id"])
-									redis.PutCKV("qyxy_id", w, id)
 								}
 							}
 						}
@@ -309,8 +320,8 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 			UpdatesLock.Lock()
 			if qutil.IntAll(update["extracttype"]) != -1 {
 				newTmp := map[string]interface{}{}
-				for field, ftype := range biddingIndexFieldsMap { //
-					if tmp[field] != nil && del[field] == nil { //
+				for field, ftype := range biddingIndexFieldsMap {
+					if tmp[field] != nil && del[field] == nil {
 						//qutil.Debug(field, tmp[field], reflect.TypeOf(tmp[field]).String(), ftype)
 						if field == "projectinfo" {
 							mp, _ := tmp[field].(map[string]interface{})
@@ -406,11 +417,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 							if len([]rune(detail)) > detailLength {
 								detail = detail[:detailLength]
 							}
-							if strings.Contains(qutil.ObjToString(tmp["title"]), detail) {
-								newTmp[field] = FilterDetail(detail)
-							}else {
-								newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
-							}
+							newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
 						} else if field == "_id" || field == "topscopeclass" { //不做处理
 							newTmp[field] = tmp[field]
 						} else if field == "publishtime" || field == "comeintime" {
@@ -431,7 +438,6 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 						}
 					}
 				}
-
 				YuceEndtime(newTmp)
 				arrEs = append(arrEs, newTmp)
 			}
@@ -532,3 +538,125 @@ func UpdateExtract() {
 		}
 	}
 }
+
+//城市标准校验
+func standardCheckCity(area string,city string,district string) map[string]string{
+	rdata := make(map[string]string)
+	if area=="香港"||area=="澳门"||area=="台湾" || (area=="全国"&&(city==""&&district=="")) {
+		return rdata
+	}
+	//第一步：区校验
+	if district!="" {
+		districtArr := DistrictDict[district]
+		if districtArr==nil {//涉及了 个别别名相关的数据
+			trim_arr := aliasDataDistrict(district)//拆分后缀
+			if len(trim_arr)>0 {
+				for _,alias_district := range trim_arr {
+					alias_districtArr := DistrictDict[alias_district]
+					for _,v:=range alias_districtArr{
+						if  city == v.C_Name && area == v.P_Name {
+							rdata["district"] = alias_district
+							return rdata
+						}
+					}
+				}
+			}
+			rdata["district"] = ""
+		}else {
+			isTrue := false
+			for _,v:=range districtArr{
+				if  city == v.C_Name && area == v.P_Name {
+					isTrue = true
+					break
+				}
+			}
+			if isTrue { //完全匹配
+				return rdata
+			}else { //未完全匹配
+				if len(districtArr)==1 {
+					rdata["area"] = districtArr[0].P_Name
+					rdata["city"] = districtArr[0].C_Name
+					rdata["district"] = districtArr[0].D_Name
+					return rdata
+				}else {
+					rdata["district"] = ""
+				}
+			}
+		}
+	}
+
+	//第二步：区校验-失败   市-校验
+	if city != "" {
+		cityArr := CityDict[city]
+		if cityArr==nil {
+			//把市当成区，匹配三级   - 存在优化空间- city:郑州  别名
+			districtArr := DistrictDict[city]
+			for _,v:=range districtArr{
+				if  city == v.C_Name && area == v.P_Name {
+					rdata["area"] = districtArr[0].P_Name
+					rdata["city"] = districtArr[0].C_Name
+					rdata["district"] = districtArr[0].D_Name
+					return rdata
+				}
+			}
+			rdata["city"] = ""
+		}else {
+			isTrue := false
+			for _,v:=range cityArr{
+				if  area == v.P_Name {
+					isTrue = true
+					break
+				}
+			}
+			if isTrue { //完全匹配
+				return rdata
+			}else { //未完全匹配
+				if len(cityArr)==1 {
+					rdata["area"] = cityArr[0].P_Name
+					rdata["city"] = cityArr[0].C_Name
+					rdata["district"] = ""
+					return rdata
+				}else {
+					rdata["city"] = ""
+				}
+			}
+		}
+	}
+
+	//第三步：省份校验
+	if ProvinceDict[area]==nil {
+		rdata["area"] = "全国"
+		rdata["city"] = ""
+		rdata["district"] = ""
+	}
+
+	return rdata
+}
+
+var cityEndReg = regexp.MustCompile("(区|县|市)$")
+//拆分三级县
+func aliasDataDistrict(district string) []string {
+	arr :=[]string{}
+	if cityEndReg.MatchString(district) {
+		str := cityEndReg.FindString(district)
+		strings.TrimRight(district, str)
+		if str=="县"{
+			arr = append(arr,fmt.Sprintf("%s区",strings.TrimRight(district, str)))
+			arr = append(arr,fmt.Sprintf("%s市",strings.TrimRight(district, str)))
+		}else if str=="区"{
+			arr = append(arr,fmt.Sprintf("%s县",strings.TrimRight(district, str)))
+			arr = append(arr,fmt.Sprintf("%s市",strings.TrimRight(district, str)))
+		} else if str=="市"{
+			arr = append(arr,fmt.Sprintf("%s县",strings.TrimRight(district, str)))
+			arr = append(arr,fmt.Sprintf("%s区",strings.TrimRight(district, str)))
+		}else {
+
+		}
+	}else { //未找到 district- 区县市  例： district : 金水
+		arr = append(arr,fmt.Sprintf("%s区",district))
+		arr = append(arr,fmt.Sprintf("%s县",district))
+		arr = append(arr,fmt.Sprintf("%s市",district))
+	}
+	return arr
+}
+

udpcreateindex/src/biddingdata.go

@@ -1,62 +1,21 @@
 package main
 
 import (
-	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
 	elastic "qfw/util/elastic"
+	"reflect"
 	"strings"
-	"sync"
 
-	"gopkg.in/mgo.v2/bson"
+	//	"strings"
+	"sync"
 )
 
-//对字段处理 bidamount  budget
-var indexfield = []string{
-	"_id",
-	"s_winner",
-	"winner",
-	"buyerclass",
-	"title",
-	"detail",
-	"area",
-	"site",
-	"bidopendate",
-	"bidopentime",
-	"buyer",
-	"city",
-	"comeintime",
-	"href",
-	"infoformat",
-	"projectcode",
-	"projectname",
-	"publishtime",
-	"s_sha",
-	"spidercode",
-	"subtype",
-	"toptype",
-	"agency",
-	"budget",
-	"bidamount",
-	"s_subscopeclass",
-	"projectscope",
-	"bidstatus",
-	"projectinfo",
-	"buyertel",
-	"buyerperson",
-	"projectid",
-	"buyerclass",
-	"district",
-	"topscopeclass",
-	"attachments",
-}
-
-//招标数据表和抽取表一一对应开始更新
 func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 	defer qutil.Catch()
-	thread := 40
-	var mpool = make(chan bool, thread)
+	qutil.Debug(mapInfo)
 	q, _ := mapInfo["query"].(map[string]interface{})
 	if q == nil {
 		q = map[string]interface{}{
@@ -69,258 +28,176 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 	//bidding库
 	session := mgo.GetMgoConn()
 	defer mgo.DestoryMongoConn(session)
-	//extract库
-	extractsession := extractmgo.GetMgoConn()
-	defer extractmgo.DestoryMongoConn(extractsession)
 	//连接信息
-	c, _ := mapInfo["coll"].(string)
-	if c == "" {
-		c, _ = bidding["collect"].(string)
-	}
-
-	extractc, _ := bidding["extractcollect"].(string)
+	c, _ := bidding["collect"].(string)
 	db, _ := bidding["db"].(string)
-	extractdb, _ := bidding["extractdb"].(string)
-	index, _ := bidding["index"].(string)
-	itype, _ := bidding["type"].(string)
+	index := "bidding_all"
+	itype := "bidding"
 	count, _ := session.DB(db).C(c).Find(&q).Count()
-	fields := strings.Split(bidding["fields"].(string), ",")
 	//线程池
 	UpdatesLock := sync.Mutex{}
-
-	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	qutil.Debug("查询语句:", q, "同步总数:", count, "elastic库:", index)
 	//查询招标数据
 	query := session.DB(db).C(c).Find(q).Select(bson.M{
 		"projectinfo.attachment": 0,
 		"contenthtml":            0,
+		"publishdept":			  0,
+		"detail": 				  0,
+		"projectscope":			  0,
+		"project_scale":		  0,
+		"bidstatus":			  0,
+		"china_bidding":		  0,
 	}).Sort("_id").Iter()
-	//查询抽取结果
-	extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Sort("_id").Iter()
-
 	n := 0
-
 	//更新数组
-
 	arrEs := []map[string]interface{}{}
-	//对比两张表数据，减少查询次数
-	var compare bson.M
-	bnil := false
+	thread := 10
+	espool := make(chan bool, 5)
+	var mpool = make(chan bool, thread)
 	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
-		update := map[string]interface{}{}
-		//对比方法----------------
-		for {
-			if compare == nil {
-				compare = make(bson.M)
-				if !extractquery.Next(compare) {
-					break
-				}
-			}
-			if compare != nil {
-				//对比
-				cid := qutil.BsonIdToSId(compare["_id"])
-				tid := qutil.BsonIdToSId(tmp["_id"])
-				if cid == tid {
-					bnil = false
-					//更新bidding表，生成索引
-					for _, k := range fields {
-						v1 := compare[k] //extract
-						v2 := tmp[k]     //bidding
-						if v2 == nil && v1 != nil {
-							update[k] = v1
-						} else if v2 != nil && v1 != nil {
-							update[k] = v1
-						} else if v2 != nil && v1 == nil {
-							update[k] = v2
-						}
-					}
-					if qutil.IntAll(compare["repeat"]) == 1 {
-						update["extracttype"] = -1
-						//} else if qutil.IntAll(tmp["extracttype"]) == -1 {
-					} else {
-						update["extracttype"] = 1
-					}
-					break
-				} else {
-					if cid < tid {
-						bnil = false
-						compare = nil
-						continue
-					} else {
-						bnil = true
-						break
-					}
-				}
-			} else {
-				bnil = false
-				break
-			}
+		if n%20000 == 0 {
+			log.Println("current:", n, tmp["_id"])
 		}
-		//下面可以多线程跑的--->
-		//处理分类
 		mpool <- true
-		go func(tmp, update, compare map[string]interface{}, bnil bool) {
+		go func(tmp map[string]interface{}) {
 			defer func() {
 				<-mpool
 			}()
-			if !bnil && compare != nil {
-				subscopeclass, _ := compare["subscopeclass"].([]interface{})
-				if subscopeclass != nil {
-					m1 := map[string]bool{}
-					newclass := []string{}
-					for _, sc := range subscopeclass {
-						sclass, _ := sc.(string)
-						if !m1[sclass] {
-							m1[sclass] = true
-							newclass = append(newclass, sclass)
-						}
-					}
-					update["s_subscopeclass"] = strings.Join(newclass, ",")
-					update["subscopeclass"] = newclass
-				}
-				//处理中标企业
-				//				winner, _ := compare["winner"].(string)
-				//				m1 := map[string]bool{}
-				//				if winner != "" {
-				//					m1[winner] = true
-				//				}
-				//				package1 := compare["package"]
-				//				if package1 != nil {
-				//					packageM, _ := package1.(map[string]interface{})
-				//					for _, p := range packageM {
-				//						pm, _ := p.(map[string]interface{})
-				//						pw, _ := pm["winner"].(string)
-				//						if pw != "" {
-				//							m1[pw] = true
-				//						}
-				//					}
-				//				}
-				compare = nil
-				//				if len(m1) > 0 {
-				//					//str := ","
-				//					winnerarr := []string{}
-				//					for k, _ := range m1 {
-				//						//str += k + ","
-				//						winnerarr = append(winnerarr, k)
-				//					}
-				//					update["s_winner"] = strings.Join(winnerarr, ",")
-				//				}
-			}
-			//------------------对比结束
-
-			//同时保存到elastic
-			for tk, tv := range update {
-				tmp[tk] = tv
-			}
-			if tmp["supervisorrate"] != nil { //临时处理supervisorrate抽取类型为string不生索引
-				if _, ok := tmp["supervisorrate"].(string); ok { //supervisorrate数据为string类型
-					delete(tmp, "supervisorrate")
-				}
-			}
-			//对projectscope字段的索引处理
-			ps, _ := tmp["projectscope"].(string)
-			if len(ps) > ESLEN {
-				tmp["projectscope"] = string(([]rune(ps))[:4000])
-			}
-			//对标的物为空处理
-			if filetext := getFileText(tmp); len(filetext) > 0 { //attach_text
-				tmp["filetext"] = filetext
-			}
 			if purchasing, ok := tmp["purchasing"].(string); ok && purchasing == "" {
 				delete(tmp, "purchasing")
 			}
 			if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok && len(purchasinglist) == 0 {
 				delete(tmp, "purchasinglist")
 			}
-			//			if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
-			//				tmp["budget"] = nil
-			//			} else if sbd, ok := tmp["budget"].(string); ok {
-			//				tmp["budget"] = ObjToMoney([]interface{}{sbd, sbd})[0]
-			//			}
-			//			if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
-			//				tmp["bidamount"] = nil
-			//			} else if sbd, ok := tmp["bidamount"].(string); ok {
-			//				tmp["bidamount"] = ObjToMoney([]interface{}{sbd, sbd})[0]
-			//			}
-
-			if qutil.IntAll(update["extracttype"]) != -1 {
-				newTmp := map[string]interface{}{}
-				for _, v := range biddingIndexFields { // indexfield
-					if tmp[v] != nil {
-						if "projectinfo" == v {
-							mp, _ := tmp[v].(map[string]interface{})
-							if mp != nil {
-								newmap := map[string]interface{}{}
-								for _, v1 := range projectinfoFields {
-									if mp[v1] != nil {
-										newmap[v1] = fmt.Sprint(mp[v1])
+			//数据为空处理
+			for _, f := range []string{"bidstatus", "city", "district", "channel"} {
+				if fVal, ok := tmp[f].(string); ok && fVal == "" {
+					delete(tmp, f)
+				}
+			}
+			UpdatesLock.Lock()
+			newTmp := map[string]interface{}{}
+			for field, ftype := range biddingIndexFieldsMap {
+				if tmp[field] != nil { //
+					if field == "projectinfo" {
+						mp, _ := tmp[field].(map[string]interface{})
+						if mp != nil {
+							newmap := map[string]interface{}{}
+							for k, ktype := range projectinfoFieldsMap {
+								mpv := mp[k]
+								if mpv != nil && reflect.TypeOf(mpv).String() == ktype {
+									newmap[k] = mp[k]
+								}
+							}
+							if len(newmap) > 0 {
+								newTmp[field] = newmap
+							}
+						}
+					} else if field == "purchasinglist" { //标的物处理
+						purchasinglist_new := []map[string]interface{}{}
+						if pcl, _ := tmp[field].([]interface{}); len(pcl) > 0 {
+							for _, ls := range pcl {
+								lsm_new := make(map[string]interface{})
+								lsm := ls.(map[string]interface{})
+								for pf, pftype := range purchasinglistFieldsMap {
+									lsmv := lsm[pf]
+									if lsmv != nil && reflect.TypeOf(lsmv).String() == pftype {
+										lsm_new[pf] = lsm[pf]
 									}
 								}
-								if len(newmap) > 0 {
-									newTmp[v] = newmap
+								if lsm_new != nil && len(lsm_new) > 0 {
+									purchasinglist_new = append(purchasinglist_new, lsm_new)
 								}
-								// attachments := mp["attachments"]
-								// con := ""
-								// if attachments != nil {
-								// 	am, _ := attachments.(map[string]interface{})
-								// 	if am != nil {
-								// 		for _, v1 := range am {
-								// 			vm, _ := v1.(map[string]interface{})
-								// 			if vm != nil {
-								// 				c, _ := vm["content"].(string)
-								// 				con += c
-								// 			}
-								// 		}
-								// 	}
-								// }
-								// if con != "" {
-								// 	con = FilterDetailSpace(con)
-								// 	newTmp["attachments"] = con
-								// }
 							}
-						} else if v == "purchasinglist" { //标的物处理
-							purchasinglist_new := []map[string]interface{}{}
-							if pcl, _ := tmp[v].([]interface{}); len(pcl) > 0 {
-								for _, ls := range pcl {
-									lsm_new := make(map[string]interface{})
-									lsm := ls.(map[string]interface{})
-									for _, pf := range purchasinglistFields {
-										if lsm[pf] != nil {
-											lsm_new[pf] = lsm[pf]
+						}
+						if len(purchasinglist_new) > 0 {
+							newTmp[field] = purchasinglist_new
+						}
+					} else if field == "winnerorder" { //中标候选
+						winnerorder_new := []map[string]interface{}{}
+						if winnerorder, _ := tmp[field].([]interface{}); len(winnerorder) > 0 {
+							for _, win := range winnerorder {
+								winMap_new := make(map[string]interface{})
+								winMap := win.(map[string]interface{})
+								for wf, wftype := range winnerorderlistFieldsMap {
+									wfv := winMap[wf]
+									if wfv != nil && reflect.TypeOf(wfv).String() == wftype {
+										if wf == "sort" && qutil.Int64All(wfv) > 100 {
+											continue
 										}
+										winMap_new[wf] = winMap[wf]
 									}
-									if lsm_new != nil && len(lsm_new) > 0 {
-										purchasinglist_new = append(purchasinglist_new, lsm_new)
-									}
+								}
+								if winMap_new != nil && len(winMap_new) > 0 {
+									winnerorder_new = append(winnerorder_new, winMap_new)
 								}
 							}
-							if len(purchasinglist_new) > 0 {
-								newTmp[v] = purchasinglist_new
+						}
+						if len(winnerorder_new) > 0 {
+							newTmp[field] = winnerorder_new
+						}
+					} else if field == "qualifies" {
+						//项目资质
+						qs := []string{}
+						if q, _ := tmp[field].([]interface{}); len(q) > 0 {
+							for _, v := range q {
+								v1 := v.(map[string]interface{})
+								qs = append(qs, qutil.ObjToString(v1["key"]))
 							}
+						}
+						if len(qs) > 0 {
+							newTmp[field] = strings.Join(qs, ",")
+						}
+					} else if field == "review_experts" {
+						// 评审专家
+						if arr, ok :=tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
+							arr1 := qutil.ObjArrToStringArr(arr)
+							newTmp[field] = strings.Join(arr1, ",")
+						}
+					} else if field == "entidlist" {
+						newTmp[field] = tmp[field]
+					} else if field == "bidopentime" {
+						if tmp[field] != nil && tmp["bidendtime"] == nil {
+							newTmp["bidendtime"] = tmp[field]
+						}
+						if tmp[field] == nil && tmp["bidendtime"] != nil {
+							newTmp[field] = tmp["bidendtime"]
+						}
+					} else if field == "_id" || field == "topscopeclass" { //不做处理
+						newTmp[field] = tmp[field]
+					} else if field == "publishtime" || field == "comeintime" {
+						//字段类型不正确，特别处理
+						if tmp[field] != nil && qutil.Int64All(tmp[field]) > 0 {
+							newTmp[field] = qutil.Int64All(tmp[field])
+						}
+					} else if field == "s" {
+						newTmp[field] = tmp[field]
+					} else { //其它字段判断数据类型，不正确舍弃
+						if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {
+							continue
 						} else {
-							if v == "detail" {
-								detail, _ := tmp[v].(string)
-								newTmp[v] = FilterDetail(detail)
-							} else {
-								newTmp[v] = tmp[v]
+							if fieldval != "" {
+								newTmp[field] = fieldval
 							}
 						}
 					}
 				}
-				UpdatesLock.Lock()
-				arrEs = append(arrEs, newTmp)
-				UpdatesLock.Unlock()
 			}
-			UpdatesLock.Lock()
-			if len(arrEs) >= BulkSize-1 {
+			newTmp["extracttype"] = qutil.IntAll(tmp["extracttype"])
+			arrEs = append(arrEs, newTmp)
+			if len(arrEs) >= BulkSizeBack {
 				tmps := arrEs
-				elastic.BulkSave(index, itype, &tmps, true)
+				espool <- true
+				go func(tmps []map[string]interface{}) {
+					defer func() {
+						<-espool
+					}()
+					elastic.BulkSave(index, itype, &tmps, true)
+				}(tmps)
 				arrEs = []map[string]interface{}{}
 			}
 			UpdatesLock.Unlock()
-		}(tmp, update, compare, bnil)
-		if n%1000 == 0 {
-			log.Println("current:", n)
-		}
+		}(tmp)
 		tmp = make(map[string]interface{})
 	}
 	for i := 0; i < thread; i++ {
@@ -332,5 +209,5 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 		elastic.BulkSave(index, itype, &tmps, true)
 	}
 	UpdatesLock.Unlock()
-	log.Println(mapInfo, "create bidding index...over", n)
-}
+	log.Println(mapInfo, "create biddingdata index...over", n)
+}

udpcreateindex/src/biddingdeletebyextract.go

@@ -1,13 +1,13 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
 
 	elastic "qfw/util/elastic"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 //根据抽取表repeat=1，删除es中重复数据

udpcreateindex/src/biddingdeletebyextracttype.go

@@ -1,13 +1,13 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
 
 	elastic "qfw/util/elastic"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 //根据bidding表extracttype=-1，删除es中重复数据

udpcreateindex/src/biddingindex.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"github.com/donnie4w/go-logger/logger"
 	"go.mongodb.org/mongo-driver/bson"
 	"mongodb"
 	"qfw/util/redis"
@@ -37,7 +36,7 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 		}
 	}
 
-	logger.SetRollingDaily("./logs", "id.log")
+	//logger.SetRollingDaily("./logs", "id.log")
 	//连接信息
 	c, _ := bidding["collect"].(string)
 	extractc, _ := bidding["extractcollect"].(string)
@@ -64,7 +63,7 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 	//bidding库
 	session := mgo.GetMgoConn()
 	count, _ := session.DB(db).C(c).Find(&q).Count()
-	log.Println("抽取表 重复数据量：", repeatCount)
+	log.Println("抽取表 重复数据量：", extCount, repeatCount)
 	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
 	n1, n2 := 0, 0
 	if count < 200000 {
@@ -79,7 +78,7 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 			tmp = make(map[string]interface{})
 		}
 		mgo.DestoryMongoConn(session)
-		log.Println("查询结果不一致", "bidding:", count, "抽取:", extCount)
+		log.Println("查询结果", "bidding:", count, "抽取:", extCount)
 		if int64(len(res)) != count {
 			time.Sleep(20 * time.Second)
 			toadd := &net.UDPAddr{
@@ -187,9 +186,12 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 					update[k] = v1
 				} else if v2 != nil && v1 == nil {
 					//update[k+"_b"] = v2
+					if k == "area" || k == "city" || k == "district" {
+						update[k] = ""
+					}
 				}
+
 			}
-			//logger.Debug("_id---", tid, ",repeat---", compare["repeat"], ",dataging---", compare["dataging"])
 			if qutil.IntAll(compare["repeat"]) == 1 {
 				update["extracttype"] = -1
 			} else {
@@ -261,6 +263,16 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 			//				}
 			//				update["s_winner"] = strings.Join(winnerarr, ",")
 			//			}
+		}else {
+			area := qutil.ObjToString(tmp["area"])
+			city := qutil.ObjToString(tmp["city"])
+			district := qutil.ObjToString(tmp["district"])
+			rdata := standardCheckCity(area, city, district)
+			if len(rdata) > 0 {
+				for k, v := range rdata {
+					update[k] = v
+				}
+			}
 		}
 		//------------------对比结束
 
@@ -284,7 +296,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 							id = qutil.ObjToString((*ents)[0]["_id"])
 							redis.PutCKV("qyxy_id", w, id)
 						}else {
-							ent, _ := qyxydb.FindOne("company_history_name", bson.M{"company_name": w})
+							ent, _ := qyxydb.FindOne("company_history_name", bson.M{"history_name": w})
 							if len(*ent) > 0 {
 								id = qutil.ObjToString((*ent)["company_id"])
 								redis.PutCKV("qyxy_id", w, id)
@@ -671,6 +683,10 @@ func YuceEndtime(tmp map[string]interface{}) {
 				}
 			}
 		}
+		if yucestarttime > 0 && yuceendtime > yucestarttime {
+			tmp["yuceendtime"] = yuceendtime
+			return
+		}
 		// 预测开始时间 合同签订日期
 		if yucestarttime == 0 {
 			if qutil.IntAll(tmp["signaturedate"]) <= 0 {
@@ -687,7 +703,7 @@ func YuceEndtime(tmp map[string]interface{}) {
 		if yucestarttime > 0 && yuceendtime == 0 {
 			if qutil.IntAll(tmp["project_duration"]) > 0 && qutil.ObjToString(tmp["project_timeunit"]) != "" {
 				yuceendtime = YcEndTime(yucestarttime, qutil.IntAll(tmp["project_duration"]), qutil.ObjToString(tmp["project_timeunit"]))
-				tmp["yeceendtime"] = yuceendtime
+				tmp["yuceendtime"] = yuceendtime
 			}
 		}
 	}

udpcreateindex/src/biddingindexback.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
@@ -8,7 +9,6 @@ import (
 	"reflect"
 	"strings"
 
-	"gopkg.in/mgo.v2/bson"
 	//elastic "qfw/util/elastic_v5"
 	"regexp"
 	//	"strings"
@@ -221,7 +221,7 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 						if len([]rune(detail)) > detailLength {
 							detail = detail[:detailLength]
 						}
-						if strings.Contains(qutil.ObjToString(tmp["title"]), detail) {
+						if strings.Contains(detail, qutil.ObjToString(tmp["title"])) {
 							newTmp[field] = FilterDetail(detail)
 						}else {
 							newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
@@ -284,3 +284,9 @@ func FilterDetail(text string) string {
 func FilterDetailSpace(text string) string {
 	return filterSpace.ReplaceAllString(text, "")
 }
+
+// 正则判断是否包含
+func checkContains(s, sub string) bool {
+	reg := regexp.MustCompile(`(?i)(^|([\s\t\n]+))(` +sub+`)($|([\s\t\n]+))`)
+	return reg.MatchString(s)
+}

udpcreateindex/src/biddingindexback2.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
@@ -13,7 +14,6 @@ import (
 	"sync"
 	"time"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 func biddingBackTask2(data []byte, mapInfo map[string]interface{}) {

udpcreateindex/src/biddingmerge.go

@@ -2,13 +2,13 @@ package main
 
 import (
 	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
 	"strings"
 	"sync"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 //对字段处理 bidamount  budget

udpcreateindex/src/bidingpurchasing.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"qfw/util"
 	elastic "qfw/util/elastic"
@@ -9,7 +10,6 @@ import (
 
 	u "util"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 //定时查询bidding中extract_state为2的数据生成索引

udpcreateindex/src/buyerindex.go

@@ -1,13 +1,13 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	"qfw/util"
 	elastic "qfw/util/elastic"
 	"sync"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 /*

udpcreateindex/src/buyertask.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"log"
 	"mongodb"
 	qu "qfw/util"
@@ -8,7 +9,6 @@ import (
 	"sync"
 	"time"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 var fieldArr = []string{"institute_type", "fixedphone", "mobilephone", "latestfixedphone", "latestmobilephone", "province", "city"}
@@ -23,8 +23,8 @@ func buyerEsTaskOnce() {
 	now := time.Now()
 	preTime := time.Date(now.Year(), now.Month(), now.Day()-1, now.Hour(), 0, 0, 0, time.Local)
 	curTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.Local)
-	task_sid := mongodb.BsonIdToSId(bson.NewObjectIdWithTime(preTime))
-	task_eid := mongodb.BsonIdToSId(bson.NewObjectIdWithTime(curTime))
+	task_sid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(preTime))
+	task_eid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(curTime))
 	log.Println("buyer 区间id:", task_sid, task_eid)
 	// task_sid = "5e6611b7aec95406dccf7151"
 	// task_eid = "5f7249164bdc0447a6c90fa5"

udpcreateindex/src/config.json

@@ -30,7 +30,7 @@
     "type": "buyer"
   },
   "biddingback": {
-    "db": "qfw",
+    "db": "wjh",
     "collect": "bidding",
     "index": "bidding",
     "type": "bidding"
@@ -41,7 +41,7 @@
     "index": "bidding_v1",
     "type": "bidding",
     "extractdb": "wjh",
-    "extractcollect": "",
+    "extractcollect": "extract",
     "indexfields": [
       "buyerzipcode", "winnertel", "winnerperson", "contractcode", "winneraddr", "agencyaddr", "buyeraddr", "signaturedate", "projectperiod", "projectaddr", "agencytel", "agencyperson",
       "buyerperson", "agency", "projectscope", "projectcode", "bidopentime", "supervisorrate", "buyertel", "bidamount", "winner", "buyer", "budget", "projectname", "bidstatus", "buyerclass",
@@ -101,6 +101,7 @@
     "addr": "192.168.3.207:27092",
     "pool": 10,
     "db": "mixdata",
+    "coll_area": "address_jy_2021",
     "winnerent": {
       "collect1": "winner_enterprise",
       "collect2": "winner_err",
@@ -120,7 +121,7 @@
     }
   },
   "elastic": {
-    "addr": "http://192.168.3.11:9800",
+    "addr": "http://127.0.0.1:9800",
     "pool": 12,
     "node": "4q7v7e6mQ5aeCwjUgM6HcA"
   }

udpcreateindex/src/default.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"gopkg.in/mgo.v2/bson"
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	"qfw/util"

udpcreateindex/src/main.go

@@ -2,20 +2,34 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"log"
 	mu "mfw/util"
 	"mongodb"
 	"net"
 	"net/http"
+	_ "net/http/pprof"
 	"qfw/util"
 	elastic "qfw/util/elastic"
-	"qfw/util/redis"
 	"strings"
 	"time"
 	u "util"
 )
 
+type Province struct {
+	P_Name    string
+}
+type City struct {
+	P_Name string
+	C_Name string
+}
+type District struct {
+	P_Name string
+	C_Name string
+	D_Name string
+}
+
 var (
 	Sysconfig                map[string]interface{} //配置文件
 	mgo                      *mongodb.MongodbSim    //mongodb操作对象
@@ -46,6 +60,10 @@ var (
 	esAddr 			 string
 	esNode			 string
 
+	ProvinceDict	map[string][]Province				//省份-map
+	CityDict		map[string][]City					//城市-map
+	DistrictDict	map[string][]District				//区县-map
+
 	winner, bidding, biddingback, project, project2, buyer, standard, qyxy_ent map[string]interface{}
 )
 var UpdataMgoCache = make(chan []map[string]interface{}, 1000)
@@ -56,8 +74,8 @@ var StopFlag = false		// 程序生索引停止标志
 func init() {
 	util.ReadConfig(&Sysconfig)
 	// company_id
-	redis.InitRedis1("qyxy_id=172.17.4.189:8379", 4)
-	inits()
+	//redis.InitRedis1("qyxy_id=172.17.4.189:8379", 4)
+	//inits()
 	//go checkMapJob()
 	detailLength = util.IntAllDef(Sysconfig["detaillength"], 50000)
 	fileLength = util.IntAllDef(Sysconfig["filelength"], 50000)
@@ -76,9 +94,8 @@ func init() {
 		MongodbAddr: mconf["addr"].(string),
 		Size:        util.IntAllDef(mconf["pool"], 5),
 		DbName:      mconf["db"].(string),
-		UserName:	 Sysconfig["uname"].(string),
-		Password:    Sysconfig["upwd"].(string),
-		ReplSet: 	 "bidding",
+		//UserName:	 Sysconfig["uname"].(string),
+		//Password:    Sysconfig["upwd"].(string),
 	}
 	mgo.InitPool()
 	project2db = &mongodb.MongodbSim{
@@ -116,8 +133,8 @@ func init() {
 		MongodbAddr: standard["addr"].(string),
 		Size:        util.IntAllDef(standard["pool"], 5),
 		DbName:      standard["db"].(string),
-		UserName:    Sysconfig["uname"].(string),
-		Password:    Sysconfig["upwd"].(string),
+		//UserName:    Sysconfig["uname"].(string),
+		//Password:    Sysconfig["upwd"].(string),
 	}
 	mgostandard.InitPool()
 
@@ -194,14 +211,17 @@ func init() {
 	}
 	log.Println(projectinfoFields)
 	log.Println(purchasinglistFields)
+
+	initCheckCity()
 	//初始化oss
 	u.InitOss()
 
 }
 
 func main() {
-	//go inspectQuery()
-	//go task_index()
+
+	go inspectQuery()
+	go task_index()
 	go UpdateExtract() //抽取表中新增entidlist字段
 	updport := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
@@ -306,7 +326,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					}()
 					biddingAllTask(data, mapInfo)
 				}()
-			case "biddingdata": //联表生成索引不合并，不生成关键词
+			case "biddingdata": //bidding全量数据
 				pool <- true
 				go func() {
 					defer func() {
@@ -384,3 +404,66 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 		log.Println("发送成功", string(data))
 	}
 }
+
+//初始化城市
+func initCheckCity()  {
+	//初始化-城市配置
+	ProvinceDict = make(map[string][]Province,0)
+	CityDict = make(map[string][]City,0)
+	DistrictDict = make(map[string][]District,0)
+
+	q := map[string]interface{}{
+		"town_code":map[string]interface{}{
+			"$exists":0,
+		},
+	}
+	sess := mgostandard.GetMgoConn()
+	defer mgostandard.DestoryMongoConn(sess)
+	it := sess.DB("mixdata").C(util.ObjToString(standard["coll_area"])).Find(&q).Iter()
+	total  := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%1000 == 0 {
+			log.Println("当前数量:", total)
+		}
+		district_code := util.IntAll(tmp["district_code"])
+		city_code := util.IntAll(tmp["city_code"])
+		if district_code > 0 {
+			province := util.ObjToString(tmp["province"])
+			city := util.ObjToString(tmp["city"])
+			district := util.ObjToString(tmp["district"])
+			data := District{province,city,district}
+			if DistrictDict[district]==nil {
+				DistrictDict[district] = []District{data}
+			}else {
+				arr := DistrictDict[district]
+				arr = append(arr,data)
+				DistrictDict[district] = arr
+			}
+		}else {
+			if city_code>0 {
+				province := util.ObjToString(tmp["province"])
+				city := util.ObjToString(tmp["city"])
+				data := City{province,city}
+				if CityDict[city]==nil {
+					CityDict[city] = []City{data}
+				}else {
+					arr := CityDict[city]
+					arr = append(arr,data)
+					CityDict[city] = arr
+				}
+			}else {
+				province := util.ObjToString(tmp["province"])
+				data := Province{province}
+				if ProvinceDict[province]==nil {
+					ProvinceDict[province] = []Province{data}
+				}else {
+					arr := ProvinceDict[province]
+					arr = append(arr,data)
+					ProvinceDict[province] = arr
+				}
+			}
+		}
+		tmp = make(map[string]interface{})
+	}
+	util.Debug(fmt.Sprintf("城市配置加载完毕...省~%d 市~%d 区~%d",len(ProvinceDict),len(CityDict),len(DistrictDict)))
+}

udpcreateindex/src/projectindex.go

@@ -55,7 +55,10 @@ func projectTask(data []byte, project, mapInfo map[string]interface{}) {
 	arr := make([]map[string]interface{}, savesizei)
 	var n int
 	i := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+	for tmp := make(map[string]interface{}); query.Next(tmp); i++ {
+		if n%10000 == 0 {
+			log.Println("current---------", n)
+		}
 		pp := map[string]map[string]interface{}{}
 		if packages, ok := tmp["package"].(map[string]interface{}); ok {
 			for _, pks := range packages {

udpcreateindex/src/standardata.go

@@ -1,12 +1,12 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	mgov "mongodb"
 	"qfw/util"
 	elastic "qfw/util/elastic"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 func standardTask(stype string, mapInfo map[string]interface{}) {

udpcreateindex/src/task.go

@@ -2,6 +2,7 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
@@ -16,19 +17,32 @@ func task_index() {
 	//c.AddFunc("0 30 * * * *", func() { task_biddingfile() }) //每30分钟执行一次
 	//c.AddFunc("0 22 14 * * *", func() { task_qyxyindex() })
 
-	c.AddFunc("0 0 0 * * ?", func() { task_winneres() }) //每天凌晨执行一次winner生索引
-	c.AddFunc("0 0 1 * * ?", func() { task_buyeres() })  //每天1点执行一次buyer生索引
+	_ = c.AddFunc("0 0 0 * * ?", func() { task_winneres() })   //每天凌晨执行一次winner生索引
+	_ = c.AddFunc("0 0 1 * * ?", func() { task_buyeres() })    //每天1点执行一次buyer生索引
+	_ = c.AddFunc("0 0 2 * * ?", func() { task_biddingAll() }) //每天3点执行 前一天的所有招标数据
 	c.Start()
 }
 func task_winneres() {
-	log.Println("定时任务，winneres")
+	qutil.Debug("定时任务，winneres")
 	winnerEsTaskOnce()
 }
 func task_buyeres() {
-	log.Println("定时任务，buyeres")
+	qutil.Debug("定时任务，buyeres")
 	buyerEsTaskOnce()
 }
 
+func task_biddingAll() {
+	qutil.Debug("定时任务，bidding_all")
+	t := time.Now()
+	currenttime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
+	endtime := time.Unix(currenttime.Unix(), 0).AddDate(0, 0, -1)
+
+	sid := primitive.NewObjectIDFromTimestamp(endtime)
+	eid := primitive.NewObjectIDFromTimestamp(currenttime)
+
+	biddingDataTask(nil, map[string]interface{}{"gtid": mongodb.BsonIdToSId(sid), "lteid": mongodb.BsonIdToSId(eid)})
+}
+
 //招标附件、标的物,临时用
 func task_biddingfile() {
 	defer qutil.Catch()

udpcreateindex/src/winnerindex.go

@@ -1,12 +1,12 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	"qfw/util"
 	elastic "qfw/util/elastic"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 func winnerTask(data []byte, mapInfo map[string]interface{}) {

udpcreateindex/src/winnertask.go

@@ -1,14 +1,13 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"log"
 	"mongodb"
 	qu "qfw/util"
 	elastic "qfw/util/elastic"
 	"sync"
 	"time"
-
-	"gopkg.in/mgo.v2/bson"
 )
 
 func winnerEsTaskOnce() {
@@ -21,8 +20,8 @@ func winnerEsTaskOnce() {
 	now := time.Now()
 	preTime := time.Date(now.Year(), now.Month(), now.Day()-1, now.Hour(), 0, 0, 0, time.Local)
 	curTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.Local)
-	task_sid := mongodb.BsonIdToSId(bson.NewObjectIdWithTime(preTime))
-	task_eid := mongodb.BsonIdToSId(bson.NewObjectIdWithTime(curTime))
+	task_sid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(preTime))
+	task_eid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(curTime))
 	// task_sid = "5e6598f82c27dc56292158da"
 	// task_eid = "5f80c8f89a0c261af872294c"
 	log.Println("winner 区间id:", task_sid, task_eid)

udps/main.go

@@ -22,13 +22,8 @@ func main() {
 	flag.IntVar(&p, "p", 6601, "端口")
 	flag.IntVar(&tmptime, "tmptime", 0, "时间查询")
 	flag.StringVar(&tmpkey, "tmpkey", "", "时间字段")
-<<<<<<< HEAD
 	flag.StringVar(&id1, "gtid", "", "gtid")
 	flag.StringVar(&id2, "lteid", "", "lteid")
-=======
-	flag.StringVar(&id1, "gtid", "114168ea1a75b8f44678a39b", "gtid")
-	flag.StringVar(&id2, "lteid", "9142e5741a75b8f4467b3276", "lteid")
->>>>>>> f5506419ade445c9ae7863e3d184d86b2b0ca84f
 	flag.StringVar(&ids, "ids", "", "id1,id2")
 	flag.StringVar(&stype, "stype", "biddingall", "stype,传递类型")
 	flag.StringVar(&bkey, "bkey", "", "bkey,加上此参数表示不生关键词和摘要")