Merge branch 'dev3.4.1' of http://192.168.3.207:8080/qmx/jy-data-extract into dev3.4.1

esmgocount/src/main.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"qfw/util"
 	"qfw/util/elastic"
+	"strconv"
 	"strings"
 	"time"
 
@@ -36,6 +37,7 @@ var (
 	esIndex, esDataIndex string
 	Ts                   = []*T{}
 	esQ                  = `{"query": {"range": {"id": {"gte": "%s","lt": "%s"}}}}`
+	esQ1                 = `{"query": {"bool": {"must": [{"range": {"id": {"gte": "%s","lt": "%s"}}},{"terms": {"bidding.site": ["元博网（采购与招标网）","中国招标与采购网"]}}]}}}`
 )
 
 func init() {
@@ -93,9 +95,11 @@ func (t *T) task() {
 	st1 := fmt.Sprintf("%x0000000000000000", st)
 	et1 := fmt.Sprintf("%x0000000000000000", et)
 	eq = fmt.Sprintf(esQ, st1, et1)
+	eq1 := fmt.Sprintf(esQ1, st1, et1)
 	es := elastic.Elastic{S_esurl: esAddr, I_size: 1}
 	es.InitElasticSize()
 	count := int(es.Count(esIndex, esIndex, eq))
+	count1 := int(es.Count(esIndex, esIndex, eq1))
 	switch t.Type {
 	case "alert":
 		if count < t.Min || count > t.Max {
@@ -103,16 +107,37 @@ func (t *T) task() {
 			t.SendMail(report)
 		}
 	case "report":
-		report := fmt.Sprintf("报告%s,统计结果%d", t.Name, count)
+		report := fmt.Sprintf("报告%s,统计结果：%d", t.Name, count)
 		if len(t.Mgo) > 5 {
 			fs := strings.Split(t.Mgo, "|")
 			fmgo := mongodb.NewMgoWithUser(fs[0], fs[3], fs[1], fs[2], 1)
 			id1 := mongodb.StringTOBsonId(st1)
 			id2 := mongodb.StringTOBsonId(et1)
-			mq := bson.M{"extracttype": bson.M{"$ne": -1}, "sensitive": bson.M{"$ne": "测试"}, "dataging": bson.M{"$ne": 1}, "_id": bson.M{"$gte": id1, "$lt": id2}}
-			count2 := fmgo.Count(fs[4], mq)
-			count3 := fmgo.Count(fs[4], bson.M{"_id": bson.M{"$gte": id1, "$lt": id2}}) //mgo总入库量
-			report += ",mgo统计" + fmt.Sprint(count2) + ",差值:" + fmt.Sprint(count2-count) + ",mgo总入库量" + fmt.Sprint(count3)
+			//mq := bson.M{"extracttype": bson.M{"$ne": -1}, "sensitive": bson.M{"$ne": "测试"}, "dataging": bson.M{"$ne": 1}, "_id": bson.M{"$gte": id1, "$lt": id2}}
+			//count2 := fmgo.Count(fs[4], mq)
+			//count3 := fmgo.Count(fs[4], bson.M{"_id": bson.M{"$gte": id1, "$lt": id2}}) //mgo总入库量
+			//report += ",mgo统计" + fmt.Sprint(count2) + ",差值:" + fmt.Sprint(count2-count) + ",mgo总入库量" + fmt.Sprint(count3)
+			//迭代
+			mq := bson.M{"_id": bson.M{"$gte": id1, "$lt": id2}}
+			fd := bson.M{"extracttype": 1, "sensitive": 1, "dataging": 1, "site": 1}
+			sess := fmgo.GetMgoConn()
+			defer fmgo.DestoryMongoConn(sess)
+			count2, count3 := 0, 0 //
+			count4, count5 := 0, 0 //竟品
+			query := sess.DB(fs[3]).C(fs[4]).Find(mq).Select(fd).Iter()
+			for tmp := make(map[string]interface{}); query.Next(tmp); count2++ {
+				if util.ObjToString(tmp["site"]) == "元博网（采购与招标网）" || util.ObjToString(tmp["site"]) == "中国招标与采购网" {
+					count4++
+				}
+				if util.IntAll(tmp["extracttype"]) != -1 && util.ObjToString(tmp["sensitive"]) != "测试" && util.IntAll(tmp["dataging"]) != 1 {
+					count3++
+					if util.ObjToString(tmp["site"]) == "元博网（采购与招标网）" || util.ObjToString(tmp["site"]) == "中国招标与采购网" {
+						count5++
+					}
+				}
+			}
+			report += ",mgo统计" + fmt.Sprint(count3) + ",差值:" + fmt.Sprint(count3-count) + ",mgo总入库量" + fmt.Sprint(count2)
+			report += "<br>" + "竟品统计结果：" + strconv.Itoa(count1) + ",mgo统计" + fmt.Sprint(count5) + ",差值:" + fmt.Sprint(count5-count1) + ",mgo总入库量" + fmt.Sprint(count4)
 		}
 		t.SendMail(report)
 	}

forecast/es/config.json

@@ -0,0 +1,39 @@
+{
+  "mgodb": "192.168.3.207:27092",
+  "dbsize": 12,
+  "dbname": "wjh",
+  "dbcoll": "project_forecast",
+  "uname": "",
+  "upwd": "",
+  "tasktime": 0,
+  "updateid": "",
+  "elastic": {
+    "addr": "http://127.0.0.1:9800",
+    "index": "forecast_v2",
+    "itype": "forecast",
+    "pool": 12,
+    "esfields": [
+      "_id",
+      "area",
+      "city",
+      "nature",
+      "buyer",
+      "buyerclass",
+      "title",
+      "projectname",
+      "projectcode",
+      "spidercode",
+      "stage",
+      "main_project",
+      "top_category",
+      "sub_category",
+      "approvestatus",
+      "href",
+      "jyhref",
+      "publishtime",
+      "yucetime",
+      "yucestarttime",
+      "yuceendtime"
+    ]
+  }
+}

forecast/es/main.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"mongodb"
+	qu "qfw/util"
+	es "qfw/util/elastic"
+)
+
+var (
+	Sysconfig map[string]interface{} //配置文件
+	Mgo       *mongodb.MongodbSim
+	Dbname    string
+	Dbcoll    string
+	Es        *es.Elastic
+	Index     string
+	Itype     string
+	EsFields  []string
+	TaskTime  int
+	UpdateId  string
+)
+var EsSaveCache = make(chan map[string]interface{}, 5000)
+var SP = make(chan bool, 5)
+
+func init() {
+	qu.ReadConfig(&Sysconfig)
+	Dbname = Sysconfig["dbname"].(string)
+	Dbcoll = Sysconfig["dbcoll"].(string)
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: Sysconfig["mgodb"].(string),
+		Size:        qu.IntAllDef(Sysconfig["dbsize"], 5),
+		DbName:      Dbname,
+		UserName:    Sysconfig["uname"].(string),
+		Password:    Sysconfig["upwd"].(string),
+	}
+	Mgo.InitPool()
+	//es
+	econf := Sysconfig["elastic"].(map[string]interface{})
+	Index = econf["index"].(string)
+	Itype = econf["itype"].(string)
+	Es = &es.Elastic{
+		S_esurl: econf["addr"].(string),
+		I_size:  qu.IntAllDef(econf["pool"], 12),
+	}
+	Es.InitElasticSize()
+	EsFields = qu.ObjArrToStringArr(econf["esfields"].([]interface{}))
+	//TaskTime = qu.IntAll(Sysconfig["tasktime"])
+	UpdateId = qu.ObjToString(Sysconfig["updateid"])
+
+}
+
+func main() {
+	go SaveEs()
+
+	go TimeTask()
+	//go SaveAll()
+	//go SaveAdd()
+	ch := make(chan bool, 1)
+	<-ch
+}

forecast/es/task.go

@@ -0,0 +1,199 @@
+package main
+
+import (
+	"github.com/cron"
+	"go.mongodb.org/mongo-driver/bson"
+	"log"
+	"mongodb"
+	"qfw/util"
+	"reflect"
+	"sync"
+	"time"
+)
+
+//定时任务
+func TimeTask() {
+	//go SaveAdd()
+	c := cron.New()
+	cronstr := "0 0 2 * * ?" //每天2点执行
+	//cronstr := "0 */" + fmt.Sprint(TaskTime) + " * * * ?" //每TaskTime小时执行一次
+	err := c.AddFunc(cronstr, func() { SaveAdd() })
+	if err != nil {
+		util.Debug(err)
+		return
+	}
+	c.Start()
+}
+
+// SaveAdd 增量数据
+func SaveAdd() {
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
+	if UpdateId == "" {
+		util.Debug("update id err...")
+		return
+	}
+	q := bson.M{"_id": bson.M{"$gt": mongodb.StringTOBsonId(UpdateId)}}
+	util.Debug("q ---", q)
+	it := sess.DB(Dbname).C(Dbcoll).Find(q).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%5000 == 0 {
+			log.Println("current:", count)
+		}
+		if UpdateId < mongodb.BsonIdToSId(tmp["_id"]) {
+			UpdateId = mongodb.BsonIdToSId(tmp["_id"])
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			esMap := map[string]interface{}{}
+			//生索引字段处理
+			for _, field := range EsFields {
+				if tmp[field] == nil {
+					continue
+				}
+				if field == "buyerclass" {
+					if reflect.TypeOf(tmp["buyerclass"]).String() == "[]interface {}" {
+						esMap["buyerclass"] = util.ObjArrToStringArr(tmp["buyerclass"].([]interface{}))[0]
+					} else {
+						esMap["buyerclass"] = tmp["buyerclass"]
+					}
+				} else {
+					esMap[field] = tmp[field]
+				}
+			}
+			// 处理result
+			if mp, ok := tmp["results"].([]interface{}); ok {
+				var mpArr []map[string]interface{}
+				for _, v := range mp {
+					v1 := v.(map[string]interface{})
+					tmp := make(map[string]interface{})
+					if v1["purchasing"] != nil {
+						tmp["purchasing"] = v1["purchasing"]
+					}
+					if v1["p_projects"] != nil {
+						tmp["p_projects"] = v1["p_projects"]
+					}
+					mpArr = append(mpArr, tmp)
+				}
+				if len(mpArr) > 0 {
+					esMap["results"] = mpArr
+				}
+			}
+			EsSaveCache <- esMap
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("Run Over...Count:", count)
+}
+
+// SaveAll 存量数据生es
+func SaveAll() {
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+	//q := bson.M{"_id": mongodb.StringTOBsonId("6227a7b18f0c45f21aabe120")}
+	it := sess.DB(Dbname).C(Dbcoll).Find(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%20000 == 0 {
+			log.Println("current:", count, tmp["_id"])
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			esMap := map[string]interface{}{}
+			//生索引字段处理
+			for _, field := range EsFields {
+				if tmp[field] == nil {
+					continue
+				}
+				if field == "buyerclass" {
+					if reflect.TypeOf(tmp["buyerclass"]).String() == "[]interface {}" {
+						esMap["buyerclass"] = util.ObjArrToStringArr(tmp["buyerclass"].([]interface{}))[0]
+					} else {
+						esMap["buyerclass"] = tmp["buyerclass"]
+					}
+				} else {
+					esMap[field] = tmp[field]
+				}
+			}
+			// 处理result
+			if mp, ok := tmp["results"].([]interface{}); ok {
+				var mpArr []map[string]interface{}
+				for _, v := range mp {
+					v1 := v.(map[string]interface{})
+					tmp := make(map[string]interface{})
+					if v1["purchasing"] != nil {
+						tmp["purchasing"] = v1["purchasing"]
+					}
+					if v1["p_projects"] != nil {
+						tmp["p_projects"] = v1["p_projects"]
+					}
+					mpArr = append(mpArr, tmp)
+				}
+				if len(mpArr) > 0 {
+					esMap["results"] = mpArr
+				}
+			}
+			EsSaveCache <- esMap
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("Run Over...Count:", count)
+}
+
+// SaveEs 过滤后数据存库
+func SaveEs() {
+	log.Println("Es Save...")
+	arru := make([]map[string]interface{}, 100)
+	indexu := 0
+	for {
+		select {
+		case v := <-EsSaveCache:
+			arru[indexu] = v
+			indexu++
+			if indexu == 100 {
+				SP <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-SP
+					}()
+					Es.BulkSave(Index, Itype, &arru, true)
+				}(arru)
+				arru = make([]map[string]interface{}, 100)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				SP <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-SP
+					}()
+					Es.BulkSave(Index, Itype, &arru, true)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, 100)
+				indexu = 0
+			}
+		}
+	}
+}

forecast/project-yece/main.go

@@ -0,0 +1,205 @@
+package main
+
+import (
+	"mongodb"
+	es "qfw/util/elastic"
+	"time"
+)
+
+var (
+	MongoTool			*mongodb.MongodbSim
+	Es              	*es.Elastic
+
+	updatePool			chan []map[string]interface{}
+	updateSp			chan bool
+	updatePool1			chan []map[string]interface{}
+	updateSp1			chan bool
+	saveSize			int
+	savePool			chan map[string]interface{}
+	saveSp				chan bool
+	savePool1			chan map[string]interface{}
+	saveSp1				chan bool
+
+)
+
+func init() {
+	MongoTool = &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",				// 172.17.4.187:27082,172.17.145.163:27083
+		Size:        10,
+		DbName:      "mixdata",
+		UserName:	 "SJZY_RWESBid_Other",
+		Password: 	 "SJZY@O17t8herB3B",
+	}
+	MongoTool.InitPool()
+
+	Es = &es.Elastic{
+		S_esurl: "http://172.17.145.170:9800",									//http://172.17.145.170:9800
+		I_size:  10,
+	}
+	Es.InitElasticSize()
+
+	saveSize = 200
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp = make(chan bool, 5)
+	updatePool1 = make(chan []map[string]interface{}, 5000)
+	updateSp1 = make(chan bool, 5)
+	savePool = make(chan map[string]interface{}, 5000)
+	saveSp = make(chan bool, 5)
+	savePool1 = make(chan map[string]interface{}, 5000)
+	saveSp1 = make(chan bool, 5)
+}
+
+func main() {
+	go saveMethod()
+	go saveMethod1()
+	go updateMethod()
+	//go updateMethod1()
+
+	//go findEs()
+	//go fcResult()
+	go TimeTask()
+
+	ch := make(chan bool, 1)
+	<-ch
+}
+
+
+func saveMethod() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-savePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MongoTool.SaveBulk("project_forecast_yece_tmp", arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MongoTool.SaveBulk("project_forecast_yece_tmp", arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func saveMethod1() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-savePool1:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveSp1 <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp1
+					}()
+					MongoTool.SaveBulk("project_forecast", arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveSp1 <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp1
+					}()
+					MongoTool.SaveBulk("project_forecast", arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func updateMethod() {
+	arru := make([][]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoTool.UpSertBulk("project_forecast_yece_tmp", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoTool.UpSertBulk("project_forecast_yece_tmp", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func updateMethod1() {
+	arru := make([][]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool1:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				updateSp1 <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp1
+					}()
+					MongoTool.UpSertBulk("project_forecast", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp1 <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp1
+					}()
+					MongoTool.UpSertBulk("project_forecast", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

forecast/project-yece/task.go

@@ -0,0 +1,331 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/cron"
+	"go.mongodb.org/mongo-driver/bson"
+	es "gopkg.in/olivere/elastic.v1"
+	"mongodb"
+	"qfw/util"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+)
+
+var date1 = regexp.MustCompile("20[0-2][0-9][年|\\-\\/|.][0-9]{1,2}[月|\\-|\\/|.][0-9]{1,2}[日]?")
+
+func TimeTask() {
+	c := cron.New()
+	cronstr := "0 0 2 * * ?"  //每天3点执行		临时表数据
+	cronstr1 := "0 0 3 * * ?" //每天4点执行		结果表数据
+	_ = c.AddFunc(cronstr, func() {
+		findEs()
+	})
+	_ = c.AddFunc(cronstr1, func() {
+		fcResult()
+	})
+
+	c.Start()
+}
+
+func findEs() {
+	util.Debug("预测结果时间-------临时表保存数据----------")
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+
+	ch := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
+	//esquery := `{
+	//"query": {
+	//	"bool": {
+	//	  "must": [
+	//		{
+	//		  "terms": {
+	//			"subtype": [
+	//			  "成交",
+	//			  "合同"
+	//			]
+	//		  }
+	//		},
+	//		{
+	//		  "terms": {
+	//			"s_subscopeclass": [
+	//			  "服务采购_法律咨询",
+	//			  "服务采购_会计",
+	//			  "服务采购_物业",
+	//			  "服务采购_审计",
+	//			  "服务采购_安保",
+	//			  "服务采购_仓储物流",
+	//			  "服务采购_广告宣传印刷"
+	//			]
+	//		  }
+	//		}
+	//	  ]
+	//	}
+	//},
+	//"_source": [
+	//	"_id",
+	//	"title",
+	//	"buyer",
+	//	"buyerclass",
+	//	"s_subscopeclass",
+	//	"yuceendtime",
+	//	"area",
+	//	"city",
+	//	"subtype",
+	//	"projectname",
+	//	"purchasing",
+	//	"href",
+	//	"projectcode",
+	//	"publishtime",
+	//	"buyerperson",
+	//	"buyertel"]
+	//}`
+
+	//esquery := `{"query":{"bool":{"must":[{"term":{"id":"6168359c06a9d911e598d573"}}]}}}`
+	currenttime := time.Now().Unix()
+	stime := time.Unix(currenttime, 0).AddDate(0, 0, -1).Unix()
+	esquery := `{"query":{"bool":{"must":[{"range":{"comeintime":{"from":` + fmt.Sprint(stime) + `,"to":` + fmt.Sprint(currenttime) + `}}}],"must_not":[{"constant_score":{"filter":{"missing":{"field":"yuceendtime"}}}}]}}}`
+	util.Debug(esquery)
+	fieldArr := []string{"_id", "title", "buyer", "buyerclass", "s_subscopeclass", "yuceendtime", "area", "city", "subtype",
+		"projectname", "purchasing", "href", "projectcode", "publishtime", "buyerperson", "buyertel", "projectperiod",
+		"project_duration", "project_timeunit", "signaturedate"}
+	escount := Es.Count("bidding", "bidding", esquery)
+	util.Debug("查询总数:", escount)
+	//查询条件类型转换
+	var q es.Query
+	tmpQuery := es.BoolQuery{
+		QueryStrings: esquery,
+	}
+	q = tmpQuery
+	numDocs := 0
+
+	//游标查询，index不支持别名，只能写索引库的名称
+	res, err := client.Scroll("bidding_v1").Query(q).Size(500).Do() //查询一条获取游标
+	if err == nil {
+		scrollId := res.ScrollId
+		for {
+			if scrollId == "" {
+				util.Debug("ScrollId Is Error")
+				break
+			}
+			searchResult, err := client.Scroll("bidding_v1").Size(500).ScrollId(scrollId).Do() //查询
+			if err != nil {
+				if err.Error() == "EOS" { //迭代完毕
+					util.Debug("Es Search Data Over：", err)
+				} else {
+					util.Debug("Es Search Data Error：", err)
+				}
+				break
+			}
+			for _, hit := range searchResult.Hits.Hits {
+				//开始处理数据
+				wg.Add(1)
+				ch <- true
+				go func(tmpHit *es.SearchHit) {
+					defer func() {
+						<-ch
+						wg.Done()
+					}()
+					tmp := make(map[string]interface{})
+					if json.Unmarshal(*tmpHit.Source, &tmp) == nil {
+						save := make(map[string]interface{})
+						for _, v := range fieldArr {
+							if tmp[v] != nil {
+								save[v] = tmp[v]
+							}
+						}
+						//istart, iend := YcTime(save)
+						//if istart > 0 && iend > 0 {
+						//	save["yucestarttime"] = istart
+						//	save["yuceendtime"] = iend
+						//
+						//	upStr := fmt.Sprintf("ctx._source.yuceendtime=%d", iend)
+						//	if tmp["yeceendtime"] != nil {
+						//		upStr += fmt.Sprintf(";ctx._source.yeceendtime=%d", 0)
+						//	}
+						//	updateEs := map[string]string{
+						//		"id": util.ObjToString(tmp["_id"]),
+						//		"updateStr": upStr,
+						//	}
+						//	Es.UpdateOne("bidding", "bidding", updateEs)
+						//}
+						savePool <- save
+						//update := []map[string]interface{}{{
+						//	"_id": save["_id"],
+						//},
+						//	{"$set": save},
+						//}
+						//updatePool <- update
+					}
+				}(hit)
+				numDocs += 1
+				if numDocs%100 == 0 {
+					util.Debug("Current：", numDocs)
+				}
+			}
+			scrollId = searchResult.ScrollId
+		}
+		wg.Wait()
+		util.Debug("over---", numDocs)
+		client.ClearScroll().ScrollId(scrollId).Do() //清理游标
+	}
+}
+
+func YcTime(tmp map[string]interface{}) (int64, int64) {
+	// yucestarttime、yuceendtime
+	yucestarttime := int64(0)
+	yuceendtime := int64(0)
+	// 项目周期中
+	if util.ObjToString(tmp["projectperiod"]) != "" {
+		dateStr := date1.FindStringSubmatch(util.ObjToString(tmp["projectperiod"]))
+		if len(dateStr) == 2 {
+			sdate := FormatDateStr(dateStr[0])
+			edate := FormatDateStr(dateStr[1])
+			if sdate < edate && sdate != 0 && edate != 0 {
+				yucestarttime = sdate
+				yuceendtime = edate
+			}
+		}
+	}
+	if yucestarttime > 0 && yuceendtime > yucestarttime {
+		return yucestarttime, yuceendtime
+	}
+	// 预测开始时间 合同签订日期
+	if util.IntAll(tmp["signaturedate"]) <= 0 {
+		if util.IntAll(tmp["publishtime"]) <= 0 {
+			return 0, 0
+		} else {
+			yucestarttime = util.Int64All(tmp["publishtime"])
+		}
+	} else {
+		yucestarttime = util.Int64All(tmp["signaturedate"])
+	}
+
+	// 预测结束时间
+	if yucestarttime > 0 {
+		if util.IntAll(tmp["project_duration"]) > 0 && util.ObjToString(tmp["project_timeunit"]) != "" {
+			yuceendtime = YcEndTime(yucestarttime, util.IntAll(tmp["project_duration"]), util.ObjToString(tmp["project_timeunit"]))
+			return yucestarttime, yuceendtime
+		}
+	}
+	return 0, 0
+}
+
+func fcResult() {
+	util.Debug("预测结果时间-------结果表迁移数据----------")
+	currenttime := time.Now().Unix()
+	endtime := time.Unix(currenttime, 0).AddDate(0, 3, 0).Unix()
+	q := bson.M{"yuceendtime": bson.M{"$gte": currenttime, "$lt": endtime}, "move": nil}
+	field := bson.M{"project_duration": 0, "project_timeunit": 0, "projectperiod": 0, "s_subscopeclass": 0}
+	util.Debug(q)
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+	result := sess.DB("mixdata").C("project_forecast_yece_tmp").Find(q).Select(field).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); result.Next(&tmp); count++ {
+		if count%200 == 0 {
+			util.Debug("count---", count)
+		}
+		update := []map[string]interface{}{{
+			"_id": tmp["_id"],
+		},
+			{"$set": bson.M{"move": true}},
+		}
+		updatePool <- update
+
+		id := mongodb.BsonIdToSId(tmp["_id"])
+		tmp["infoid"] = id
+		delete(tmp, "_id")
+		tmp["yucetime"] = currenttime
+		tmp["jyhref"] = `/jyapp/article/content/` + util.CommonEncodeArticle("content", id) + `.html`
+		if tmp["buyer"] == nil || tmp["buyerperson"] == nil || tmp["buyertel"] == nil {
+			esq := `{"query":{"bool":{"must":[{"term":{"ids":"` + id + `"}}]}}}`
+			info := Es.Get("project", "project", esq)
+			if len(*info) > 0 {
+				if (*info)[0]["buyer"] != nil {
+					tmp["buyer"] = (*info)[0]["buyer"]
+				}
+				if (*info)[0]["buyerperson"] != nil {
+					tmp["buyerperson"] = (*info)[0]["buyerperson"]
+				}
+				if (*info)[0]["buyertel"] != nil {
+					tmp["buyertel"] = (*info)[0]["buyertel"]
+				}
+			}
+		}
+
+		tpmp := make(map[string]interface{})
+		tpmp["p_rate"] = "60%"
+		if tmp["purchasing"] != nil {
+			tpmp["purchasing"] = tmp["purchasing"]
+			tpmp["purchasing"] = util.ObjToString(tpmp["purchasing"]) + "," + util.ObjToString(tmp["projectname"])
+		} else {
+			tpmp["purchasing"] = tmp["projectname"]
+		}
+		var arr []map[string]interface{}
+		for _, v := range strings.Split(util.ObjToString(tpmp["purchasing"]), ",") {
+			p := make(map[string]interface{})
+			p["p_purchasing"] = v
+			p["p_id"] = id
+			p["p_orther"] = tmp["projectname"]
+			if tmp["buyerperson"] != nil {
+				p["p_person"] = tmp["buyerperson"]
+			}
+			if tmp["buyertel"] != nil {
+				p["p_phone"] = tmp["buyertel"]
+			}
+			arr = append(arr, p)
+		}
+		tpmp["p_projects"] = arr
+		delete(tmp, "buyerperson")
+		delete(tmp, "buyertel")
+		tmp["results"] = append([]map[string]interface{}{}, tpmp)
+
+		savePool1 <- tmp
+		//update1 := []map[string]interface{}{{
+		//	"infoid": id,
+		//},
+		//	{"$set": tmp},
+		//}
+		//updatePool1 <- update1
+	}
+
+	util.Debug("over ---", count)
+}
+
+func YcEndTime(starttime int64, num int, unit string) int64 {
+	yuceendtime := int64(0)
+	if unit == "日历天" || unit == "天" || unit == "日" {
+		yuceendtime = starttime + int64(num*86400)
+	} else if unit == "周" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num*7).Unix()
+	} else if unit == "月" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, num, 0).Unix()
+	} else if unit == "年" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(num, 0, 0).Unix()
+	} else if unit == "工作日" {
+		n := num / 7 * 2
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num+n).Unix()
+	}
+	return yuceendtime
+}
+
+func FormatDateStr(ds string) int64 {
+	ds = strings.Replace(ds, "年", "-", -1)
+	ds = strings.Replace(ds, "月", "-", -1)
+	ds = strings.Replace(ds, "日", "", -1)
+	ds = strings.Replace(ds, "/", "-", -1)
+	ds = strings.Replace(ds, ".", "-", -1)
+	util.Debug(ds)
+
+	location, err := time.ParseInLocation(util.Date_Short_Layout, ds, time.Local)
+	if err != nil {
+		util.Debug(err)
+		return 0
+	} else {
+		return location.Unix()
+	}
+}

fullproject/src_v1/config.json

@@ -6,8 +6,8 @@
     "mongodbPoolSize": 10,
     "mongodbName": "wjh",
 	"hints":"publishtime_1",
-    "extractColl": "bidding",
-    "extractColl1": "bidding",
+    "extractColl": "extract",
+    "extractColl1": "extract",
     "projectColl": "projectset",
     "backupFlag": true,
     "siteColl": "site",
@@ -16,6 +16,11 @@
         "to": "wangjianghan@topnet.net.cn",
         "api": "http://172.17.145.179:19281/_send/_mail"
     },
+    "redis": {
+        "dbname": "",
+        "addr": "",
+        "db": ""
+    },
     "bidding": {
         "addr": "192.168.3.207:27092",
         "dbname": "qfw",

fullproject/src_v1/init.go

@@ -19,7 +19,7 @@ var (
 	MongoTool, MgoBidding, MgoSpider               *MongodbSim            //mongodb连接
 	ExtractColl, ProjectColl, BackupColl, SiteColl string                 //抽取表、项目表、项目快照表、站点表
 	ExtractColl1                                   string
-	Thread                                         int                    //配置项线程数
+	Thread                                         int //配置项线程数
 	BlackList                                      []interface{}
 	BlaskListMap                                   map[string]bool
 )
@@ -57,8 +57,6 @@ func init() {
 		MongodbAddr: Sysconfig["mongodbServers"].(string),
 		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
 		DbName:      Sysconfig["mongodbName"].(string),
-		//UserName:    "root",
-		//Password:    "root",
 	}
 	MongoTool.InitPool()
 	bidding, _ := Sysconfig["bidding"].(map[string]interface{})
@@ -95,6 +93,7 @@ func init() {
 	for _, v := range BlackList {
 		BlaskListMap[util.ObjToString(v)] = true
 	}
+
 	initWinnerRegexp()
 	initBuyerRegexp()
 	initAgencyRegexp()
@@ -344,12 +343,13 @@ type ProjectInfo struct {
 	BidGuarantee       bool                   `json:"bid_guarantee"`           //投标保证金 是否支持包含
 	Qualifies          string                 `json:"qualifies"`               //资质条件
 	EntIdList          []string               `json:"entidlist"`               //企业id
-	score              int
-	comStr             string
-	resVal, pjVal      int
-	InfoFiled          map[string]InfoField `json:"infofield"`    //逻辑处理需要的info字段
-	Budgettag          int                  `json:"budgettag"`    //预算是否有效标记
-	Bidamounttag       int                  `json:"bidamounttag"` //中标金额是否有效标记
+	//FirstCooperation   []string               `json:"first_cooperation"`       //first_cooperation
+	score         int
+	comStr        string
+	resVal, pjVal int
+	InfoFiled     map[string]InfoField `json:"infofield"`    //逻辑处理需要的info字段
+	Budgettag     int                  `json:"budgettag"`    //预算是否有效标记
+	Bidamounttag  int                  `json:"bidamounttag"` //中标金额是否有效标记
 }
 
 //存储部分招标信息字段，业务逻辑处理需要
@@ -462,36 +462,36 @@ func CosineSimilar(srcWords1, dstWords1 string) float64 {
 
 func initWinnerRegexp() {
 	winRegMap := Sysconfig["winner"].(map[string]interface{})
-	preRegexps := winRegMap["pre_regexp"].([]interface{})
-	backRegexps := winRegMap["back_regexp"].([]interface{})
-	backRepRegexps := winRegMap["back_rep_regexp"].([]interface{})
+	//preRegexps := winRegMap["pre_regexp"].([]interface{})
+	//backRegexps := winRegMap["back_regexp"].([]interface{})
+	//backRepRegexps := winRegMap["back_rep_regexp"].([]interface{})
 	backBlack := winRegMap["blacklist"].([]interface{})
-	var winPreRegexps []*regexp.Regexp
-	for _, v := range preRegexps {
-		reg := regexp.MustCompile("^" + v.(string))
-		winPreRegexps = append(winPreRegexps, reg)
-	}
-	PreRegexp["winner"] = winPreRegexps
-
-	var winBackRegexps []*regexp.Regexp
-	for _, v := range backRegexps {
-		reg := regexp.MustCompile(v.(string))
-		winBackRegexps = append(winBackRegexps, reg)
-	}
-	BackRegexp["winner"] = winBackRegexps
-
-	var winBackRepRegexps []RegexpInfo
-	for _, v := range backRepRegexps {
-		reps := strings.Split(v.(string), "#")
-		if len(reps) > 1 {
-			reg := RegexpInfo{
-				regs:   regexp.MustCompile(reps[0]),
-				repstr: reps[1],
-			}
-			winBackRepRegexps = append(winBackRepRegexps, reg)
-		}
-	}
-	BackRepRegexp["winner"] = winBackRepRegexps
+	//var winPreRegexps []*regexp.Regexp
+	//for _, v := range preRegexps {
+	//	reg := regexp.MustCompile("^" + v.(string))
+	//	winPreRegexps = append(winPreRegexps, reg)
+	//}
+	//PreRegexp["winner"] = winPreRegexps
+
+	//var winBackRegexps []*regexp.Regexp
+	//for _, v := range backRegexps {
+	//	reg := regexp.MustCompile(v.(string))
+	//	winBackRegexps = append(winBackRegexps, reg)
+	//}
+	//BackRegexp["winner"] = winBackRegexps
+
+	//var winBackRepRegexps []RegexpInfo
+	//for _, v := range backRepRegexps {
+	//	reps := strings.Split(v.(string), "#")
+	//	if len(reps) > 1 {
+	//		reg := RegexpInfo{
+	//			regs:   regexp.MustCompile(reps[0]),
+	//			repstr: reps[1],
+	//		}
+	//		winBackRepRegexps = append(winBackRepRegexps, reg)
+	//	}
+	//}
+	//BackRepRegexp["winner"] = winBackRepRegexps
 
 	var winBlackRegexps []*regexp.Regexp
 	for _, v := range backBlack {
@@ -503,36 +503,36 @@ func initWinnerRegexp() {
 
 func initBuyerRegexp() {
 	buyRegMap := Sysconfig["buyer"].(map[string]interface{})
-	preRegexps := buyRegMap["pre_regexp"].([]interface{})
-	backRegexps := buyRegMap["back_regexp"].([]interface{})
-	backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
+	//preRegexps := buyRegMap["pre_regexp"].([]interface{})
+	//backRegexps := buyRegMap["back_regexp"].([]interface{})
+	//backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
 	backBlack := buyRegMap["blacklist"].([]interface{})
-	var winPreRegexps []*regexp.Regexp
-	for _, v := range preRegexps {
-		reg := regexp.MustCompile("^" + v.(string))
-		winPreRegexps = append(winPreRegexps, reg)
-	}
-	PreRegexp["buyer"] = winPreRegexps
-
-	var winBackRegexps []*regexp.Regexp
-	for _, v := range backRegexps {
-		reg := regexp.MustCompile(v.(string))
-		winBackRegexps = append(winBackRegexps, reg)
-	}
-	BackRegexp["buyer"] = winBackRegexps
-
-	var winBackRepRegexps []RegexpInfo
-	for _, v := range backRepRegexps {
-		reps := strings.Split(v.(string), "#")
-		if len(reps) > 1 {
-			reg := RegexpInfo{
-				regs:   regexp.MustCompile(reps[0]),
-				repstr: reps[1],
-			}
-			winBackRepRegexps = append(winBackRepRegexps, reg)
-		}
-	}
-	BackRepRegexp["buyer"] = winBackRepRegexps
+	//var winPreRegexps []*regexp.Regexp
+	//for _, v := range preRegexps {
+	//	reg := regexp.MustCompile("^" + v.(string))
+	//	winPreRegexps = append(winPreRegexps, reg)
+	//}
+	//PreRegexp["buyer"] = winPreRegexps
+
+	//var winBackRegexps []*regexp.Regexp
+	//for _, v := range backRegexps {
+	//	reg := regexp.MustCompile(v.(string))
+	//	winBackRegexps = append(winBackRegexps, reg)
+	//}
+	//BackRegexp["buyer"] = winBackRegexps
+
+	//var winBackRepRegexps []RegexpInfo
+	//for _, v := range backRepRegexps {
+	//	reps := strings.Split(v.(string), "#")
+	//	if len(reps) > 1 {
+	//		reg := RegexpInfo{
+	//			regs:   regexp.MustCompile(reps[0]),
+	//			repstr: reps[1],
+	//		}
+	//		winBackRepRegexps = append(winBackRepRegexps, reg)
+	//	}
+	//}
+	//BackRepRegexp["buyer"] = winBackRepRegexps
 
 	var winBlackRegexps []*regexp.Regexp
 	for _, v := range backBlack {
@@ -544,36 +544,36 @@ func initBuyerRegexp() {
 
 func initAgencyRegexp() {
 	buyRegMap := Sysconfig["agency"].(map[string]interface{})
-	preRegexps := buyRegMap["pre_regexp"].([]interface{})
-	backRegexps := buyRegMap["back_regexp"].([]interface{})
-	backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
+	//preRegexps := buyRegMap["pre_regexp"].([]interface{})
+	//backRegexps := buyRegMap["back_regexp"].([]interface{})
+	//backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
 	backBlack := buyRegMap["blacklist"].([]interface{})
-	var winPreRegexps []*regexp.Regexp
-	for _, v := range preRegexps {
-		reg := regexp.MustCompile("^" + v.(string))
-		winPreRegexps = append(winPreRegexps, reg)
-	}
-	PreRegexp["agency"] = winPreRegexps
-
-	var winBackRegexps []*regexp.Regexp
-	for _, v := range backRegexps {
-		reg := regexp.MustCompile(v.(string))
-		winBackRegexps = append(winBackRegexps, reg)
-	}
-	BackRegexp["agency"] = winBackRegexps
-
-	var winBackRepRegexps []RegexpInfo
-	for _, v := range backRepRegexps {
-		reps := strings.Split(v.(string), "#")
-		if len(reps) > 1 {
-			reg := RegexpInfo{
-				regs:   regexp.MustCompile(reps[0]),
-				repstr: reps[1],
-			}
-			winBackRepRegexps = append(winBackRepRegexps, reg)
-		}
-	}
-	BackRepRegexp["agency"] = winBackRepRegexps
+	//var winPreRegexps []*regexp.Regexp
+	//for _, v := range preRegexps {
+	//	reg := regexp.MustCompile("^" + v.(string))
+	//	winPreRegexps = append(winPreRegexps, reg)
+	//}
+	//PreRegexp["agency"] = winPreRegexps
+
+	//var winBackRegexps []*regexp.Regexp
+	//for _, v := range backRegexps {
+	//	reg := regexp.MustCompile(v.(string))
+	//	winBackRegexps = append(winBackRegexps, reg)
+	//}
+	//BackRegexp["agency"] = winBackRegexps
+
+	//var winBackRepRegexps []RegexpInfo
+	//for _, v := range backRepRegexps {
+	//	reps := strings.Split(v.(string), "#")
+	//	if len(reps) > 1 {
+	//		reg := RegexpInfo{
+	//			regs:   regexp.MustCompile(reps[0]),
+	//			repstr: reps[1],
+	//		}
+	//		winBackRepRegexps = append(winBackRepRegexps, reg)
+	//	}
+	//}
+	//BackRepRegexp["agency"] = winBackRepRegexps
 
 	var winBlackRegexps []*regexp.Regexp
 	for _, v := range backBlack {

fullproject/src_v1/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"log"
 	mu "mfw/util"
 	"net"
@@ -9,6 +10,7 @@ import (
 	"os/signal"
 	"qfw/util"
 	"qfw/util/elastic"
+	"qfw/util/redis"
 	"syscall"
 	"time"
 )
@@ -19,9 +21,10 @@ var (
 	SingleClear  = 0
 	toaddr       = []*net.UDPAddr{} //下节点对象
 	ChSign       = make(chan os.Signal)
-	Es 			 *elastic.Elastic
+	Es           *elastic.Elastic
 	Index        string
 	Itype        string
+	RedisCode    string
 
 	sid, eid string //测试使用
 )
@@ -44,6 +47,13 @@ func init() {
 	Index = util.ObjToString(es["index"])
 	Itype = util.ObjToString(es["itype"])
 	Es.InitElasticSize()
+
+	redisCon := Sysconfig["redis"].(map[string]interface{})
+	RedisCode = util.ObjToString(redisCon["dbname"])
+	redis_addr := util.ObjToString(redisCon["addr"])
+	redis_db := util.IntAll(redisCon["db"])
+	//"qyxy_buyer=172.17.4.189:8379"
+	redis.InitRedis1(fmt.Sprintf("%s=%s", RedisCode, redis_addr), redis_db) // 采购单位与中标单位初次合作项目
 }
 
 var queryClose = make(chan bool)
@@ -91,8 +101,8 @@ func main() {
 
 //测试组人员使用
 func mainT() {
-	sid = "6062826adeed5af79ca930c9"
-	eid = "6062826adeed5af79ca930ca"
+	sid = "61ace36c45a326c6c325093e"
+	eid = "61b1738445a326c6c32c29e8"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()
@@ -117,7 +127,6 @@ func mainT() {
 	P_QL.currentType = mapinfo["stype"].(string)
 	P_QL.pici = time.Now().Unix()
 	P_QL.taskQl(mapinfo)
-	//P_QL.taskQuery()
 	time.Sleep(99999 * time.Hour)
 }
 
@@ -176,7 +185,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					P_QL.pici = time.Now().Unix()
 					P_QL.taskUpdatePro(mapInfo)
 				}()
-			case "deleteInfo":	// 删除招标公告
+			case "deleteInfo": // 删除招标公告
 				go func() {
 					defer func() {
 						<-SingleThread

fullproject/src_v1/mgotool.go

@@ -292,4 +292,3 @@ func StringTOBsonId(id string) primitive.ObjectID {
 	objectId, _ := primitive.ObjectIDFromHex(id)
 	return objectId
 }
-

fullproject/src_v1/project.go

@@ -2,10 +2,12 @@ package main
 
 import (
 	"encoding/json"
+	"jy/util"
 	"log"
 	"math"
 	"mongodb"
 	qu "qfw/util"
+	"qfw/util/redis"
 	"sort"
 	"strings"
 	"time"
@@ -95,14 +97,13 @@ func (p *ProjectTask) getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb
 func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{}) {
 	p.findLock.Lock()
 	defer p.findLock.Unlock()
-	// 3.18 isfow=0数据不参与项目合并
+	// 3.18 isfow=0数据不参与项目合并		(1表示正常数据招标流程)
 	code := strings.ReplaceAll(qu.ObjToString(tmp["spidercode"]), " ", "")
 	p.mapSpiderLock.Lock()
 	isflow := p.mapSpider[code]
 	p.mapSpiderLock.Unlock()
 	if isflow == 0 {
 		p.NewProject(tmp, info)
-		//qu.Debug("直接新建项目，", "project id", id)
 		return
 	}
 
@@ -135,7 +136,6 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 		//问题出地LastTime!!!!!
 		diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime)))
 		if diffTime <= p.validTime {
-
 			//代理机构完全不相同，直接新建项目
 			if CheckContain(compareProject.Agency, info.Agency) == 3 {
 				continue
@@ -210,7 +210,6 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 									resArr[0].MPN = append(resArr[0].MPN, info.ProjectName)
 								}
 							}
-
 						} else if k2 < 3 {
 							if resArr[0].ProjectCode == "" {
 								resArr[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
@@ -499,8 +498,8 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	set["ids"] = []string{thisinfo.Id}
 	if thisinfo.TopType == "招标" {
 		//if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
-			set["zbtime"] = tmp["publishtime"]
-			p1.Zbtime = qu.Int64All(tmp["publishtime"])
+		set["zbtime"] = tmp["publishtime"]
+		p1.Zbtime = qu.Int64All(tmp["publishtime"])
 		//}
 	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
 		set["jgtime"] = tmp["publishtime"]
@@ -526,7 +525,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	}
 	//标的物
 	if thisinfo.Purchasing != "" {
-		list := Duplicate(strings.Split(thisinfo.Purchasing, ","))		//标的物 去重 03/03
+		list := Duplicate(strings.Split(thisinfo.Purchasing, ",")) //标的物 去重 03/03
 		p := strings.Join(qu.ObjArrToStringArr(list), ",")
 		set["purchasing"] = p
 		p1.Purchasing = p
@@ -585,7 +584,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	// 资质条件
 	if len(thisinfo.Qualifies) > 0 {
 		var str []string
-		for _, v := range thisinfo.Qualifies{
+		for _, v := range thisinfo.Qualifies {
 			if len(qu.ObjToString(v["key"])) > 0 {
 				if BinarySearch(str, qu.ObjToString(v["key"])) == -1 {
 					str = append(str, qu.ObjToString(v["key"]))
@@ -600,6 +599,11 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	if len(p1.EntIdList) > 0 {
 		set["entidlist"] = p1.EntIdList
 	}
+	// first_cooperation
+	if p1.Buyer != "" && len(thisinfo.Winners) > 0 {
+		FirstCooperation(set, p1.Buyer, thisinfo.Winners, thisinfo.EntIdList)
+	}
+
 	p1.InfoFiled = make(map[string]InfoField)
 	infofield := InfoField{
 		Budget:       thisinfo.Budget,
@@ -632,8 +636,10 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	}
 	if p1.Bidamount > 0 {
 		set["sortprice"] = p1.Bidamount
-	} else if p1.Budget > 0 {
-		set["sortprice"] = p1.Budget
+	} else {
+		if p1.Budget > 0 {
+			set["sortprice"] = p1.Budget
+		}
 	}
 	push := p.PushListInfo(tmp, thisinfo.Id)
 	push["s_winner"] = strings.Join(thisinfo.Winners, ",")
@@ -825,20 +831,24 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	if thisinfo.Area != "全国" {
 		pInfo.Area = thisinfo.Area
 		set["area"] = thisinfo.Area
-		pInfo.City = thisinfo.City
-		set["city"] = thisinfo.City
+		if thisinfo.City != "" {
+			pInfo.City = thisinfo.City
+			set["city"] = thisinfo.City
+		}
 		if thisinfo.District != "" {
 			pInfo.District = thisinfo.District
 			set["district"] = thisinfo.District
 		}
 	}
-	//6--项目名称
-	if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
+	//项目名称
+	//if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
+	if pInfo.ProjectName == "" && thisinfo.ProjectName != "" {
 		pInfo.ProjectName = thisinfo.ProjectName
 		set["projectname"] = thisinfo.ProjectName
 	}
 	//7--项目编号
-	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+	//if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+	if pInfo.ProjectCode == "" && thisinfo.ProjectCode != "" {
 		pInfo.ProjectCode = thisinfo.ProjectCode
 		set["projectcode"] = thisinfo.ProjectCode
 	}
@@ -872,7 +882,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	if thisinfo.ContractCode != "" {
 		if pInfo.ContractCode == "" {
 			set["contractcode"] = thisinfo.ContractCode
-		}else {
+		} else {
 			list := strings.Split(pInfo.ContractCode, ",")
 			if BinarySearch(list, thisinfo.ContractCode) == -1 {
 				list = append(list, thisinfo.ContractCode)
@@ -906,7 +916,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	}
 	if thisinfo.Purchasing != "" {
 		if pInfo.Purchasing == "" {
-			list := Duplicate(strings.Split(thisinfo.Purchasing, ","))		//标的物 去重 03/03
+			list := Duplicate(strings.Split(thisinfo.Purchasing, ",")) //标的物 去重 03/03
 			p := strings.Join(qu.ObjArrToStringArr(list), ",")
 			pInfo.Purchasing = p
 			set["purchasing"] = p
@@ -951,22 +961,37 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		if len(pInfo.Winners) <= 0 {
 			set["winner"] = qu.ObjToString(tmp["winner"])
 		}
-		sort.Strings(pInfo.Winners)
+		//sort.Strings(pInfo.Winners)
 		for _, k := range thisinfo.Winners {
 			if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
 				if BinarySearch(pInfo.Winners, k) != -1 {
 					deleteSlice(pInfo.Winners, k, "")
-					sort.Strings(pInfo.Winners)
+					//sort.Strings(pInfo.Winners)
 				}
 			} else {
 				if BinarySearch(pInfo.Winners, k) == -1 {
 					pInfo.Winners = append(pInfo.Winners, k)
-					sort.Strings(pInfo.Winners)
+					//sort.Strings(pInfo.Winners)
 				}
 			}
 		}
-		set["s_winner"] = strings.Join(pInfo.Winners, ",")
+
+		set["s_winner"] = strings.Join(qu.ObjArrToStringArr(Duplicate(pInfo.Winners)), ",")
+	}
+
+	if len(thisinfo.EntIdList) > 0 {
+		for _, v := range thisinfo.EntIdList {
+			if BinarySearch(pInfo.EntIdList, v) == -1 {
+				pInfo.EntIdList = append(pInfo.EntIdList, v)
+			}
+		}
+		set["entidlist"] = pInfo.EntIdList
+	}
+	// first_cooperation
+	if pInfo.Buyer != "" && len(pInfo.Winners) > 0 {
+		FirstCooperation(set, pInfo.Buyer, pInfo.Winners, pInfo.EntIdList)
 	}
+
 	//项目规模
 	if len(thisinfo.ProjectScale) > 0 {
 		pInfo.ProjectScale = thisinfo.ProjectScale
@@ -985,11 +1010,11 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	//开工日期
 	if thisinfo.ProjectStartDate > 0 {
 		if pInfo.ProjectStartDate > 0 {
-			if pInfo.ProjectStartDate < thisinfo.ProjectStartDate  {
+			if pInfo.ProjectStartDate < thisinfo.ProjectStartDate {
 				pInfo.ProjectStartDate = thisinfo.ProjectStartDate
 				set["project_startdate"] = thisinfo.ProjectStartDate
 			}
-		}else {
+		} else {
 			pInfo.ProjectStartDate = thisinfo.ProjectStartDate
 			set["project_startdate"] = thisinfo.ProjectStartDate
 		}
@@ -1001,7 +1026,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 				pInfo.ProjctCompleteDate = thisinfo.ProjectCompleteDate
 				set["project_completedate"] = thisinfo.ProjectCompleteDate
 			}
-		}else {
+		} else {
 			pInfo.ProjctCompleteDate = thisinfo.ProjectCompleteDate
 			set["project_completedate"] = thisinfo.ProjectCompleteDate
 		}
@@ -1027,7 +1052,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		if len(pInfo.Qualifies) > 0 {
 			str = append(str, strings.Split(pInfo.Qualifies, ",")...)
 		}
-		for _, v := range thisinfo.Qualifies{
+		for _, v := range thisinfo.Qualifies {
 			if len(qu.ObjToString(v["key"])) > 0 {
 				if BinarySearch(str, qu.ObjToString(v["key"])) == -1 {
 					str = append(str, qu.ObjToString(v["key"]))
@@ -1039,14 +1064,6 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 			set["qualifies"] = strings.Join(str, ",")
 		}
 	}
-	if len(thisinfo.EntIdList) > 0 {
-		for _, v := range thisinfo.EntIdList{
-			if BinarySearch(pInfo.EntIdList, v) == -1 {
-				pInfo.EntIdList = append(pInfo.EntIdList, v)
-			}
-		}
-		set["entidlist"] = pInfo.EntIdList
-	}
 
 	if thisinfo.HasPackage { //多包处理
 		set["multipackage"] = 1
@@ -1064,10 +1081,12 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		set["bidamount"] = pInfo.Bidamount
 		set["bidamounttag"] = 0
 	}
-	if pInfo.Bidamount >= pInfo.Budget {
+	if pInfo.Bidamount > 0 {
 		set["sortprice"] = pInfo.Bidamount
-	} else if pInfo.Budget >= pInfo.Bidamount {
-		set["sortprice"] = pInfo.Budget
+	} else {
+		if pInfo.Budget > 0 {
+			set["sortprice"] = pInfo.Budget
+		}
 	}
 
 	infofield := InfoField{
@@ -1418,9 +1437,9 @@ func IsCreatePro(info *Info) (bol bool) {
 		info.SubType == "结果变更" || info.SubType == "其它" {
 		if info.ProjectName == "" && info.ProjectCode == "" {
 			bol = false
-		}else if info.ProjectName == "" && info.Buyer == "" {
+		} else if info.ProjectName == "" && info.Buyer == "" {
 			bol = false
-		}else if info.ProjectCode == "" && info.Buyer == "" {
+		} else if info.ProjectCode == "" && info.Buyer == "" {
 			bol = false
 		}
 	}
@@ -1448,5 +1467,28 @@ func (p *ProjectTask) GetBidTypeAndBidStatus(info *Info) (string, string) {
 		}
 	}
 	return typeStr, statusStr
+}
 
+func FirstCooperation(set map[string]interface{}, b string, winns, entidlist []string) {
+	defer func() {
+		// 处理数组越界异常
+		if r := recover(); r != nil {
+			util.Debug("recover...:", r)
+		}
+	}()
+	pid := mongodb.BsonIdToSId(set["_id"])
+	var eids []string
+	for i, eid := range entidlist {
+		if eid != "-" {
+			text := b + "," + winns[i]
+			ex, _ := redis.Exists(RedisCode, text)
+			if !ex {
+				redis.PutCKV(RedisCode, text, pid)
+				eids = append(eids, eid)
+			}
+		}
+	}
+	if len(eids) > 0 {
+		set["first_cooperation"] = entidlist
+	}
 }

fullproject/src_v1/project_tool.go

@@ -1,5 +1,7 @@
 package main
 
+var DateTimeSelect = []string{"bidopentime", "signaturedate", "project_completedate", "comeintime"}
+
 //项目中的字段
 var FIELDS = []string{
 	"area",
@@ -93,4 +95,3 @@ var bidtype = map[string]string{
 	"竞价": "竞价",
 	"竞谈": "竞谈",
 }
-

fullproject/src_v1/task.go

@@ -26,9 +26,9 @@ import (
 转换成info对象
 **/
 
-var PreRegexp = map[string][]*regexp.Regexp{}
-var BackRegexp = map[string][]*regexp.Regexp{}
-var BackRepRegexp = map[string][]RegexpInfo{}
+//var PreRegexp = map[string][]*regexp.Regexp{}
+//var BackRegexp = map[string][]*regexp.Regexp{}
+//var BackRepRegexp = map[string][]RegexpInfo{}
 var BlackRegexp = map[string][]*regexp.Regexp{}
 
 var (
@@ -89,12 +89,12 @@ type ProjectTask struct {
 	//当前时间
 	currentTime int64
 	//保存长度
-	saveSize   	int
-	pici       	int64
-	validTime  	int64
-	statusTime 	int64
+	saveSize   int
+	pici       int64
+	validTime  int64
+	statusTime int64
 	//结果时间的更新		最近两天的公告不再更新jgtime
-	jgTime		int64
+	jgTime int64
 	//	LockPool     chan *sync.Mutex
 	//	LockPoolLock sync.Mutex
 	//	m1, m23, m4  map[int]int
@@ -124,7 +124,7 @@ func NewPT() *ProjectTask {
 		coll:       ProjectColl,
 		validTime:  int64(util.IntAllDef(Sysconfig["validdays"], 150) * 86400),
 		statusTime: int64(util.IntAllDef(Sysconfig["statusdays"], 15) * 86400),
-		jgTime:		int64(util.IntAllDef(7, 7) * 86400),
+		jgTime:     int64(util.IntAllDef(7, 7) * 86400),
 	}
 	return p
 }
@@ -345,12 +345,12 @@ func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
 	}
 	client := Es.GetEsConn()
 	defer Es.DestoryEsConn(client)
-	esquery := `{"query": {"bool": {"must": [{"match": {"ids": "`+infoid+`"}}]}}}`
+	esquery := `{"query": {"bool": {"must": [{"match": {"ids": "` + infoid + `"}}]}}}`
 	data := Es.Get(Index, Itype, esquery)
 	if len(*data) > 0 {
 		pid := util.ObjToString(((*data)[0])["_id"])
 		p.updateJudge(infoMap, pid)
-	}else {
+	} else {
 		util.Debug("not find project---,", infoid)
 	}
 }
@@ -371,10 +371,10 @@ func (p *ProjectTask) taskUpdatePro(udpInfo map[string]interface{}) {
 		delete(proMap, "reason")
 		updataMap := make(map[string]interface{})
 		modifyInfo := make(map[string]interface{})
-		for k, v := range *updateMap{
+		for k, v := range *updateMap {
 			if strings.Contains(k, "time") {
 				updataMap[k] = util.Int64All(v)
-			}else {
+			} else {
 				updataMap[k] = v
 			}
 			modifyInfo[k] = true
@@ -406,7 +406,7 @@ func (p *ProjectTask) taskUpdatePro(udpInfo map[string]interface{}) {
 			v.P = &pro
 		}
 		p.AllIdsMapLock.Unlock()
-	}else {
+	} else {
 		util.Debug("Not find project---", pid)
 	}
 }
@@ -421,12 +421,12 @@ func (p *ProjectTask) delInfoPro(udpInfo map[string]interface{}) {
 	}
 	client := Es.GetEsConn()
 	defer Es.DestoryEsConn(client)
-	esquery := `{"query": {"bool": {"must": [{"term": {"ids": "`+infoid+`"}}]}}}`
+	esquery := `{"query": {"bool": {"must": [{"term": {"ids": "` + infoid + `"}}]}}}`
 	data := Es.Get(Index, Itype, esquery)
 	if len(*data) > 0 {
 		pid := util.ObjToString(((*data)[0])["_id"])
 		p.delJudge(infoid, pid)
-	}else {
+	} else {
 		util.Debug("not find project---,", infoid)
 	}
 }
@@ -490,7 +490,7 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 						//普通合并
 						p.CommonMerge(tmp, info)
 					}(tmp)
-				case <- over:
+				case <-over:
 					break L
 				}
 			}
@@ -523,20 +523,22 @@ L:
 					if count%20000 == 0 {
 						log.Println("current", count, lastid)
 					}
-				}else {
-					if count%1000 == 0 {
+				} else {
+					if count%2000 == 0 {
 						log.Println("current", count, lastid)
 					}
 				}
 				if util.IntAll(tmp["repeat"]) == 0 {
-					if P_QL.currentType == "ql" {
-						infoPool <- tmp
-					}else if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {
-						infoPool <- tmp
-					}else {
-						util.Debug("增量   dataging == 1 ", tmp["_id"])
+					if util.ObjToString(tmp["toptype"]) != "采购意向" {
+						if P_QL.currentType == "ql" {
+							infoPool <- tmp
+						} else if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {
+							infoPool <- tmp
+						} else {
+							util.Debug("增量   dataging == 1 ", tmp["_id"])
+						}
 					}
-				}else {
+				} else {
 					countRepeat++
 					//if P_QL.currentType == "project" {
 					//	util.Debug("repeat err---", tmp["_id"])
@@ -561,7 +563,7 @@ L:
 func (p *ProjectTask) CommonMerge(tmp map[string]interface{}, info *Info) {
 	if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
 		if jsonData, ok := tmp["jsondata"].(map[string]interface{}); ok {
-			proHref := util.ObjToString(jsonData["projecthref"])
+			proHref := util.ObjToString(jsonData["projecthref"]) // 网站本身发布的公告具有招投标流程，直接参与合并
 			if jsonData != nil && proHref != "" {
 				//projectHref字段合并
 				tmp["projecthref"] = proHref
@@ -602,6 +604,16 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 		return nil
 	}
 
+	// 处理publishtime为空
+	if thisinfo.Publishtime <= 0 {
+		for _, d := range DateTimeSelect {
+			if tmp[d] != nil {
+				thisinfo.Publishtime = util.Int64All(tmp[d])
+				tmp["publishtime"] = tmp[d]
+			}
+		}
+	}
+
 	if len(thisinfo.Topscopeclass) == 0 {
 		thisinfo.Topscopeclass = []string{}
 	}
@@ -631,7 +643,7 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
 		//thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
 		//if thisinfo.ProjectName != "" {
-			thisinfo.pnbval++
+		thisinfo.pnbval++
 		//}
 	}
 
@@ -647,9 +659,7 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 				thisinfo.PTC = StrOrNum.ReplaceAllString(thisinfo.PTC, "")
 			}
 		}
-		if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
-			thisinfo.pnbval++
-		}
+		thisinfo.pnbval++
 	}
 	if thisinfo.ProjectCode == thisinfo.PTC || strings.Index(thisinfo.ProjectCode, thisinfo.PTC) > -1 {
 		thisinfo.PTC = ""
@@ -701,7 +711,6 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	tmp["buyer"] = buyer
 	thisinfo.Buyer = buyer
 
-
 	thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
 	thisinfo.LenPTC = len([]rune(thisinfo.PTC))
 	thisinfo.LenPN = len([]rune(thisinfo.ProjectName))
@@ -742,16 +751,16 @@ func (p *ProjectTask) updateJudge(infoMap map[string]interface{}, pid string) {
 	_, ok := p.AllIdsMap[pid]
 	p.AllIdsMapLock.Unlock()
 	ids := []interface{}(tmpPro["ids"].(primitive.A))
-	index, position := -1, 0		// index 0：第一个，1：中间，2：最后一个   position list中位置
+	index, position := -1, 0 // index 0：第一个，1：中间，2：最后一个   position list中位置
 
 	for i, v := range ids {
 		if util.ObjToString(v) == mongodb.BsonIdToSId(infoMap["_id"]) {
 			position = i
 			if i == 0 {
 				index = 0
-			}else if i == len(ids) - 1 {
+			} else if i == len(ids)-1 {
 				index = 2
-			}else {
+			} else {
 				index = 1
 			}
 		}
@@ -767,10 +776,10 @@ func (p *ProjectTask) updateJudge(infoMap map[string]interface{}, pid string) {
 				p.mapHrefLock.Unlock()
 				if pid == tempId {
 					p.modifyUpdate(pid, index, position, tmpPro, modifyProMap)
-				}else {
-					util.Debug("projecthref data id err---pid=" + pid, "---"+tempId)
+				} else {
+					util.Debug("projecthref data id err---pid="+pid, "---"+tempId)
 				}
-			}else {
+			} else {
 				f := modifyEle(modifyProMap)
 				if f {
 					//合并、修改
@@ -782,7 +791,7 @@ func (p *ProjectTask) updateJudge(infoMap map[string]interface{}, pid string) {
 					p.modifyUpdate(pid, index, position, tmpPro, modifyProMap)
 				}
 			}
-		}else {
+		} else {
 			f := modifyEle(modifyProMap)
 			if f {
 				//合并、修改
@@ -794,7 +803,7 @@ func (p *ProjectTask) updateJudge(infoMap map[string]interface{}, pid string) {
 				p.modifyUpdate(pid, index, position, tmpPro, modifyProMap)
 			}
 		}
-	}else {
+	} else {
 		// 周期外
 		util.Debug("周期外数据直接修改", "----------------------------")
 		p.modifyUpdate(pid, index, position, tmpPro, modifyProMap)
@@ -814,8 +823,8 @@ var Elements = []string{
 }
 
 /**
-	修改的字段
-	修改的字段是否是影响合并流程的要素字段
+修改的字段
+修改的字段是否是影响合并流程的要素字段
 */
 func modifyEle(tmp map[string]interface{}) bool {
 	merge := false
@@ -905,18 +914,18 @@ func ClearRp(tmp []string) []string {
 func QyFilter(name, stype string) string {
 	name = strings.ReplaceAll(name, " ", "")
 
-	preReg := PreRegexp[stype]
-	for _, v := range preReg {
-		name = v.ReplaceAllString(name, "")
-	}
-	backReg := BackRegexp[stype]
-	for _, v := range backReg {
-		name = v.ReplaceAllString(name, "")
-	}
-	backRepReg := BackRepRegexp[stype]
-	for _, v := range backRepReg {
-		name = v.regs.ReplaceAllString(name, v.repstr)
-	}
+	//preReg := PreRegexp[stype]
+	//for _, v := range preReg {
+	//	name = v.ReplaceAllString(name, "")
+	//}
+	//backReg := BackRegexp[stype]
+	//for _, v := range backReg {
+	//	name = v.ReplaceAllString(name, "")
+	//}
+	//backRepReg := BackRepRegexp[stype]
+	//for _, v := range backRepReg {
+	//	name = v.regs.ReplaceAllString(name, v.repstr)
+	//}
 
 	blackReg := BlackRegexp[stype]
 	for _, v := range blackReg {
@@ -934,4 +943,3 @@ func QyFilter(name, stype string) string {
 	}
 	return name
 }
-

fullproject/src_v1/update.go

@@ -16,8 +16,8 @@ import (
 )
 
 /**
-	直接修改保存项目字段信息
- */
+直接修改保存项目字段信息
+*/
 func (p *ProjectTask) modifyUpdate(pid string, index, position int, tmpPro, modifyProMap map[string]interface{}) {
 	// 1-0 备份
 	tmpPro["reason"] = "直接在原有项目上修改字段信息"
@@ -69,7 +69,7 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 			// 修改内存
 			p.modifyMem(tmpPro)
 		}
-	}else if index == 0 {
+	} else if index == 0 {
 		// 招标公告信息在原有项目中位于第1个
 		merge := p.ReMerge(info, tmp, tmpPro)
 		if merge {
@@ -83,8 +83,8 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 			Id := p.AllIdsMap[pInfoId]
 			p.AllIdsMapLock.Unlock()
 			proMem := Id.P
-			flag := true		// 标记是否需要删除原有项目信息
-			for _, v := range proList{
+			flag := true // 标记是否需要删除原有项目信息
+			for _, v := range proList {
 				v1 := v.(map[string]interface{})
 				temp := MongoTool.FindById(ExtractColl, qu.ObjToString(v1["infoid"]))
 				if len(temp) == 0 {
@@ -105,7 +105,7 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 			}
 			if flag {
 				delOldPro(pInfoId)
-			}else {
+			} else {
 				//合并到原有项目中， 不用继续重新合并， 原有项目内部合并即可
 				p.innerMerge(proList, tmpPro)
 				bol := MongoTool.UpdateById(ProjectColl, pInfoId, map[string]interface{}{"$set": tmpPro})
@@ -128,13 +128,13 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 				}
 				p.AllIdsMapLock.Unlock()
 			}
-		}else {
+		} else {
 			//没有合并到新项目中，更新list字段，有条件更新项目外围字段
 			p.modifyUpdate(pInfoId, index, position, tmpPro, modifyMap)
 			// 修改内存
 			p.modifyMem(tmpPro)
 		}
-	}else if index == 1 {
+	} else if index == 1 {
 		// 招标公告信息在原有项目中处于中间某一条
 		merge := p.ReMerge(info, tmp, tmpPro)
 		if merge {
@@ -155,13 +155,13 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 				qu.Debug(string(by))
 				_ = udpclient.WriteUdp(by, mu.OP_TYPE_DATA, toaddr[1])
 			}
-		}else {
+		} else {
 			// 未合并到新项目中
 			p.modifyUpdate(pInfoId, index, position, tmpPro, modifyMap)
 			// 修改内存
 			p.modifyMem(tmpPro)
 		}
-	}else if index == 2 {
+	} else if index == 2 {
 		// 招标公告信息在原有项目中位于最后一条
 		merge := p.ReMerge(info, tmp, tmpPro)
 		if merge {
@@ -184,7 +184,7 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 				qu.Debug(string(by))
 				_ = udpclient.WriteUdp(by, mu.OP_TYPE_DATA, toaddr[1])
 			}
-		}else {
+		} else {
 			// 未合并到新项目中
 			p.modifyUpdate(pInfoId, index, position, tmpPro, modifyMap)
 			// 修改内存
@@ -209,14 +209,14 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 		}
 		return
 	}
-	var index = -1		//0：第一个，1：中间，2：最后一个
+	var index = -1 //0：第一个，1：中间，2：最后一个
 	for i, v := range ids {
 		if qu.ObjToString(v) == infoid {
 			if i == 0 {
 				index = 0
-			}else if i == len(ids) - 1 {
+			} else if i == len(ids)-1 {
 				index = 2
-			}else {
+			} else {
 				index = 1
 			}
 		}
@@ -227,8 +227,8 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 		delete(tmpPro, "reason")
 		proList = deleteSlice1(proList, proList[0])
 		var pro *ProjectInfo
-		flag := true		// 标记是否需要删除原有项目信息
-		for _, v := range proList{
+		flag := true // 标记是否需要删除原有项目信息
+		for _, v := range proList {
 			v1 := v.(map[string]interface{})
 			temp := MongoTool.FindById(ExtractColl, qu.ObjToString(v1["infoid"]))
 			if len(temp) == 0 {
@@ -249,7 +249,7 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 		}
 		if flag {
 			delOldPro(pid)
-		}else {
+		} else {
 			tmpPro, pro = p.innerMerge(proList, tmpPro)
 			bol := MongoTool.UpdateById(ProjectColl, pid, map[string]interface{}{"$set": tmpPro})
 			if bol {
@@ -271,7 +271,7 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 			}
 			p.AllIdsMapLock.Unlock()
 		}
-	}else if index == 1 {
+	} else if index == 1 {
 		tmpPro["reason"] = "删除list中间某一条招标公告信息"
 		backupPro(tmpPro)
 		delete(tmpPro, "reason")
@@ -296,7 +296,7 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 			v.P = pro
 		}
 		p.AllIdsMapLock.Unlock()
-	}else if index == 2 {
+	} else if index == 2 {
 		tmpPro["reason"] = "删除list中最后一条招标公告信息"
 		backupPro(tmpPro)
 		delete(tmpPro, "reason")
@@ -457,7 +457,7 @@ func (p *ProjectTask) innerMerge(infoList []interface{}, tmpPro map[string]inter
 	newP := make(map[string]interface{})
 	newP["_id"] = tmpPro["_id"]
 	var p1 *ProjectInfo
-	for k, m := range infoList{
+	for k, m := range infoList {
 		m1 := m.(map[string]interface{})
 		temp := MongoTool.FindById(ExtractColl, qu.ObjToString(m1["infoid"]))
 		if len(temp) == 0 {
@@ -470,7 +470,7 @@ func (p *ProjectTask) innerMerge(infoList []interface{}, tmpPro map[string]inter
 		tempInfo := ParseInfo(temp)
 		if k == 0 {
 			p1 = p.newPro(temp, newP, tempInfo)
-		}else {
+		} else {
 			p.updateOldProField(p1, tempInfo, newP, temp, m1)
 		}
 	}
@@ -481,7 +481,7 @@ func (p *ProjectTask) innerMerge1(infoList []interface{}, infoid string, tmpPro
 	newP := make(map[string]interface{})
 	newP["_id"] = tmpPro["_id"]
 	var p1 *ProjectInfo
-	for k, m := range infoList{
+	for k, m := range infoList {
 		m1 := m.(map[string]interface{})
 		if m1["infoid"] == infoid {
 			continue
@@ -497,7 +497,7 @@ func (p *ProjectTask) innerMerge1(infoList []interface{}, infoid string, tmpPro
 		tempInfo := ParseInfo(temp)
 		if k == 0 {
 			p1 = p.newPro(temp, newP, tempInfo)
-		}else {
+		} else {
 			p.updateOldProField(p1, tempInfo, newP, temp, m1)
 		}
 	}
@@ -710,10 +710,12 @@ func (p *ProjectTask) updateProFiled(tmp map[string]interface{}, thisinfo *Info,
 		set["bidamount"] = pInfo.Bidamount
 		set["bidamounttag"] = 0
 	}
-	if pInfo.Bidamount >= pInfo.Budget {
+	if pInfo.Bidamount > 0 {
 		set["sortprice"] = pInfo.Bidamount
-	} else if pInfo.Budget >= pInfo.Bidamount {
-		set["sortprice"] = pInfo.Budget
+	} else {
+		if pInfo.Budget > 0 {
+			set["sortprice"] = pInfo.Budget
+		}
 	}
 
 	infofield := InfoField{
@@ -889,6 +891,7 @@ func (p *ProjectTask) newPro(tmp, tmpPro map[string]interface{}, thisinfo *Info)
 	}
 	return &p1
 }
+
 // 合并字段到老项目中
 func (p *ProjectTask) updateOldProField(pInfo *ProjectInfo, thisinfo *Info, tmpPro, tmp, m1 map[string]interface{}) {
 	if len(pInfo.Ids) > 30 {
@@ -1150,7 +1153,7 @@ func UpdateValue(proMap map[string]interface{}, index, position int, modifyMap m
 				updataSet[k] = modifyMap[k]
 				proMap[k] = modifyMap[k]
 				tempMap[k] = modifyMap[k]
-			}else {
+			} else {
 				updataSet[k] = modifyMap[k]
 				proMap[k] = modifyMap[k]
 				tempMap[k] = modifyMap[k]
@@ -1178,7 +1181,7 @@ func UpdateValue(proMap map[string]interface{}, index, position int, modifyMap m
 //备份(快照)
 func backupPro(tmp map[string]interface{}) {
 	tmp1 := make(map[string]interface{})
-	for k, v := range tmp{
+	for k, v := range tmp {
 		tmp1[k] = v
 	}
 	if Sysconfig["backupFlag"].(bool) {
@@ -1191,7 +1194,7 @@ func backupPro(tmp map[string]interface{}) {
 // 删除原有项目数据
 func delOldPro(pid string) {
 	t := MongoTool.Delete(ProjectColl, pid)
-	if t >=0 {
+	if t >= 0 {
 		client := Es.GetEsConn()
 		defer Es.DestoryEsConn(client)
 		Es.DelById(Index, Itype, pid)
@@ -1199,7 +1202,7 @@ func delOldPro(pid string) {
 }
 
 // 修改内存中的数据
-func (p *ProjectTask) modifyMem(tmpPro map[string]interface{})  {
+func (p *ProjectTask) modifyMem(tmpPro map[string]interface{}) {
 	pid := qu.ObjToString(tmpPro["_id"])
 	var pro ProjectInfo
 	err := mapstructure.Decode(tmpPro, &pro)
@@ -1231,4 +1234,4 @@ func (p *ProjectTask) printMemPro(pid string) {
 		qu.Debug("mem pro ----------", *v.P)
 	}
 	p.AllIdsMapLock.Unlock()
-}
+}

projectinfo/src/task.go

@@ -166,12 +166,14 @@ func GetProjectData(sid, eid string) {
 				return
 			}
 			//projectinfo.projecttype 报建、核准类、核准、审批类、审批、null
-			projectinfo := pro["projectinfo"].(map[string]interface{})
-			if projecttype, ok := projectinfo["projecttype"].(string); ok {
-				if !ProjectTypeMap[projecttype] { //有值且不在范围内不进行项目预测
-					return
+			if projectinfo, ok := pro["projectinfo"].(map[string]interface{}); ok {
+				if projecttype, ok := projectinfo["projecttype"].(string); ok {
+					if !ProjectTypeMap[projecttype] { //有值且不在范围内不进行项目预测
+						return
+					}
 				}
 			}
+
 			delete(pro, "projectinfo")
 			//nature
 			nature := qu.ObjToString(pro["nature"])

qyxy/qyxy_change/config.json

@@ -1,9 +1,9 @@
 {
-  "dbServer": "172.17.4.187:27082,172.17.145.163:27083",
-  "dbName": "mixdata",
+  "dbServer": "192.168.3.207:27092",
+  "dbName": "wjh",
   "dbSize": 10,
-  "uname": "SJZY_RWMIX_Other",
-  "upwd": "SJZY@M34I6x7D9ata",
+  "uname": "",
+  "upwd": "",
   "coll_qy": "qyxy_std",
   "coll_change": "qyxy_change",
   "lastId": 0,

qyxy/qyxy_change/main.go

@@ -25,12 +25,12 @@ func init() {
 		MongodbAddr: util.ObjToString(Sysconfig["dbServer"]),
 		Size:        util.IntAll(Sysconfig["dbSize"]),
 		DbName:      dbname,
-		UserName: 	 util.ObjToString(Sysconfig["uname"]),
-		Password: 	 util.ObjToString(Sysconfig["upwd"]),
+		UserName:    util.ObjToString(Sysconfig["uname"]),
+		Password:    util.ObjToString(Sysconfig["upwd"]),
 	}
 	MgoMix.InitPool()
 	Mgo = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.4.181:27001",
+		MongodbAddr: "172.17.4.181:27001", // 172.17.4.181:27001
 		Size:        10,
 		DbName:      "mixdata",
 	}
@@ -61,12 +61,23 @@ func initChangeMap() {
 }
 
 func main() {
-	go SaveData()
+	//go SaveData()
 	//go updateMethod()
 
 	//go TimeTask()
-	//go GetData()
-	go TaskAll()
-	ch := make(chan bool, 1)
-	<-ch
+	go GetData()
+	//go TaskAll()
+	//ch := make(chan bool, 1)
+	//<-ch
+
+	//var id string
+	//flag.StringVar(&id, "id", "", "公司id")
+	//flag.Parse()
+	//if id == "" {
+	//	flag.PrintDefaults()
+	//	log.Println("参数错误.")
+	//} else {
+	//	taskInfo(id)
+	//}
+
 }

qyxy/qyxy_change/task.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"github.com/cron"
 	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
 	"log"
 	"qfw/util"
 	"regexp"
@@ -21,7 +20,7 @@ var updateSp = make(chan bool, 5)
 func TimeTask() {
 	c := cron.New()
 	//cronstr := "0 0 " + fmt.Sprint(TaskTime) + " * * ?"			//每天TaskTime跑一次
-	cronstrPa := "0 0 15 ? * WED"								//凭安增量数据每周二跑一次
+	cronstrPa := "0 0 15 ? * WED" //凭安增量数据每周二跑一次
 	_ = c.AddFunc(cronstrPa, func() { GetData() })
 	c.Start()
 }
@@ -35,14 +34,15 @@ func GetData() {
 	wg := &sync.WaitGroup{}
 
 	q := bson.M{"_id": bson.M{"$gt": lastId}}
-	//q := bson.M{"_id": 262454280}
+	//q := bson.M{"_id": 368454367}
+	var zid int
 	it := sess.DB("mixdata").C("company_change").Find(q).Select(nil).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
-		lastId = util.Int64All(tmp["_id"])
 		if count%20000 == 0 {
 			log.Println("current:", count)
 		}
+		zid = util.IntAll(tmp["_id"])
 		pool <- true
 		wg.Add(1)
 		go func(tmp map[string]interface{}) {
@@ -54,23 +54,23 @@ func GetData() {
 			query := bson.M{"company_id": tmp["company_id"]}
 			info, b := MgoMix.FindOneByField(CollSave, query, bson.M{"changes": 1})
 			if b && len(*info) > 0 {
-				update := make(map[string]interface{})
-				item := make(map[string]interface{})
-				item["change_field"] = tmp["change_field"]
-				item["content_before"] = tmp["content_before"]
-				item["content_after"] = tmp["content_after"]
-				item["change_date"] = tmp["change_date"]
-				setMark(item)					//change_name_new
-				//update["changes"] = changes
-				update["update_time"] = currentTime
-				saveInfo := []map[string]interface{}{
-					{"company_id": tmp["company_id"]},
-					{"$set": update, "$push": item},
+				if util.ObjToString(tmp["_operation_type"]) == "insert" {
+					update := make(map[string]interface{})
+					item := make(map[string]interface{})
+					item["change_field"] = tmp["change_field"]
+					item["content_before"] = tmp["content_before"]
+					item["content_after"] = tmp["content_after"]
+					item["change_date"] = tmp["change_date"]
+					setMark(item) //change_name_new
+					//update["changes"] = changes
+					update["update_time"] = currentTime
+					saveInfo := map[string]interface{}{"$set": update, "$push": map[string]interface{}{"changes": item}}
+					//updatePool <- saveInfo
+					MgoMix.UpdateById(CollSave, tmp["company_id"], saveInfo)
 				}
-				updatePool <- saveInfo
-			}else {
-				query := bson.M{"company_id": tmp["company_id"]}
-				qyxy, b1 := MgoMix.FindOneByField("qyxy_std", query, bson.M{"company_name": 1})
+			} else {
+				query := bson.M{"_id": tmp["company_id"]}
+				qyxy, b1 := MgoMix.FindOne("qyxy_std", query)
 				if b1 && len(*qyxy) > 0 {
 					save := make(map[string]interface{})
 					var changes []map[string]interface{}
@@ -79,24 +79,23 @@ func GetData() {
 					item["content_before"] = tmp["content_before"]
 					item["content_after"] = tmp["content_after"]
 					item["change_date"] = tmp["change_date"]
-					setMark(item)					//change_name_new
+					setMark(item) //change_name_new
 					changes = append(changes, item)
 
 					save["company_name"] = (*qyxy)["company_name"]
+					save["company_id"] = (*qyxy)["_id"]
 					save["changes"] = changes
 					save["create_time"] = currentTime
 					save["update_time"] = currentTime
-					save["_id"] = primitive.NewObjectID()
-					saveInfo := []map[string]interface{}{
-						{"company_id": tmp["company_id"]},
-						{"$set": save},
-					}
-					updatePool <- saveInfo
+					//save["_id"] = primitive.NewObjectID()
+					saveInfo := map[string]interface{}{"$set": save}
+					//updatePool <- saveInfo
+					MgoMix.Save(CollSave, saveInfo)
 				}
 			}
 		}(tmp)
 	}
-	util.Debug("over---", count)
+	util.Debug("over---", count, zid)
 }
 
 func setMark(tmp map[string]interface{}) {
@@ -116,17 +115,17 @@ func setMark(tmp map[string]interface{}) {
 func clearRepeat(list []interface{}) []interface{} {
 	var tmp []interface{}
 	if len(list) > 1 {
-		for k, v := range list{
+		for k, v := range list {
 			if k < len(list)-1 {
 				if fmt.Sprint(list[k]) != fmt.Sprint(list[k+1]) {
 					tmp = append(tmp, v)
 				}
-			}else {
+			} else {
 				tmp = append(tmp, v)
 			}
 		}
 		return tmp
-	}else {
+	} else {
 		return list
 	}
 }
@@ -167,16 +166,39 @@ func TaskAll() {
 	}
 }
 
+func taskInfo(id string) {
+	update := make(map[string]interface{})
+	q := map[string]interface{}{"company_id": id}
+	info, _ := Mgo.Find("company_change", q, nil, nil, false, -1, -1)
+	if len(*info) > 0 {
+		var changes []map[string]interface{}
+		currentTime := time.Now().Unix()
+		for _, v := range *info {
+			item := make(map[string]interface{})
+			item["change_field"] = v["change_field"]
+			item["content_before"] = v["content_before"]
+			item["content_after"] = v["content_after"]
+			item["change_date"] = v["change_date"]
+			setMark(item) //change_name_new
+			changes = append(changes, item)
+		}
+		update["changes"] = changes
+		update["update_time"] = currentTime
+	}
+	util.Debug(update)
+	MgoMix.Update("qyxy_change", q, map[string]interface{}{"$set": update}, false, false)
+}
+
 func disposeFuc(maps []map[string]interface{}, tmp map[string]interface{}) {
 	var changes []map[string]interface{}
 	currentTime := time.Now().Unix()
-	for _, v := range maps{
+	for _, v := range maps {
 		item := make(map[string]interface{})
 		item["change_field"] = v["change_field"]
 		item["content_before"] = v["content_before"]
 		item["content_after"] = v["content_after"]
 		item["change_date"] = v["change_date"]
-		setMark(item)					//change_name_new
+		setMark(item) //change_name_new
 		changes = append(changes, item)
 	}
 	tmp["changes"] = changes
@@ -222,7 +244,6 @@ func SaveData() {
 	}
 }
 
-
 func updateMethod() {
 	arru := make([][]map[string]interface{}, 200)
 	indexu := 0
@@ -237,6 +258,7 @@ func updateMethod() {
 					defer func() {
 						<-updateSp
 					}()
+
 					MgoMix.UpSertBulk(CollSave, arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 200)
@@ -257,6 +279,3 @@ func updateMethod() {
 		}
 	}
 }
-
-
-

qyxy/qyxy_es/config.json

@@ -1,21 +1,22 @@
 {
-  "mgodb": "172.17.4.187:27082,172.17.145.163:27083",
+  "mgodb": "192.168.3.207:27092",
   "dbsize": 12,
-  "dbname": "mixdata",
-  "dbcoll": "qyxy_std",
+  "dbname": "wjh",
+  "dbcoll": "qyxy_oprd",
   "uname": "SJZY_RWMIX_Other",
   "upwd": "SJZY@M34I6x7D9ata",
   "tasktime": 0,
-  "updatetime": 1631771072,
+  "updatetime": 0,
   "elastic": {
-    "addr": "http://172.17.145.170:9800",
-    "index": "qyxy_v1",
+    "addr": "http://192.168.3.206:9800",
+    "index": "oprd_qyxy_v1",
     "itype": "qyxy",
     "pool": 12,
     "esfields": [
       "_id",
       "company_name",
       "history_name",
+      "credit_no",
       "org_code",
       "tax_code",
       "company_code",
@@ -52,7 +53,8 @@
       "company_shortname",
       "website_url",
       "partners",
-      "employees"
+      "employees",
+      "tag_business"
     ]
   }
 }

qyxy/qyxy_es/main.go

@@ -7,16 +7,16 @@ import (
 )
 
 var (
-	Sysconfig        map[string]interface{} //配置文件
-	Mgo              *mongodb.MongodbSim
-	Dbname           string
-	Dbcoll           string
-	Es               *es.Elastic
-	Index            string
-	Itype            string
-	EsFields         []string
-	TaskTime         int
-	Updatetime       int64
+	Sysconfig  map[string]interface{} //配置文件
+	Mgo        *mongodb.MongodbSim
+	Dbname     string
+	Dbcoll     string
+	Es         *es.Elastic
+	Index      string
+	Itype      string
+	EsFields   []string
+	TaskTime   int
+	Updatetime int64
 )
 var EsSaveCache = make(chan map[string]interface{}, 5000)
 var SP = make(chan bool, 5)
@@ -29,8 +29,8 @@ func init() {
 		MongodbAddr: Sysconfig["mgodb"].(string),
 		Size:        qu.IntAllDef(Sysconfig["dbsize"], 5),
 		DbName:      Dbname,
-		UserName: 	 Sysconfig["uname"].(string),
-		Password: 	 Sysconfig["upwd"].(string),
+		UserName:    Sysconfig["uname"].(string),
+		Password:    Sysconfig["upwd"].(string),
 	}
 	Mgo.InitPool()
 	//es

qyxy/qyxy_es/task.go

@@ -13,20 +13,20 @@ import (
 )
 
 var (
-	partner = []string{"identify_no", "stock_type", "stock_name", "identify_type", "stock_capital", "stock_realcapital"}
+	partner  = []string{"identify_no", "stock_type", "stock_name", "identify_type", "stock_capital", "stock_realcapital"}
 	employee = []string{"employee_name", "position"}
 
 	TypeMap = map[string]string{
 		"采购单位": "1",
-		"投标企业": "2",
+		"中标单位": "2", // 投标企业
 		"代理机构": "3",
-		"厂商": "4",
+		"厂商":   "4",
 	}
 	TypeMap1 = map[string]string{
 		"固定电话": "1",
-		"手机号": "2",
-		"邮箱": "3",
-		"不存在": "4",
+		"手机号":  "2",
+		"邮箱":   "3",
+		"不存在":  "4",
 	}
 )
 
@@ -38,7 +38,7 @@ func TimeTask() {
 	err := c.AddFunc(cronstr, func() { StdAdd() })
 	if err != nil {
 		util.Debug(err)
-		return 
+		return
 	}
 	c.Start()
 }
@@ -53,10 +53,11 @@ func StdAdd() {
 	wg := &sync.WaitGroup{}
 	//q := bson.M{"_id": "affe29f8d061f3faa4170cafba41f316"}
 	q := bson.M{"updatetime": bson.M{"$gt": Updatetime}}
+	util.Debug(q)
 	it := sess.DB(Dbname).C(Dbcoll).Find(q).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
-		if count%5000 == 0 {
+		if count%10000 == 0 {
 			log.Println("current:", count)
 		}
 		if util.IntAll(tmp["use_flag"]) > 5 {
@@ -78,57 +79,56 @@ func StdAdd() {
 				if field == "company_name" {
 					esMap[field] = tmp["company_name"]
 					esMap["name"] = tmp["company_name"]
-				}else if field == "history_name" {
+				} else if field == "history_name" {
 					var nameArr []string
-					for _, v := range strings.Split(util.ObjToString(esMap["history_name"]), "；") {
-						if v != "" {
-							nameArr = append(nameArr, v)
-						}
+					names := util.ObjToString(tmp["history_name"])
+					if strings.Contains(names, ",") {
+						nameArr = append(nameArr, strings.Split(names, ",")...)
 					}
 					if len(nameArr) > 0 {
 						esMap["history_name"] = nameArr
 					}
-				}else if field == "establish_date" {
+				} else if field == "establish_date" {
 					// 成立日期修改成时间戳
 					location, err := time.ParseInLocation(util.Date_Short_Layout, util.ObjToString(tmp["establish_date"]), time.Local)
 					if err != nil {
 						util.Debug(err)
-					}else {
+					} else {
 						esMap["establish_date"] = location.Unix()
 					}
-				}else if field == "lastupdatetime" {
+				} else if field == "lastupdatetime" {
 					esMap["lastupdatetime"] = tmp["update_time_msql"]
 				} else if field == "bid_projectname" {
 					if pname, ok := tmp["bid_projectname"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(pname)
 						esMap["bid_projectname"] = strings.Join(p1, ",")
 					}
-				}else if field == "bid_purchasing" {
+				} else if field == "bid_purchasing" {
 					if pur, ok := tmp["bid_purchasing"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(pur)
 						esMap["bid_purchasing"] = strings.Join(p1, ",")
 					}
-				}else if field == "bid_area" {
+				} else if field == "bid_area" {
 					if areas, ok := tmp["bid_area"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(areas)
 						esMap["bid_area"] = strings.Join(p1, ",")
 					}
-				}else if field == "partners" {
+				} else if field == "partners" {
 					if ps, ok := tmp["partners"].([]interface{}); ok {
 						var parr []map[string]interface{}
-						for _, v := range ps{
+						for _, v := range ps {
 							p := make(map[string]interface{})
 							v1 := v.(map[string]interface{})
-							for _, field := range partner{
+							for _, field := range partner {
 								if v1[field] == nil {
 									continue
 								}
 								if field == "stock_capital" || field == "stock_realcapital" {
 									if v, err := strconv.ParseFloat(util.ObjToString(v1[field]), 64); err == nil {
-										v, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", v), 64)	//保留小数点两位
+										v, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", v), 64) //保留小数点两位
 										p[field] = v
 									}
-								}else {
+								} else {
 									p[field] = v1[field]
 								}
 							}
@@ -140,16 +140,16 @@ func StdAdd() {
 							esMap[field] = parr
 						}
 					}
-				}else if field == "employees" {
+				} else if field == "employees" {
 					if ps, ok := tmp["employees"].([]interface{}); ok {
 						var parr []map[string]interface{}
-						for _, v := range ps{
+						for _, v := range ps {
 							p := make(map[string]interface{})
 							v1 := v.(map[string]interface{})
-							for _, field := range employee{
+							for _, field := range employee {
 								if v1[field] == nil {
 									continue
-								}else {
+								} else {
 									p[field] = v1[field]
 								}
 							}
@@ -161,23 +161,23 @@ func StdAdd() {
 							esMap[field] = parr
 						}
 					}
-				}else if field == "bid_unittype" {
+				} else if field == "bid_unittype" {
 					if t2, ok := tmp["bid_unittype"].([]interface{}); ok {
 						var arr []string
-						for _, v := range util.ObjArrToStringArr(t2){
+						for _, v := range util.ObjArrToStringArr(t2) {
 							arr = append(arr, TypeMap[v])
 						}
 						esMap["bid_unittype"] = strings.Join(arr, ",")
 					}
-				}else if field == "bid_contracttype" {
+				} else if field == "bid_contracttype" {
 					if t2, ok := tmp["bid_contracttype"].([]interface{}); ok {
 						var arr []string
-						for _, v := range util.ObjArrToStringArr(t2){
+						for _, v := range util.ObjArrToStringArr(t2) {
 							arr = append(arr, TypeMap1[v])
 						}
 						esMap["bid_contracttype"] = strings.Join(arr, ",")
 					}
-				}else {
+				} else {
 					esMap[field] = tmp[field]
 				}
 			}
@@ -186,25 +186,25 @@ func StdAdd() {
 			if company_type == "个体工商户" {
 				if len([]rune(company_name)) >= 5 {
 					esMap["company_type_int"] = 31
-				}else {
+				} else {
 					esMap["company_type_int"] = 32
 				}
-			}else if company_type == "其他" || company_type == "" {
+			} else if company_type == "其他" || company_type == "" {
 				if len([]rune(company_name)) >= 4 {
 					esMap["company_type_int"] = 21
-				}else {
+				} else {
 					esMap["company_type_int"] = 22
 				}
-			}else {
+			} else {
 				if company_type == "内资分公司" {
 					esMap["company_type_int"] = 12
-				}else if len([]rune(company_name)) >= 4 {
+				} else if len([]rune(company_name)) >= 4 {
 					esMap["company_type_int"] = 11
-				}else {
+				} else {
 					esMap["company_type_int"] = 13
 				}
 			}
-			EsSaveCache <- esMap		// 保存es
+			EsSaveCache <- esMap // 保存es
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
@@ -243,9 +243,9 @@ func StdAll() {
 				if field == "company_name" {
 					esMap[field] = tmp["company_name"]
 					esMap["name"] = tmp["company_name"]
-				}else if field == "history_name" {
+				} else if field == "history_name" {
 					var nameArr []string
-					for _, v := range strings.Split(util.ObjToString(esMap["history_name"]), "；") {
+					for _, v := range strings.Split(util.ObjToString(tmp["history_name"]), "；") {
 						if v != "" {
 							nameArr = append(nameArr, v)
 						}
@@ -253,47 +253,51 @@ func StdAll() {
 					if len(nameArr) > 0 {
 						esMap["history_name"] = nameArr
 					}
-				}else if field == "establish_date" {
+				} else if field == "establish_date" {
 					// 成立日期修改成时间戳
 					location, err := time.ParseInLocation(util.Date_Short_Layout, util.ObjToString(tmp["establish_date"]), time.Local)
 					if err != nil {
 						util.Debug(err)
-					}else {
+					} else {
 						esMap["establish_date"] = location.Unix()
 					}
-				}else if field == "lastupdatetime" {
+				} else if field == "lastupdatetime" {
 					esMap["lastupdatetime"] = tmp["update_time_msql"]
 				} else if field == "bid_projectname" {
 					if pname, ok := tmp["bid_projectname"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(pname)
 						esMap["bid_projectname"] = strings.Join(p1, ",")
 					}
-				}else if field == "bid_purchasing" {
+				} else if field == "bid_purchasing" {
 					if pur, ok := tmp["bid_purchasing"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(pur)
 						esMap["bid_purchasing"] = strings.Join(p1, ",")
 					}
-				}else if field == "bid_area" {
+				} else if field == "bid_area" {
 					if areas, ok := tmp["bid_area"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(areas)
 						esMap["bid_area"] = strings.Join(p1, ",")
 					}
-				}else if field == "partners" {
+				} else if field == "partners" {
 					if ps, ok := tmp["partners"].([]interface{}); ok {
 						var parr []map[string]interface{}
-						for _, v := range ps{
+						for _, v := range ps {
 							p := make(map[string]interface{})
 							v1 := v.(map[string]interface{})
-							for _, field := range partner{
+							for _, field := range partner {
 								if v1[field] == nil {
 									continue
 								}
 								if field == "stock_capital" || field == "stock_realcapital" {
-									if v, err := strconv.ParseFloat(util.ObjToString(v1[field]), 64); err == nil {
-										v, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", v), 64)	//保留小数点两位
+									text := util.ObjToString(v1[field])
+									if strings.Contains(text, "万元") {
+										text = strings.Replace(text, "万元", "", -1)
+									}
+									if v, err := strconv.ParseFloat(text, 64); err == nil {
+										v, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", v), 64) //保留小数点两位
 										p[field] = v
 									}
-								}else {
+								} else {
 									p[field] = v1[field]
 								}
 							}
@@ -305,16 +309,16 @@ func StdAll() {
 							esMap[field] = parr
 						}
 					}
-				}else if field == "employees" {
+				} else if field == "employees" {
 					if ps, ok := tmp["employees"].([]interface{}); ok {
 						var parr []map[string]interface{}
-						for _, v := range ps{
+						for _, v := range ps {
 							p := make(map[string]interface{})
 							v1 := v.(map[string]interface{})
-							for _, field := range employee{
+							for _, field := range employee {
 								if v1[field] == nil {
 									continue
-								}else {
+								} else {
 									p[field] = v1[field]
 								}
 							}
@@ -326,23 +330,23 @@ func StdAll() {
 							esMap[field] = parr
 						}
 					}
-				}else if field == "bid_unittype" {
+				} else if field == "bid_unittype" {
 					if t2, ok := tmp["bid_unittype"].([]interface{}); ok {
 						var arr []string
-						for _, v := range util.ObjArrToStringArr(t2){
+						for _, v := range util.ObjArrToStringArr(t2) {
 							arr = append(arr, TypeMap[v])
 						}
 						esMap["bid_unittype"] = strings.Join(arr, ",")
 					}
-				}else if field == "bid_contracttype" {
+				} else if field == "bid_contracttype" {
 					if t2, ok := tmp["bid_contracttype"].([]interface{}); ok {
 						var arr []string
-						for _, v := range util.ObjArrToStringArr(t2){
+						for _, v := range util.ObjArrToStringArr(t2) {
 							arr = append(arr, TypeMap1[v])
 						}
 						esMap["bid_contracttype"] = strings.Join(arr, ",")
 					}
-				}else {
+				} else {
 					esMap[field] = tmp[field]
 				}
 			}
@@ -351,25 +355,25 @@ func StdAll() {
 			if company_type == "个体工商户" {
 				if len([]rune(company_name)) >= 5 {
 					esMap["company_type_int"] = 31
-				}else {
+				} else {
 					esMap["company_type_int"] = 32
 				}
-			}else if company_type == "其他" || company_type == "" {
+			} else if company_type == "其他" || company_type == "" {
 				if len([]rune(company_name)) >= 4 {
 					esMap["company_type_int"] = 21
-				}else {
+				} else {
 					esMap["company_type_int"] = 22
 				}
-			}else {
+			} else {
 				if company_type == "内资分公司" {
 					esMap["company_type_int"] = 12
-				}else if len([]rune(company_name)) >= 4 {
+				} else if len([]rune(company_name)) >= 4 {
 					esMap["company_type_int"] = 11
-				}else {
+				} else {
 					esMap["company_type_int"] = 13
 				}
 			}
-			EsSaveCache <- esMap		// 保存es
+			EsSaveCache <- esMap // 保存es
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
@@ -412,4 +416,4 @@ func SaveEs() {
 			}
 		}
 	}
-}
+}

qyxy/qyxy_inc_data/main.go

@@ -10,29 +10,34 @@ import (
 	"io/ioutil"
 	"log"
 	"mongodb"
+	"net/http"
+	"net/url"
 	"os"
 	"qfw/util"
+	"strconv"
+	"strings"
 	"time"
 )
 
 var (
-	MongoTool			*mongodb.MongodbSim
+	MongoTool *mongodb.MongodbSim
 
-	updatePool			chan []map[string]interface{}
-	updateSp			chan bool
-	saveSize			int
+	updatePool chan []map[string]interface{}
+	updateSp   chan bool
+	saveSize   int
 
-	CurrentColl			string
+	CurrentColl string
 
-	SkipCollName		string
-
-	saveArr				[][]map[string]interface{}
+	SkipCollName string
+	sendMsg      string
+	collCount    int
 
+	saveArr [][]map[string]interface{}
 )
 
 func init() {
 	MongoTool = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.4.181:27002",
+		MongodbAddr: "172.17.4.181:27001",
 		Size:        10,
 		DbName:      "mixdata",
 	}
@@ -44,7 +49,6 @@ func init() {
 
 	SkipCollName = "annual_report_base,annual_report_website,company_base,company_employee,company_history_name,company_partner"
 
-
 }
 
 func main() {
@@ -60,7 +64,6 @@ func main() {
 	// /nas/qyxy/ftp/ftpuser/upload/20210811/
 	task(path)
 
-
 	//sess := MongoTool.GetMgoConn()
 	//defer MongoTool.DestoryMongoConn(sess)
 	//
@@ -102,10 +105,11 @@ func task(path string) {
 			//	continue
 			//}
 			CurrentColl = f.Name()
+			collCount = 0
 			util.Debug("collection name：---", f.Name())
 			subPath := path + f.Name() + "/"
 			subFiles, _ := ioutil.ReadDir(subPath)
-			for _, s := range subFiles{
+			for _, s := range subFiles {
 				if s.IsDir() {
 					taskinfo(subPath + s.Name())
 				}
@@ -115,8 +119,10 @@ func task(path string) {
 				MongoTool.UpSertBulk(CurrentColl, tmps...)
 				saveArr = [][]map[string]interface{}{}
 			}
+			sendMsg += f.Name() + "：" + strconv.Itoa(collCount) + "；"
 		}
 	}
+	SendMail(sendMsg)
 }
 
 func taskinfo(path string) {
@@ -143,29 +149,29 @@ func taskinfo(path string) {
 	bfRd := bufio.NewReader(gr)
 	for {
 		line, err := bfRd.ReadBytes('\n')
-		count  = hookfn(line, count)
+		count = hookfn(line, count)
 		if err != nil {
 			if err == io.EOF {
 				fmt.Println("read gzip data finish! ")
 				break
-			}else {
+			} else {
 				fmt.Println("[read gzip data err]: ", err)
 			}
 		}
-		if count % 5000 == 0 {
+		if count%5000 == 0 {
 			util.Debug("current exc---", file, count)
 		}
 	}
 }
 
-
 func hookfn(line []byte, count int) int {
 	tmp := make(map[string]interface{})
-	err:= json.Unmarshal(line, &tmp)
+	err := json.Unmarshal(line, &tmp)
 	if err != nil {
 		util.Debug("err---", err)
 	}
-	count ++
+	count++
+	collCount++
 	tmp["_id"] = util.IntAll(tmp["id"])
 	tmp["id"] = fmt.Sprintf("%d", util.IntAll(tmp["id"]))
 
@@ -217,3 +223,27 @@ func updateMethod() {
 		}
 	}
 }
+
+func SendMail(report string) {
+	api := "http://172.17.145.179:19281/_send/_mail"
+	to := "wangjianghan@topnet.net.cn"
+	_, _ = http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, to, "凭安数据推送！", report))
+}
+
+func SendMsg(msg string) {
+	v := url.Values{}
+	v.Set("token", "9d1554434aed46018d382737fff25288")
+	v.Set("title", "凭安数据推送")
+	v.Set("content", msg)
+	v.Set("template", "json")
+	body := ioutil.NopCloser(strings.NewReader(v.Encode()))
+	client := &http.Client{}
+	req, _ := http.NewRequest("POST", "http://pushplus.hxtrip.com/send", body)
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
+	resp, err := client.Do(req)
+	defer func(Body io.ReadCloser) {
+		_ = Body.Close()
+	}(resp.Body)
+	data, _ := ioutil.ReadAll(resp.Body)
+	fmt.Println(string(data), err)
+}

qyxy/qyxy_inc_tmp/main.go

@@ -13,29 +13,30 @@ import (
 	"mongodb"
 	"os"
 	"qfw/util"
+	"strings"
 	"time"
 )
 
 var (
-	MongoTool			*mongodb.MongodbSim
+	MongoTool *mongodb.MongodbSim
 
-	updatePool			chan []map[string]interface{}
-	updateSp			chan bool
-	savePool			chan map[string]interface{}
-	saveSp				chan bool
-	saveSize			int
-
-	CollArr				[]string
-	CurrentColl			string
+	updatePool chan []map[string]interface{}
+	updateSp   chan bool
+	savePool   chan map[string]interface{}
+	saveSp     chan bool
+	saveSize   int
 
+	saveArr [][]map[string]interface{}
 
+	CollArr     []string
+	CurrentColl string
 )
 
 func init() {
 	MongoTool = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.4.181:27001",
+		MongodbAddr: "192.168.3.207:29098",
 		Size:        10,
-		DbName:      "mixdata",
+		DbName:      "qfw",
 	}
 	MongoTool.InitPool()
 
@@ -49,7 +50,7 @@ func init() {
 
 }
 
-func main() {
+func main1() {
 
 	var path string
 	flag.StringVar(&path, "p", "", "路径 /nas/qyxy/ftp/ftpuser/upload/20210811/")
@@ -68,12 +69,68 @@ func main() {
 	<-c
 }
 
+func main() {
+	go updateMethod()
+
+	var path string
+	flag.StringVar(&path, "p", "", "路径 /")
+	flag.Parse()
+	if path == "" {
+		flag.PrintDefaults()
+		log.Fatal("参数错误.")
+	}
+	subFiles, _ := ioutil.ReadDir(path)
+	for _, s := range subFiles {
+		count := 0
+		CurrentColl = strings.Split(s.Name(), ".")[0]
+		file := path + s.Name()
+		util.Debug("current date：  ", file)
+		// 打开本地gz格式压缩包
+		fr, err := os.Open(file)
+		if err != nil {
+			panic(err)
+		} else {
+			println("open file success!")
+		}
+		// defer: 在函数退出时,执行关闭文件
+		defer fr.Close()
+		// 创建gzip文件读取对象
+		gr, err := gzip.NewReader(fr)
+		if err != nil {
+			panic(err)
+		}
+		// defer: 在函数退出时,执行关闭gzip对象
+		defer gr.Close()
+		bfRd := bufio.NewReader(gr)
+		for {
+			line, err := bfRd.ReadBytes('\n')
+			count = hookfn(line, count)
+			if err != nil {
+				if err == io.EOF {
+					fmt.Println("read gzip data finish! ")
+					if len(saveArr) > 0 {
+						tmps := saveArr
+						MongoTool.UpSertBulk(CurrentColl, tmps...)
+						saveArr = [][]map[string]interface{}{}
+					}
+					break
+				} else {
+					fmt.Println("[read gzip data err]: ", err)
+				}
+			}
+			if count%5000 == 0 {
+				util.Debug("current exc---", file, count)
+			}
+		}
+	}
+}
+
 func task(path string) {
 	for _, c := range CollArr {
 		CurrentColl = c
 		subPath := path + c + "/"
 		subFiles, _ := ioutil.ReadDir(subPath)
-		for _, s := range subFiles{
+		for _, s := range subFiles {
 			if s.IsDir() {
 				taskinfo(subPath + s.Name())
 			}
@@ -105,29 +162,28 @@ func taskinfo(path string) {
 	bfRd := bufio.NewReader(gr)
 	for {
 		line, err := bfRd.ReadBytes('\n')
-		count  = hookfn(line, count)
+		count = hookfn(line, count)
 		if err != nil {
 			if err == io.EOF {
 				fmt.Println("read gzip data finish! ")
 				break
-			}else {
+			} else {
 				fmt.Println("[read gzip data err]: ", err)
 			}
 		}
-		if count % 5000 == 0 {
+		if count%5000 == 0 {
 			util.Debug("current exc---", file, count)
 		}
 	}
 }
 
-
-func hookfn(line []byte, count int) int {
+func hookfn1(line []byte, count int) int {
 	tmp := make(map[string]interface{})
-	err:= json.Unmarshal(line, &tmp)
+	err := json.Unmarshal(line, &tmp)
 	if err != nil {
 		util.Debug("err---", err)
 	}
-	count ++
+	count++
 
 	save := make(map[string]interface{})
 	save["$push"] = map[string]interface{}{
@@ -143,6 +199,32 @@ func hookfn(line []byte, count int) int {
 	return count
 }
 
+func hookfn(line []byte, count int) int {
+	tmp := make(map[string]interface{})
+	err := json.Unmarshal(line, &tmp)
+	if err != nil {
+		util.Debug("err---", err)
+	}
+	count++
+
+	if len(tmp) > 0 {
+		id := tmp["_id"].(map[string]interface{})["$oid"]
+		delete(tmp, "_id")
+		saveInfo := []map[string]interface{}{
+			{"_id": mongodb.StringTOBsonId(util.ObjToString(id))},
+			tmp,
+		}
+		saveArr = append(saveArr, saveInfo)
+		if len(saveArr) > 500 {
+			tmps := saveArr
+			MongoTool.UpSertBulk(CurrentColl, tmps...)
+			saveArr = [][]map[string]interface{}{}
+		}
+	}
+
+	return count
+}
+
 func updateMethod() {
 	arru := make([][]map[string]interface{}, saveSize)
 	indexu := 0
@@ -157,7 +239,7 @@ func updateMethod() {
 					defer func() {
 						<-updateSp
 					}()
-					MongoTool.UpSertBulk("qyxy_tmp", arru...)
+					MongoTool.UpSertBulk("jy_tmp", arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, saveSize)
 				indexu = 0
@@ -169,11 +251,11 @@ func updateMethod() {
 					defer func() {
 						<-updateSp
 					}()
-					MongoTool.UpSertBulk("qyxy_tmp", arru...)
+					MongoTool.UpSertBulk("jy_tmp", arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, saveSize)
 				indexu = 0
 			}
 		}
 	}
-}
+}

qyxy/qyxy_redis/main.go

@@ -12,30 +12,30 @@ import (
 )
 
 var (
-	MongoTool		*mongodb.MongodbSim
+	MongoTool *mongodb.MongodbSim
 
-	PnameLength			int
-	PurchasingLength 	int
+	PnameLength      int
+	PurchasingLength int
 )
 
 func init() {
+	MongoTool = &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.85:27080", // 172.17.4.85:27080
+		Size:        10,
+		DbName:      "qfw",
+	}
+	MongoTool.InitPool()
 	//MongoTool = &mongodb.MongodbSim{
-	//	MongodbAddr: "172.17.4.85:27080",					// 172.17.4.85:27080
+	//	MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",
 	//	Size:        10,
 	//	DbName:      "mixdata",
+	//	UserName:    "SJZY_RWMIX_Other",
+	//	Password:    "SJZY@M34I6x7D9ata",
 	//}
 	//MongoTool.InitPool()
-	MongoTool = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",
-		Size:        10,
-		DbName:      "mixdata",
-		UserName:	 "SJZY_RWMIX_Other",
-		Password: 	 "SJZY@M34I6x7D9ata",
-	}
-	MongoTool.InitPool()
 
-	redis.InitRedis1("qyxy_id=172.17.4.189:8379", 2)				// 企业company_id（中标单位）
-	//redis.InitRedis1("qyxy_winner=172.17.4.189:8379", 3)			//	剑鱼中标企业搜索搜索
+	//redis.InitRedis1("qyxy_id=172.17.4.189:8379", 2)				// 企业company_id（中标单位）
+	redis.InitRedis1("qyxy=172.17.4.189:8379", 3) //	剑鱼企业搜索搜索
 
 	PnameLength = 500
 	PurchasingLength = 500
@@ -43,7 +43,7 @@ func init() {
 
 var name string
 
-func main1()  {
+func main1() {
 	flag.StringVar(&name, "n", "", "企业名称")
 	flag.Parse()
 	if name == "" {
@@ -63,7 +63,7 @@ func main() {
 	sess := MongoTool.GetMgoConn()
 	defer MongoTool.DestoryMongoConn(sess)
 
-	ch := make(chan bool, 2)
+	ch := make(chan bool, 3)
 	wg := &sync.WaitGroup{}
 
 	//q := map[string]interface{}{
@@ -72,20 +72,14 @@ func main() {
 	//		"$lte": mongodb.StringTOBsonId("5e0a1f000000000000000000"),
 	//	},
 	//}
-	//field := map[string]interface{}{"agency": 1, "buyer": 1, "s_winner": 1, "projectname": 1, "purchasing": 1, "area": 1}
-	field := map[string]interface{}{"use_flag": 1, "company_type": 1, "company_name": 1}
-	query := sess.DB("mixdata").C("qyxy_std").Find(nil).Select(field).Skip(52035000).Iter()
+	field := map[string]interface{}{"agency": 1, "buyer": 1, "s_winner": 1, "projectname": 1, "purchasing": 1, "area": 1}
+	//field := map[string]interface{}{"use_flag": 1, "company_type": 1, "company_name": 1}
+	query := sess.DB("qfw").C("projectset_20200918").Find(nil).Select(field).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
 		if count%5000 == 0 {
 			util.Debug("current ---", count)
 		}
-		if util.IntAll(tmp["use_flag"]) > 5 {
-			continue
-		}
-		if strings.Contains(util.ObjToString(tmp["company_type"]), "个体") {
-			continue
-		}
 		ch <- true
 		wg.Add(1)
 		go func(tmp map[string]interface{}) {
@@ -93,7 +87,8 @@ func main() {
 				<-ch
 				wg.Done()
 			}()
-			redis.PutCKV("qyxy_id", util.ObjToString(tmp["company_name"]), util.ObjToString(tmp["_id"]))
+			//redis.PutCKV("qyxy_id", util.ObjToString(tmp["company_name"]), util.ObjToString(tmp["_id"]))
+			taskinfo1(tmp)
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
@@ -104,10 +99,10 @@ func main() {
 func taskinfo1(tmp map[string]interface{}) {
 	if util.ObjToString(tmp["s_winner"]) != "" {
 		winners := strings.Split(util.ObjToString(tmp["s_winner"]), ",")
-		for _, v := range winners{
+		for _, v := range winners {
 			winnermap := make(map[string]interface{})
-			if b, err := redis.Exists("qyxy_winner", v); err == nil && b {
-				text := redis.GetStr("qyxy_winner", v)
+			if b, err := redis.Exists("qyxy", v); err == nil && b {
+				text := redis.GetStr("qyxy", v)
 				err1 := json.Unmarshal([]byte(text), &winnermap)
 				if err1 != nil {
 					util.Debug(v, "winner-----map解析异常")
@@ -124,10 +119,10 @@ func taskinfo1(tmp map[string]interface{}) {
 						if len(pnameArr) > PnameLength {
 							pnameArr1 := pnameArr[len(pnameArr)-PnameLength:]
 							winnermap["bid_projectname"] = pnameArr1
-						}else {
+						} else {
 							winnermap["bid_projectname"] = pnameArr
 						}
-					}else {
+					} else {
 						pname := []string{util.ObjToString(tmp["projectname"])}
 						winnermap["bid_projectname"] = pname
 					}
@@ -143,10 +138,10 @@ func taskinfo1(tmp map[string]interface{}) {
 						if len(purs) > PurchasingLength {
 							purs1 := purs[len(purs)-PurchasingLength:]
 							winnermap["bid_purchasing"] = purs1
-						}else {
+						} else {
 							winnermap["bid_purchasing"] = purs
 						}
-					}else {
+					} else {
 						purs := []string{util.ObjToString(tmp["purchasing"])}
 						winnermap["bid_purchasing"] = purs
 					}
@@ -159,7 +154,7 @@ func taskinfo1(tmp map[string]interface{}) {
 						areas = append(areas, util.ObjToString(tmp["area"]))
 					}
 					winnermap["bid_area"] = areas
-				}else {
+				} else {
 					areas := []string{util.ObjToString(tmp["area"])}
 					winnermap["bid_area"] = areas
 				}
@@ -171,11 +166,11 @@ func taskinfo1(tmp map[string]interface{}) {
 						types = append(types, "中标单位")
 					}
 					winnermap["bid_unittype"] = types
-				}else {
+				} else {
 					types := []string{"中标单位"}
 					winnermap["bid_unittype"] = types
 				}
-			}else {
+			} else {
 				// bid_projectname
 				if tmp["projectname"] != nil && tmp["projectname"] != "" {
 					winnermap["bid_projectname"] = []string{util.ObjToString(tmp["projectname"])}
@@ -190,16 +185,16 @@ func taskinfo1(tmp map[string]interface{}) {
 				winnermap["bid_unittype"] = []string{"中标单位"}
 			}
 			if len(winnermap) > 0 {
-				data , _ := json.Marshal(winnermap)
-				redis.Put("qyxy_winner", v, string(data), 0)
+				data, _ := json.Marshal(winnermap)
+				redis.Put("qyxy", v, string(data), 0)
 
 			}
 		}
 	}
 	if buyer := util.ObjToString(tmp["buyer"]); buyer != "" {
 		buyermap := make(map[string]interface{})
-		if b, err := redis.Exists("qyxy_winner", buyer); err == nil && b {
-			text := redis.GetStr("qyxy_winner", buyer)
+		if b, err := redis.Exists("qyxy", buyer); err == nil && b {
+			text := redis.GetStr("qyxy", buyer)
 			err1 := json.Unmarshal([]byte(text), &buyermap)
 			if err1 != nil {
 				util.Debug(buyer, "buyer-----map解析异常")
@@ -212,22 +207,22 @@ func taskinfo1(tmp map[string]interface{}) {
 					types = append(types, "采购单位")
 				}
 				buyermap["bid_unittype"] = types
-			}else {
+			} else {
 				types := []string{"采购单位"}
 				buyermap["bid_unittype"] = types
 			}
-		}else {
+		} else {
 			buyermap["bid_unittype"] = []string{"采购单位"}
 		}
 		if len(buyermap) > 0 {
-			data , _ := json.Marshal(buyermap)
-			redis.Put("qyxy_winner", buyer, string(data), 0)
+			data, _ := json.Marshal(buyermap)
+			redis.Put("qyxy", buyer, string(data), 0)
 		}
 	}
 	if agency := util.ObjToString(tmp["agency"]); agency != "" {
 		agencymap := make(map[string]interface{})
-		if b, err := redis.Exists("qyxy_winner", agency); err == nil && b {
-			text := redis.GetStr("qyxy_winner", agency)
+		if b, err := redis.Exists("qyxy", agency); err == nil && b {
+			text := redis.GetStr("qyxy", agency)
 			err1 := json.Unmarshal([]byte(text), &agencymap)
 			if err1 != nil {
 				util.Debug(agency, "agency----map解析异常")
@@ -240,16 +235,16 @@ func taskinfo1(tmp map[string]interface{}) {
 					types = append(types, "代理机构")
 				}
 				agencymap["bid_unittype"] = types
-			}else {
+			} else {
 				types := []string{"代理机构"}
 				agencymap["bid_unittype"] = types
 			}
-		}else {
+		} else {
 			agencymap["bid_unittype"] = []string{"代理机构"}
 		}
 		if len(agencymap) > 0 {
-			data , _ := json.Marshal(agencymap)
-			redis.Put("qyxy_winner", agency, string(data), 0)
+			data, _ := json.Marshal(agencymap)
+			redis.Put("qyxy", agency, string(data), 0)
 		}
 	}
 }
@@ -268,4 +263,4 @@ func BinarySearch(s []string, k string) int {
 		}
 	}
 	return -1
-}
+}

qyxy/qyxy_std/config.json

@@ -1,8 +1,8 @@
 {
-  "dbServer": "172.17.4.187:27082,172.17.145.163:27083",
-  "dbName": "mixdata",
+  "dbServer": "192.168.3.207:27092",
+  "dbName": "wjh",
   "dbSize": 10,
-  "dbSave": "qyxy_std_1",
+  "dbSave": "qyxy_std",
   "uname": "SJZY_RWMIX_Other",
   "upwd": "SJZY@M34I6x7D9ata",
   "lastId": 261511180

qyxy/qyxy_std/init.go

@@ -6,28 +6,29 @@ import (
 	"log"
 	"mongodb"
 	"qfw/util"
+	"qfw/util/elastic"
 	"qfw/util/redis"
 )
 
 var (
-	Sysconfig        	map[string]interface{}
-	MongoTool			*mongodb.MongodbSim
-	MongoTool1			*mongodb.MongodbSim
-	MysqlTool			*Mysql
-	DbSave				string
-
-	updatePool			chan []map[string]interface{}
-	updateSp			chan bool
-	saveSize			int
-	savePool			chan map[string]interface{}
-	saveSp				chan bool
-
-	QyStypeMap       	map[string]string
-	CompanyStatusMap 	map[string]string
-	AddressMap       	map[string]*City
-	AddressOldMap    	map[string]*City
-
-	lastId				int64
+	Sysconfig  map[string]interface{}
+	MongoTool  *mongodb.MongodbSim
+	MongoTool1 *mongodb.MongodbSim
+	MysqlTool  *Mysql
+	DbSave     string
+
+	updatePool chan []map[string]interface{}
+	updateSp   chan bool
+	saveSize   int
+	savePool   chan map[string]interface{}
+	saveSp     chan bool
+
+	QyStypeMap       map[string]string
+	CompanyStatusMap map[string]string
+	AddressMap       map[string]*City
+	AddressOldMap    map[string]*City
+
+	Es *elastic.Elastic
 )
 
 func init() {
@@ -38,13 +39,13 @@ func init() {
 		MongodbAddr: util.ObjToString(Sysconfig["dbServer"]),
 		Size:        util.IntAll(Sysconfig["dbSize"]),
 		DbName:      util.ObjToString(Sysconfig["dbName"]),
-		UserName: 	 util.ObjToString(Sysconfig["uname"]),
-		Password: 	 util.ObjToString(Sysconfig["upwd"]),
+		UserName:    util.ObjToString(Sysconfig["uname"]),
+		Password:    util.ObjToString(Sysconfig["upwd"]),
 	}
 	MongoTool.InitPool()
 
 	MongoTool1 = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.4.181:27001",
+		MongodbAddr: "172.17.4.181:27001", // 172.17.4.181:27001
 		Size:        10,
 		DbName:      "mixdata",
 	}
@@ -58,14 +59,18 @@ func init() {
 	//}
 	//MysqlTool.Init()
 
+	Es = &elastic.Elastic{
+		S_esurl: "http://172.17.145.170:9800", //http://172.17.145.170:9800
+		I_size:  10,
+	}
+	Es.InitElasticSize()
+
 	saveSize = 200
 	updatePool = make(chan []map[string]interface{}, 5000)
 	updateSp = make(chan bool, 5)
 	savePool = make(chan map[string]interface{}, 5000)
 	saveSp = make(chan bool, 5)
 
-	lastId = 0
-
 	InitQyStype()
 	InitCompanyStatus()
 	InitAddress()
@@ -74,7 +79,7 @@ func init() {
 		log.Fatal("结巴分词出错...")
 		return
 	}
-	redis.InitRedis1("qyxy_winner=127.0.0.1:8379", 3)
+	redis.InitRedis1("qyxy=172.17.4.189:8379", 3)
 }
 
 func InitQyStype() {
@@ -125,7 +130,7 @@ func InitAddress() {
 	result := sess.DB("mixdata").C("address_new_2020").Find(nil).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); result.Next(&tmp); count++ {
-		if count % 50000 == 0 {
+		if count%50000 == 0 {
 			util.Debug("current---", count)
 		}
 		code := util.ObjToString(tmp["code"])
@@ -149,4 +154,4 @@ func InitAddress() {
 		}
 	}
 	util.Debug("Init Address end...", len(AddressMap), len(AddressOldMap))
-}
+}

qyxy/qyxy_std/task.go

@@ -15,11 +15,11 @@ import (
 )
 
 var (
-	AreaFiled				 = []string{"credit_no", "company_code"}
-	WordsArr 				= []string{"研发", "研制", "开发", "生产", "制造", "制作", "加工", "种植"}
+	AreaFiled = []string{"credit_no", "company_code"}
+	WordsArr  = []string{"研发", "研制", "开发", "生产", "制造", "制作", "加工", "种植"}
 
-	seg 					jiebago.Segmenter
-	regPre, _ 				= regexp.Compile(`^(.+[省|市|区|县|州])?(.+)`)
+	seg       jiebago.Segmenter
+	regPre, _ = regexp.Compile(`^(.+[省|市|区|县|州])?(.+)`)
 )
 
 //	企业基本信息
@@ -30,7 +30,7 @@ var company_base = []string{"company_name", "company_code", "credit_no", "org_co
 }
 
 var province_map = map[string]string{
-	"BJ": "北京", "TJ": "天津", "SH": "上海", "CQ": "重庆", "HB": "河北", "SX": "山西", "NMG": "内蒙", "LN": "辽宁", "JL": "吉林",
+	"BJ": "北京", "TJ": "天津", "SH": "上海", "CQ": "重庆", "HB": "河北", "SX": "山西", "NMG": "内蒙古", "LN": "辽宁", "JL": "吉林",
 	"HLJ": "黑龙江", "JS": "江苏", "ZJ": "浙江", "AH": "安徽", "FJ": "福建", "JX": "江西", "SD": "山东", "HEN": "河南", "HUB": "湖北",
 	"HUN": "湖南", "GD": "广东", "GX": "广西", "HAIN": "海南", "SC": "四川", "GZ": "贵州", "YN": "云南", "XZ": "西藏", "SAX": "陕西",
 	"GS": "甘肃", "QH": "青海", "NX": "宁夏", "XJ": "新疆",
@@ -39,7 +39,7 @@ var province_map = map[string]string{
 func TimeTask() {
 	c := cron.New()
 	//cronstr := "0 0 " + fmt.Sprint(TaskTime) + " * * ?"			//每天TaskTime跑一次
-	cronstrPa := "0 0 15 ? * WED"								//凭安增量数据每周二跑一次
+	cronstrPa := "0 0 15 ? * WED" //凭安增量数据每周二跑一次
 	_ = c.AddFunc(cronstrPa, func() {
 		TaskFun()
 	})
@@ -51,16 +51,20 @@ func TaskFun() {
 	sess := MongoTool1.GetMgoConn()
 	defer MongoTool1.DestoryMongoConn(sess)
 
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+
 	pool := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
 
 	//q := bson.M{"_id": bson.M{"$gt": lastId}}
 	//q := bson.M{"_id": 262454280}
+	//q := map[string]interface{}{"_id": "3fd1352a95f552bda9bac1248eb24f03"}
+	//q := map[string]interface{}{"company_name": nil}
 	it := sess.DB("mixdata").C("qyxy_tmp").Find(nil).Select(nil).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
-		lastId = util.Int64All(tmp["_id"])
-		if count%20000 == 0 {
+		if count%2000 == 0 {
 			util.Debug("current:", count)
 		}
 		pool <- true
@@ -71,6 +75,8 @@ func TaskFun() {
 				wg.Done()
 			}()
 			IncStd(tmp)
+			//taskinfo(tmp)
+
 		}(tmp)
 	}
 
@@ -86,7 +92,7 @@ func IncStd(tmp map[string]interface{}) {
 	if tmp["company_base"] != nil {
 		companyArr := util.ObjArrToMapArr(tmp["company_base"].([]interface{}))
 		company := companyArr[len(companyArr)-1]
-		for _, v := range company_base{
+		for _, v := range company_base {
 			if company[v] == nil {
 				continue
 			}
@@ -94,7 +100,7 @@ func IncStd(tmp map[string]interface{}) {
 			if v == "company_type" {
 				save["company_type_old"] = company[v]
 				if text := util.ObjToString(company["company_type"]); text != "" {
-					if strings.Contains(text, "个体") || strings.Contains(text, "非公司")  {
+					if strings.Contains(text, "个体") || strings.Contains(text, "非公司") {
 						save["company_type"] = "个体工商户"
 					} else {
 						text = strings.ReplaceAll(text, "（", "(")
@@ -107,7 +113,7 @@ func IncStd(tmp map[string]interface{}) {
 					}
 				}
 				// company_status
-			}else if v == "company_status" {
+			} else if v == "company_status" {
 				save["company_status_old"] = company[v]
 				if text := util.ObjToString(company["company_status"]); text != "" {
 					text = strings.ReplaceAll(text, "（", "(")
@@ -118,7 +124,7 @@ func IncStd(tmp map[string]interface{}) {
 						save["company_status"] = "其他"
 					}
 				}
-			}else if v == "capital" {
+			} else if v == "capital" {
 				// capital/currency
 				text := util.ObjToString(company[v])
 				if currency := GetCurrency(text); currency != "" {
@@ -129,9 +135,9 @@ func IncStd(tmp map[string]interface{}) {
 				if capital != 0 {
 					save[v] = capital
 				}
-			}else if v == "use_flag" {
+			} else if v == "use_flag" {
 				save[v] = util.IntAll(company[v])
-			}else {
+			} else {
 				save[v] = company[v]
 			}
 		}
@@ -152,7 +158,7 @@ func IncStd(tmp map[string]interface{}) {
 					code = code[:6]
 				}
 				if city := AddressMap[code]; city != nil { //未作废中取
-					if city.Province != "" && city.Province == save["company_area"] {
+					if city.Province != "" && city.Province == util.ObjToString(save["company_area"]) {
 						if city.City != "" {
 							save["company_city"] = city.City //市
 						}
@@ -163,7 +169,7 @@ func IncStd(tmp map[string]interface{}) {
 					}
 				} else { //作废中取
 					if city := AddressOldMap[code]; city != nil {
-						if city.Province != "" && city.Province == save["company_area"] {
+						if city.Province != "" && city.Province == util.ObjToString(save["company_area"]) {
 							if city.City != "" {
 								save["company_city"] = city.City //市
 							}
@@ -184,76 +190,191 @@ func IncStd(tmp map[string]interface{}) {
 		if tmp["company_history_name"] != nil {
 			history_name := util.ObjArrToMapArr(tmp["company_history_name"].([]interface{}))
 			var names []string
-			for _, h := range history_name{
+			for _, h := range history_name {
 				names = append(names, util.ObjToString(h["history_name"]))
 			}
-			save["history_name"] =  strings.Join(names, ",")
+			save["history_name"] = strings.Join(names, ",")
+		}
+
+		// company_employee、company_partner、annual_report_base涉及存量更新动作
+		oldTmp := &map[string]interface{}{}
+		if tmp["company_employee"] != nil || tmp["company_partner"] != nil || tmp["annual_report_base"] != nil {
+			fields := map[string]interface{}{"employees": 1, "partners": 1, "annual_reports": 1}
+			oldTmp, _ = MongoTool.FindOneByField("qyxy_std", map[string]interface{}{"_id": tmp["_id"]}, fields)
 		}
+
 		// company_employee
 		if tmp["company_employee"] != nil {
 			employees := util.ObjArrToMapArr(tmp["company_employee"].([]interface{}))
 			var names []string
 			var arr []map[string]interface{}
-			for _, v := range employees{
-				tmp := make(map[string]interface{})
-				tmp["employee_name"] = v["employee_name"]
-				tmp["position"] = v["position"]
-				tmp["is_history"] = v["is_history"]
-				arr = append(arr, tmp)
-				names = append(names, util.ObjToString(v["employee_name"]))
-			}
-			//save["employees"] = arr
-			push["employees"] = bson.M{"$each": arr}
-			save["employee_name"] =  strings.Join(names, ",")
+			if (*oldTmp)["employees"] != nil {
+				arr = util.ObjArrToMapArr((*oldTmp)["employees"].([]interface{}))
+			} else {
+				arr = make([]map[string]interface{}, 0)
+			}
+			//arr := (*oldTmp)["employees"].([]map[string]interface{})
+			for _, v := range employees {
+				ep := make(map[string]interface{})
+				if util.ObjToString(v["_operation_type"]) == "insert" {
+					ep["employee_name"] = v["employee_name"]
+					ep["position"] = v["position"]
+					ep["is_history"] = v["is_history"]
+					ep["_id"] = util.IntAll(v["id"])
+					arr = append(arr, ep)
+					names = append(names, util.ObjToString(v["employee_name"]))
+				} else {
+					eq_flag := true
+					for _, m := range arr {
+						if util.IntAll(v["id"]) == util.IntAll(m["_id"]) {
+							eq_flag = false
+							m["employee_name"] = v["employee_name"]
+							m["position"] = v["position"]
+							m["is_history"] = v["is_history"]
+							break
+						}
+					}
+					if eq_flag {
+						ep := make(map[string]interface{})
+						ep["employee_name"] = v["employee_name"]
+						ep["position"] = v["position"]
+						ep["is_history"] = v["is_history"]
+						ep["_id"] = util.IntAll(v["id"])
+						arr = append(arr, ep)
+						names = append(names, util.ObjToString(v["employee_name"]))
+					}
+				}
+
+			}
+			save["employees"] = arr
+			//push["employees"] = bson.M{"$each": arr}
+			save["employee_name"] = strings.Join(names, ",")
 		}
 		// company_partner
 		if tmp["company_partner"] != nil {
 			partners := util.ObjArrToMapArr(tmp["company_partner"].([]interface{}))
 			var names []string
 			var arr []map[string]interface{}
-			tmp["partners"] =  partners
-			for _, v := range partners{
-				if util.IntAll(tmp["is_history"]) == 0 {
-					tmp := make(map[string]interface{})
-					tmp["stock_capital"] = v["stock_capital"]
-					tmp["stock_name"] = v["stock_name"]
-					tmp["identify_no"] = v["identify_no"]
-					tmp["stock_realcapital"] = v["stock_realcapital"]
-					tmp["is_history"] = v["is_history"]
-					tmp["is_personal"] = v["is_personal"]
-					tmp["stock_type"] = v["stock_type"]
-					tmp["identify_type"] = v["identify_type"]
-					arr = append(arr, tmp)
+			if (*oldTmp)["partners"] != nil {
+				arr = util.ObjArrToMapArr((*oldTmp)["partners"].([]interface{}))
+			} else {
+				arr = make([]map[string]interface{}, 0)
+			}
+			for _, v := range partners {
+				if util.ObjToString(v["_operation_type"]) == "insert" {
+					exp := make(map[string]interface{})
+					exp["stock_capital"] = v["stock_capital"]
+					exp["stock_name"] = v["stock_name"]
+					exp["identify_no"] = v["identify_no"]
+					exp["stock_realcapital"] = v["stock_realcapital"]
+					exp["is_history"] = v["is_history"]
+					exp["is_personal"] = v["is_personal"]
+					exp["stock_type"] = v["stock_type"]
+					exp["identify_type"] = v["identify_type"]
+					exp["_id"] = util.IntAll(v["id"])
+					arr = append(arr, exp)
 					names = append(names, util.ObjToString(v["stock_name"]))
+				} else {
+					eqFlag := true
+					for _, m := range arr {
+						if util.IntAll(v["id"]) == util.IntAll(m["_id"]) {
+							eqFlag = false
+							m["stock_capital"] = v["stock_capital"]
+							m["stock_name"] = v["stock_name"]
+							m["identify_no"] = v["identify_no"]
+							m["stock_realcapital"] = v["stock_realcapital"]
+							m["is_history"] = v["is_history"]
+							m["is_personal"] = v["is_personal"]
+							m["stock_type"] = v["stock_type"]
+							m["identify_type"] = v["identify_type"]
+							break
+						}
+					}
+					if eqFlag {
+						exp := make(map[string]interface{})
+						exp["stock_capital"] = v["stock_capital"]
+						exp["stock_name"] = v["stock_name"]
+						exp["identify_no"] = v["identify_no"]
+						exp["stock_realcapital"] = v["stock_realcapital"]
+						exp["is_history"] = v["is_history"]
+						exp["is_personal"] = v["is_personal"]
+						exp["stock_type"] = v["stock_type"]
+						exp["identify_type"] = v["identify_type"]
+						exp["_id"] = util.IntAll(v["id"])
+						arr = append(arr, exp)
+						names = append(names, util.ObjToString(v["stock_name"]))
+					}
 				}
+
 			}
-			//save["partners"] =  arr
-			push["partners"] = bson.M{"$each": arr}
-			save["stock_name"] =  strings.Join(names, ",")
+			save["partners"] = arr
+			//push["partners"] = bson.M{"$each": arr}
+			save["stock_name"] = strings.Join(names, ",")
 		}
 		// annual_report_base
 		if tmp["annual_report_base"] != nil {
 			reports := util.ObjArrToMapArr(tmp["annual_report_base"].([]interface{}))
 			var arr []map[string]interface{}
+			if (*oldTmp)["annual_reports"] != nil {
+				arr = util.ObjArrToMapArr((*oldTmp)["annual_reports"].([]interface{}))
+			} else {
+				arr = make([]map[string]interface{}, 0)
+			}
 			year := 0
 			phone, email := "", ""
-			for _, v := range reports{
-				tmp := make(map[string]interface{})
-				tmp["operator_name"] = v["operator_name"]
-				tmp["report_year"] = v["report_year"]
-				tmp["zip_code"] = v["zip_code"]
-				tmp["employee_no"] = v["employee_no"]
-				tmp["company_phone"] = v["company_phone"]
-				tmp["company_email"] = v["company_email"]
-				arr = append(arr, tmp)
-				if year < util.IntAll(v["report_year"]) {
-					year = util.IntAll(v["report_year"])
-					phone = util.ObjToString(v["company_phone"])
-					email = util.ObjToString(v["company_email"])
+			for _, v := range reports {
+				if util.ObjToString(v["_operation_type"]) == "insert" {
+					exp := make(map[string]interface{})
+					exp["operator_name"] = v["operator_name"]
+					exp["report_year"] = v["report_year"]
+					exp["zip_code"] = v["zip_code"]
+					exp["employee_no"] = v["employee_no"]
+					exp["company_phone"] = v["company_phone"]
+					exp["company_email"] = v["company_email"]
+					exp["_id"] = util.IntAll(v["id"])
+					arr = append(arr, exp)
+					if year < util.IntAll(v["report_year"]) {
+						year = util.IntAll(v["report_year"])
+						phone = util.ObjToString(v["company_phone"])
+						email = util.ObjToString(v["company_email"])
+					}
+				} else {
+					eqFlag := true
+					for _, m := range arr {
+						if util.IntAll(v["id"]) == util.IntAll(m["_id"]) {
+							eqFlag = false
+							m["operator_name"] = v["operator_name"]
+							m["report_year"] = v["report_year"]
+							m["zip_code"] = v["zip_code"]
+							m["employee_no"] = v["employee_no"]
+							m["company_phone"] = v["company_phone"]
+							m["company_email"] = v["company_email"]
+							break
+						}
+					}
+					if eqFlag {
+						exp := make(map[string]interface{})
+						exp["stock_capital"] = v["stock_capital"]
+						exp["stock_name"] = v["stock_name"]
+						exp["identify_no"] = v["identify_no"]
+						exp["stock_realcapital"] = v["stock_realcapital"]
+						exp["is_history"] = v["is_history"]
+						exp["is_personal"] = v["is_personal"]
+						exp["stock_type"] = v["stock_type"]
+						exp["identify_type"] = v["identify_type"]
+						exp["_id"] = util.IntAll(v["id"])
+						arr = append(arr, exp)
+						if year < util.IntAll(v["report_year"]) {
+							year = util.IntAll(v["report_year"])
+							phone = util.ObjToString(v["company_phone"])
+							email = util.ObjToString(v["company_email"])
+						}
+					}
 				}
+
 			}
-			//save["annual_reports"] =  arr
-			push["annual_reports"] = bson.M{"$each": arr}
+			save["annual_reports"] = arr
+			//push["annual_reports"] = bson.M{"$each": arr}
 			if year != 0 {
 				save["company_phone"] = phone
 				save["company_email"] = email
@@ -264,14 +385,14 @@ func IncStd(tmp map[string]interface{}) {
 			webs := util.ObjArrToMapArr(tmp["annual_report_website"].([]interface{}))
 			year := 0
 			web := ""
-			for _, v := range webs{
+			for _, v := range webs {
 				if year < util.IntAll(v["report_year"]) && util.IntAll(v["is_history"]) == 0 {
 					year = util.IntAll(v["report_year"])
 					web = util.ObjToString(v["website_url"])
 				}
 			}
 			if year != 0 {
-				save["website_url"] =  web
+				save["website_url"] = web
 			}
 		}
 		// bid_contracttype
@@ -279,28 +400,27 @@ func IncStd(tmp map[string]interface{}) {
 		if phone := util.ObjToString(save["company_phone"]); phone != "" {
 			if len(phone) == 11 {
 				types = append(types, "手机号")
-			}else {
+			} else {
 				types = append(types, "固定电话")
 			}
 		}
 		if util.ObjToString(save["company_email"]) != "" {
 			types = append(types, "邮箱")
 		}
-
 		companyName := util.ObjToString(save["company_name"])
 		// redis bid_unittype、bid_purchasing、bid_projectname
-		if b, err := redis.Exists("qyxy_winner", companyName); err == nil && b {
+		if b, err := redis.Exists("qyxy", companyName); err == nil && b {
 			maps := make(map[string]interface{})
-			text := redis.GetStr("qyxy_winner", companyName)
+			text := redis.GetStr("qyxy", companyName)
 			err1 := json.Unmarshal([]byte(text), &maps)
 			if err1 != nil {
 				util.Debug(companyName, "winner-----map解析异常")
-			}else {
-				for k := range maps{
+			} else {
+				for k := range maps {
 					if k == "bid_contracttype" {
 						t1 := util.ObjArrToStringArr(maps[k].([]interface{}))
 						types = append(types, t1...)
-					}else {
+					} else {
 						save[k] = maps[k]
 					}
 				}
@@ -312,14 +432,18 @@ func IncStd(tmp map[string]interface{}) {
 		save["bid_contracttype"] = types
 		// bid_unittype
 		flag := false
-		for _, v := range WordsArr{
+		for _, v := range WordsArr {
 			if strings.Contains(util.ObjToString(save["business_scope"]), v) {
 				flag = true
 				break
 			}
 		}
 		if flag {
-			save["bid_unittype"] = []string{"厂商"}
+			if save["bid_unittype"] != nil {
+				save["bid_unittype"] = append(util.ObjArrToStringArr(save["bid_unittype"].([]interface{})), "厂商")
+			} else {
+				save["bid_unittype"] = []string{"厂商"}
+			}
 		}
 		// search_type
 		if t := util.ObjToString(save["company_type"]); t != "" {
@@ -328,12 +452,12 @@ func IncStd(tmp map[string]interface{}) {
 				name := util.ObjToString(save["company_name"])
 				if strings.Contains(t1, "有限合伙") {
 					save["search_type"] = "有限合伙"
-				}else if strings.Contains(t1, "合伙") {
+				} else if strings.Contains(t1, "合伙") {
 					save["search_type"] = "普通合伙"
-				}else if strings.Contains(name, "股份") ||
+				} else if strings.Contains(name, "股份") ||
 					(strings.Contains(t1, "上市") && !strings.Contains(t1, "非上市")) {
 					save["search_type"] = "股份有限公司"
-				}else {
+				} else {
 					save["search_type"] = "有限责任公司"
 				}
 			}
@@ -356,9 +480,9 @@ func IncStd(tmp map[string]interface{}) {
 }
 
 // InfoStd 存量全量数据
-func InfoStd(tmp map[string]interface{})  {
+func InfoStd(tmp map[string]interface{}) {
 	save := make(map[string]interface{})
-	for _, v := range company_base{
+	for _, v := range company_base {
 		if tmp[v] == nil {
 			continue
 		}
@@ -366,7 +490,7 @@ func InfoStd(tmp map[string]interface{})  {
 		if v == "company_type" {
 			save["company_type_old"] = tmp[v]
 			if text := util.ObjToString(tmp["company_type"]); text != "" {
-				if strings.Contains(text, "个体") || strings.Contains(text, "非公司")  {
+				if strings.Contains(text, "个体") || strings.Contains(text, "非公司") {
 					save["company_type"] = "个体工商户"
 				} else {
 					text = strings.ReplaceAll(text, "（", "(")
@@ -378,8 +502,8 @@ func InfoStd(tmp map[string]interface{})  {
 					}
 				}
 			}
-		// company_status
-		}else if v == "company_status" {
+			// company_status
+		} else if v == "company_status" {
 			save["company_status_old"] = tmp[v]
 			if text := util.ObjToString(tmp["company_status"]); text != "" {
 				text = strings.ReplaceAll(text, "（", "(")
@@ -390,7 +514,7 @@ func InfoStd(tmp map[string]interface{})  {
 					save["company_status"] = "其他"
 				}
 			}
-		}else if v == "capital" {
+		} else if v == "capital" {
 			// capital/currency
 			text := util.ObjToString(tmp[v])
 			if currency := GetCurrency(text); currency != "" {
@@ -401,9 +525,9 @@ func InfoStd(tmp map[string]interface{})  {
 			if capital != 0 {
 				save[v] = capital
 			}
-		}else if v == "use_flag" {
+		} else if v == "use_flag" {
 			save[v] = util.IntAll(tmp[v])
-		}else {
+		} else {
 			save[v] = tmp[v]
 		}
 	}
@@ -427,7 +551,7 @@ func InfoStd(tmp map[string]interface{})  {
 				code = code[:6]
 			}
 			if city := AddressMap[code]; city != nil { //未作废中取
-				if city.Province != "" && city.Province == save["company_area"] {
+				if city.Province != "" && strings.Contains(city.Province, util.ObjToString(save["company_area"])) {
 					if city.City != "" {
 						save["company_city"] = city.City //市
 					}
@@ -467,7 +591,7 @@ func InfoStd(tmp map[string]interface{})  {
 		if phone := util.ObjToString(save["company_phone"]); phone != "" {
 			if len(phone) == 11 {
 				types = append(types, "手机号")
-			}else {
+			} else {
 				types = append(types, "固定电话")
 			}
 		}
@@ -477,18 +601,18 @@ func InfoStd(tmp map[string]interface{})  {
 
 		companyName := util.ObjToString(tmp["company_name"])
 		// redis bid_unittype、bid_purchasing、bid_projectname
-		if b, err := redis.Exists("qyxy_winner", companyName); err == nil && b {
+		if b, err := redis.Exists("qyxy", companyName); err == nil && b {
 			maps := make(map[string]interface{})
-			text := redis.GetStr("qyxy_winner", companyName)
+			text := redis.GetStr("qyxy", companyName)
 			err1 := json.Unmarshal([]byte(text), &maps)
 			if err1 != nil {
 				util.Debug(companyName, "winner-----map解析异常")
-			}else {
-				for k := range maps{
+			} else {
+				for k := range maps {
 					if k == "bid_contracttype" {
 						t1 := util.ObjArrToStringArr(maps[k].([]interface{}))
 						types = append(types, t1...)
-					}else {
+					} else {
 						save[k] = maps[k]
 					}
 				}
@@ -500,7 +624,7 @@ func InfoStd(tmp map[string]interface{})  {
 		save["bid_contracttype"] = types
 		// bid_unittype
 		flag := false
-		for _, v := range WordsArr{
+		for _, v := range WordsArr {
 			if strings.Contains(util.ObjToString(tmp["business_scope"]), v) {
 				flag = true
 				break
@@ -516,12 +640,12 @@ func InfoStd(tmp map[string]interface{})  {
 				name := util.ObjToString(save["company_name"])
 				if strings.Contains(t1, "有限合伙") {
 					save["search_type"] = "有限合伙"
-				}else if strings.Contains(t1, "合伙") {
+				} else if strings.Contains(t1, "合伙") {
 					save["search_type"] = "普通合伙"
-				}else if strings.Contains(name, "股份") ||
+				} else if strings.Contains(name, "股份") ||
 					(strings.Contains(t1, "上市") && !strings.Contains(t1, "非上市")) {
 					save["search_type"] = "股份有限公司"
-				}else {
+				} else {
 					save["search_type"] = "有限责任公司"
 				}
 			}
@@ -546,10 +670,10 @@ func historynamefunMysql(tmp map[string]interface{}) {
 	info := MysqlTool.SelectBySql(query, util.ObjToString(tmp["_id"]))
 	if len(*info) > 0 {
 		var names []string
-		for _, v := range *info{
+		for _, v := range *info {
 			names = append(names, util.ObjToString(v["history_name"]))
 		}
-		tmp["history_name"] =  strings.Join(names, ",")
+		tmp["history_name"] = strings.Join(names, ",")
 	}
 }
 func historyNameFun(tmp map[string]interface{}) {
@@ -557,36 +681,36 @@ func historyNameFun(tmp map[string]interface{}) {
 	info, b := MongoTool1.Find("company_history_name", bson.M{"company_id": tmp["_id"]}, nil, field, false, -1, -1)
 	if b && len(*info) > 0 {
 		var names []string
-		for _, v := range *info{
+		for _, v := range *info {
 			names = append(names, util.ObjToString(v["history_name"]))
 		}
-		tmp["history_name"] =  strings.Join(names, ",")
+		tmp["history_name"] = strings.Join(names, ",")
 	}
 }
 
 // company_employee 高管
-func employeeFunMySql(tmp map[string]interface{})  {
+func employeeFunMySql(tmp map[string]interface{}) {
 	query := "SELECT employee_name, position, is_history FROM company_employee WHERE company_id=?"
 	info := MysqlTool.SelectBySql(query, util.ObjToString(tmp["_id"]))
 	if len(*info) > 0 {
 		var names []string
-		tmp["employees"] =  *info
-		for _, v := range *info{
+		tmp["employees"] = *info
+		for _, v := range *info {
 			names = append(names, util.ObjToString(v["employee_name"]))
 		}
-		tmp["employee_name"] =  strings.Join(names, ",")
+		tmp["employee_name"] = strings.Join(names, ",")
 	}
 }
-func employeeFun(tmp map[string]interface{})  {
+func employeeFun(tmp map[string]interface{}) {
 	field := bson.M{"employee_name": 1, "position": 1, "is_history": 1}
 	info, b := MongoTool1.Find("company_employee", bson.M{"company_id": tmp["_id"]}, nil, field, false, -1, -1)
 	if b && len(*info) > 0 {
 		var names []string
-		tmp["employees"] =  *info
-		for _, v := range *info{
+		tmp["employees"] = *info
+		for _, v := range *info {
 			names = append(names, util.ObjToString(v["employee_name"]))
 		}
-		tmp["employee_name"] =  strings.Join(names, ",")
+		tmp["employee_name"] = strings.Join(names, ",")
 	}
 }
 
@@ -596,13 +720,13 @@ func partnerFunMySql(tmp map[string]interface{}) {
 	info := MysqlTool.SelectBySql(query, util.ObjToString(tmp["_id"]))
 	if len(*info) > 0 {
 		var names []string
-		tmp["partners"] =  *info
-		for _, v := range *info{
+		tmp["partners"] = *info
+		for _, v := range *info {
 			if util.IntAll(tmp["is_history"]) == 0 {
 				names = append(names, util.ObjToString(v["stock_name"]))
 			}
 		}
-		tmp["stock_name"] =  strings.Join(names, ",")
+		tmp["stock_name"] = strings.Join(names, ",")
 	}
 }
 func partnerFun(tmp map[string]interface{}) {
@@ -610,13 +734,13 @@ func partnerFun(tmp map[string]interface{}) {
 	info, b := MongoTool1.Find("company_partner", bson.M{"company_id": tmp["_id"]}, nil, field, false, -1, -1)
 	if b && len(*info) > 0 {
 		var names []string
-		tmp["partners"] =  *info
-		for _, v := range *info{
+		tmp["partners"] = *info
+		for _, v := range *info {
 			if util.IntAll(tmp["is_history"]) == 0 {
 				names = append(names, util.ObjToString(v["stock_name"]))
 			}
 		}
-		tmp["stock_name"] =  strings.Join(names, ",")
+		tmp["stock_name"] = strings.Join(names, ",")
 	}
 }
 
@@ -625,10 +749,10 @@ func reportFunMysql(tmp map[string]interface{}) {
 	query := "SELECT report_year, company_phone, company_email, zip_code, employee_no, operator_name FROM annual_report_base WHERE company_id=?"
 	info := MysqlTool.SelectBySql(query, util.ObjToString(tmp["_id"]))
 	if len(*info) > 0 {
-		tmp["annual_reports"] =  *info
+		tmp["annual_reports"] = *info
 		year := 0
 		phone, email := "", ""
-		for _, v := range *info{
+		for _, v := range *info {
 			if year < util.IntAll(v["report_year"]) {
 				year = util.IntAll(v["report_year"])
 				phone = util.ObjToString(v["company_phone"])
@@ -645,10 +769,10 @@ func reportFun(tmp map[string]interface{}) {
 	field := bson.M{"report_year": 1, "company_phone": 1, "company_email": 1, "zip_code": 1, "employee_no": 1, "operator_name": 1}
 	info, b := MongoTool1.Find("annual_report_base", bson.M{"company_id": tmp["_id"]}, nil, field, false, -1, -1)
 	if b && len(*info) > 0 {
-		tmp["annual_reports"] =  *info
+		tmp["annual_reports"] = *info
 		year := 0
 		phone, email := "", ""
-		for _, v := range *info{
+		for _, v := range *info {
 			if year < util.IntAll(v["report_year"]) {
 				year = util.IntAll(v["report_year"])
 				phone = util.ObjToString(v["company_phone"])
@@ -669,14 +793,14 @@ func websiteFunMySql(tmp map[string]interface{}) {
 	if len(*info) > 0 {
 		year := 0
 		web := ""
-		for _, v := range *info{
+		for _, v := range *info {
 			if year < util.IntAll(v["report_year"]) && util.IntAll(tmp["is_history"]) == 0 {
 				year = util.IntAll(v["report_year"])
 				web = util.ObjToString(v["website_url"])
 			}
 		}
 		if year != 0 {
-			tmp["website_url"] =  web
+			tmp["website_url"] = web
 		}
 	}
 }
@@ -686,14 +810,14 @@ func websiteFun(tmp map[string]interface{}) {
 	if b && len(*info) > 0 {
 		year := 0
 		web := ""
-		for _, v := range *info{
+		for _, v := range *info {
 			if year < util.IntAll(v["report_year"]) && util.IntAll(tmp["is_history"]) == 0 {
 				year = util.IntAll(v["report_year"])
 				web = util.ObjToString(v["website_url"])
 			}
 		}
 		if year != 0 {
-			tmp["website_url"] =  web
+			tmp["website_url"] = web
 		}
 	}
 }
@@ -722,4 +846,58 @@ func getStName(name string) string {
 	return name2
 }
 
+func taskinfo(tmp map[string]interface{}) {
+	q := map[string]interface{}{"company_id": tmp["_id"]}
+	info, _ := MongoTool1.FindOne("company_base", q)
+	if len(*info) > 0 {
+		InfoStd(*info)
+	}
+}
 
+func taskinfo1() {
+	//updataMap := make(map[string]interface{})
+	//updataMap["company_area"] = "内蒙古"
+	//for i, field := range AreaFiled {
+	//	if tmp[field] == nil {
+	//		continue
+	//	}
+	//	if code := fmt.Sprint(tmp[field]); code != "" {
+	//		if i == 0 && len(code) >= 8 { //credit_no企业信用代码
+	//			code = code[2:8]
+	//		} else if i == 1 && len(code) >= 6 { //company_code注册号
+	//			code = code[:6]
+	//		}
+	//		if city := AddressMap[code]; city != nil {
+	//			if city.Province != "" && city.Province == util.ObjToString(updataMap["company_area"]) {
+	//				if city.City != "" {
+	//					updataMap["company_city"] = city.City //市
+	//				}
+	//				if city.District != "" {
+	//					updataMap["company_district"] = city.District //县
+	//				}
+	//				break
+	//			}
+	//		} else { //作废中取
+	//			if city := AddressOldMap[code]; city != nil {
+	//				if city.Province != "" && city.Province == util.ObjToString(updataMap["company_area"]) {
+	//					if city.City != "" {
+	//						updataMap["company_city"] = city.City //市
+	//					}
+	//					if city.District != "" {
+	//						updataMap["company_district"] = city.District //县
+	//					}
+	//				}
+	//				break
+	//			}
+	//		}
+	//	}
+	//}
+	//
+	//_, _ = client.Update().Index("qyxy_v2").Type("qyxy").Id(util.ObjToString(tmp["_id"])).
+	//	Doc(updataMap).Refresh(true).Do()
+	//updataInfo := []map[string]interface{}{
+	//	{"_id": tmp["_id"]},
+	//	{"$set": updataMap},
+	//}
+	//updatePool <- updataInfo
+}

qyxy/save_es/main.go

@@ -12,10 +12,10 @@ import (
 )
 
 var (
-	MongoTool			*mongodb.MongodbSim
-	Es              	*es.Elastic
-	EsSaveCache			chan map[string]interface{}
-	SP					chan bool
+	MongoTool   *mongodb.MongodbSim
+	Es          *es.Elastic
+	EsSaveCache chan map[string]interface{}
+	SP          chan bool
 
 	EsField = []string{"_id", "company_name", "history_name", "updatetime", "legal_person", "legal_person_certno", "credit_no", "org_code", "tax_code",
 		"company_code", "area_code", "company_area", "company_city", "company_district", "currency", "capital", "company_type", "company_status",
@@ -26,15 +26,14 @@ var (
 		"采购单位": "1",
 		"投标企业": "2",
 		"代理机构": "3",
-		"厂商": "4",
+		"厂商":   "4",
 	}
 	TypeMap1 = map[string]string{
 		"固定电话": "1",
-		"手机号": "2",
-		"邮箱": "3",
-		"不存在": "4",
+		"手机号":  "2",
+		"邮箱":   "3",
+		"不存在":  "4",
 	}
-
 )
 
 func init() {
@@ -42,15 +41,15 @@ func init() {
 		MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
 		Size:        10,
 		DbName:      "mixdata",
-		UserName: 	 "SJZY_RWESBid_Other",
-		Password:	 "SJZY@O17t8herB3B",
+		UserName:    "SJZY_RWESBid_Other",
+		Password:    "SJZY@O17t8herB3B",
 		//MongodbAddr: "192.168.3.207:27092",
 		//Size:        10,
 		//DbName:      "wjh",
 	}
 	MongoTool.InitPool()
 	Es = &es.Elastic{
-		S_esurl: "http://172.17.145.170:9800",				// http://172.17.145.170:9800
+		S_esurl: "http://172.17.145.170:9800", // http://172.17.145.170:9800
 		I_size:  10,
 	}
 	Es.InitElasticSize()
@@ -81,7 +80,7 @@ func main() {
 				<-ch
 				wg.Done()
 			}()
-			if  p1, ok := tmp["partners"].([]interface{}); ok {
+			if p1, ok := tmp["partners"].([]interface{}); ok {
 				p2 := p1[0].(map[string]interface{})
 				if p2["stock_capital"] != "" {
 					taskinfo(tmp)
@@ -97,9 +96,9 @@ func main() {
 	<-c
 }
 
-func taskinfo(tmp map[string]interface{})  {
+func taskinfo(tmp map[string]interface{}) {
 	esMap := make(map[string]interface{})
-	for _, v := range EsField{
+	for _, v := range EsField {
 		if tmp[v] != nil {
 			esMap[v] = tmp[v]
 		}
@@ -110,21 +109,21 @@ func taskinfo(tmp map[string]interface{})  {
 	if company_type == "个体工商户" {
 		if len([]rune(company_name)) >= 5 {
 			esMap["company_type_int"] = 31
-		}else {
+		} else {
 			esMap["company_type_int"] = 32
 		}
-	}else if company_type == "其他" || company_type == "" {
+	} else if company_type == "其他" || company_type == "" {
 		if len([]rune(company_name)) >= 4 {
 			esMap["company_type_int"] = 21
-		}else {
+		} else {
 			esMap["company_type_int"] = 22
 		}
-	}else {
+	} else {
 		if company_type == "内资分公司" {
 			esMap["company_type_int"] = 12
-		}else if len([]rune(company_name)) >= 4 {
+		} else if len([]rune(company_name)) >= 4 {
 			esMap["company_type_int"] = 11
-		}else {
+		} else {
 			esMap["company_type_int"] = 13
 		}
 	}
@@ -142,34 +141,34 @@ func taskinfo(tmp map[string]interface{})  {
 	}
 	if t1, ok := tmp["bid_unittype"].([]interface{}); ok {
 		var arr []string
-		for _, v := range util.ObjArrToStringArr(t1){
+		for _, v := range util.ObjArrToStringArr(t1) {
 			arr = append(arr, TypeMap[v])
 		}
 		esMap["bid_unittype"] = strings.Join(arr, ",")
 	}
 	if t2, ok := tmp["bid_contracttype"].([]interface{}); ok {
 		var arr []string
-		for _, v := range util.ObjArrToStringArr(t2){
+		for _, v := range util.ObjArrToStringArr(t2) {
 			arr = append(arr, TypeMap1[v])
 		}
 		esMap["bid_contracttype"] = strings.Join(arr, ",")
 	}
 	if p1, ok := tmp["partners"].([]interface{}); ok {
-		for _, v := range p1{
+		for _, v := range p1 {
 			v1 := v.(map[string]interface{})
 			if text := util.ObjToString(v1["stock_capital"]); text != "" {
 				if c1, err := strconv.ParseFloat(text, 64); err == nil {
-					c1, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", c1), 64)	//保留小数点两位
+					c1, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", c1), 64) //保留小数点两位
 					v1["stock_capital"] = c1
-				}else {
+				} else {
 					delete(v1, "stock_capital")
 				}
 			}
 			if text := util.ObjToString(v1["stock_realcapital"]); text != "" {
 				if c2, err := strconv.ParseFloat(text, 64); err == nil {
-					c2, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", c2), 64)	//保留小数点两位
+					c2, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", c2), 64) //保留小数点两位
 					v1["stock_realcapital"] = c2
-				}else {
+				} else {
 					delete(v1, "stock_realcapital")
 				}
 			}

qyxy/save_mgo/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"github.com/go-redis/redis"
 	"go.mongodb.org/mongo-driver/bson"
@@ -10,36 +9,43 @@ import (
 	"qfw/util"
 	"reflect"
 	"strconv"
+	"strings"
 	"time"
 )
 
 var (
-	MongoTool		*mongodb.MongodbSim
-	rdb 			*redis.Client
+	MongoTool *mongodb.MongodbSim
+	rdb       *redis.Client
 )
 
 func init() {
+	//MongoTool = &mongodb.MongodbSim{
+	//	MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
+	//	Size:        10,
+	//	DbName:      "mixdata",
+	//	UserName:    "SJZY_RWESBid_Other",
+	//	Password:    "SJZY@O17t8herB3B",
+	//}
+	//MongoTool.InitPool()
 	MongoTool = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
+		MongodbAddr: "172.17.4.85:27080",
 		Size:        10,
-		DbName:      "mixdata",
-		UserName: 	 "SJZY_RWESBid_Other",
-		Password:	 "SJZY@O17t8herB3B",
+		DbName:      "qfw",
 	}
 	MongoTool.InitPool()
 }
 
 func initRedis() (err error) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second * 10)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 	defer cancel()
 	rdb = redis.NewClient(&redis.Options{
-		Addr: "127.0.0.1:8379",
+		Addr:     "127.0.0.1:8379",
 		PoolSize: 200,
-		DB: 3,
+		DB:       5,
 	})
 	_, err = rdb.Ping(ctx).Result()
-	if err !=nil{
-		fmt.Println("ping redis failed err:",err)
+	if err != nil {
+		fmt.Println("ping redis failed err:", err)
 		return err
 	}
 	return nil
@@ -47,8 +53,8 @@ func initRedis() (err error) {
 
 func main() {
 	err := initRedis()
-	if err !=nil{
-		fmt.Println("init redis failed err :",err)
+	if err != nil {
+		fmt.Println("init redis failed err :", err)
 		return
 	}
 	ctx := context.Background()
@@ -56,26 +62,31 @@ func main() {
 	var keys []string
 	n := 0
 	for {
-		keys, cursor, err = rdb.Scan(ctx, cursor,"*",500).Result()
-		if err !=nil{
-			fmt.Println("scan keys failed err:",err)
+		keys, cursor, err = rdb.Scan(ctx, cursor, "*", 500).Result()
+		if err != nil {
+			fmt.Println("scan keys failed err:", err)
 			return
 		}
 		util.Debug("---keys---", len(keys))
 		n += len(keys)
-		for _,key := range keys{
+		for _, key := range keys {
 			val, err := rdb.Get(ctx, key).Result()
-			if err != nil{
+			if err != nil {
 				fmt.Println("get key values failed err:", err)
 				return
 			}
-			val, _ = strconv.Unquote(val)						//  处理json字符串带转义符号
-			maps := make(map[string]interface{})
-			err1 := json.Unmarshal([]byte(val), &maps)
-			if err1 != nil {
-				util.Debug("-----map解析异常")
-			}
-			taskinfo(ctx, key, maps)
+
+			val, _ = strconv.Unquote(val)
+			arr := strings.Split(key, ",")
+			MongoTool.Save("first_cooperation", map[string]interface{}{"buyer": arr[0], "winner": arr[1], "project_id": val})
+
+			//val, _ = strconv.Unquote(val) //  处理json字符串带转义符号
+			//maps := make(map[string]interface{})
+			//err1 := json.Unmarshal([]byte(val), &maps)
+			//if err1 != nil {
+			//	util.Debug("-----map解析异常")
+			//}
+			//taskinfo(ctx, key, maps)
 		}
 		util.Debug("current---", n, cursor)
 		if cursor == 0 {
@@ -85,7 +96,7 @@ func main() {
 	}
 }
 
-func taskinfo(ctx context.Context, name string, tmp map[string]interface{})  {
+func taskinfo(ctx context.Context, name string, tmp map[string]interface{}) {
 	q := bson.M{"company_name": name}
 	info, b := MongoTool.FindOneByField("qyxy_std", q, nil)
 	if b && len(*info) > 0 {
@@ -95,8 +106,9 @@ func taskinfo(ctx context.Context, name string, tmp map[string]interface{})  {
 			t2 = append(t2, t1...)
 			tmp["bid_unittype"] = Duplicate(t2)
 		}
+		tmp["updatetime"] = time.Now().Unix()
 		MongoTool.Update("qyxy_std", bson.M{"_id": (*info)["_id"]}, bson.M{"$set": tmp}, true, false)
-	}else {
+	} else {
 		tmp["company_name"] = name
 		MongoTool.Save("qyxy_std_err", tmp)
 		rdb.Del(ctx, name)

qyxy/winner_tag/main.go

@@ -0,0 +1,169 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"mongodb"
+	"qfw/util"
+	"qfw/util/elastic"
+	"strings"
+	"time"
+)
+
+var (
+	Mgo						*mongodb.MongodbSim
+	Es						*elastic.Elastic
+
+	updatePool            	chan []map[string]interface{}
+	updateSp              	chan bool
+)
+
+func init() {
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",
+		Size:        10,
+		DbName:      "mixdata",
+		UserName:	 "SJZY_RWESBid_Other",
+		Password: 	 "SJZY@O17t8herB3B",
+	}
+	Mgo.InitPool()
+	Es = &elastic.Elastic{
+		S_esurl: "http://172.17.145.170:9800",									//http://172.17.145.170:9800
+		I_size:  10,
+	}
+	Es.InitElasticSize()
+
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp = make(chan bool, 5)
+}
+
+func main() {
+	go updateMethod()
+
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+
+	//q := map[string]interface{}{"_id": mongodb.StringTOBsonId("5e659c142c27dc56292df860")}
+	//q := bson.M{"extracttype": -1}
+	filed := map[string]interface{}{"company_name": 1}
+	query := sess.DB("mixdata").C("winner_enterprise").Find(nil).Select(filed).Iter()
+
+	es := `{
+	  "query": {
+		"bool": {
+		  "must": [],
+		  "must_not": [],
+		  "should": [
+			{
+			  "multi_match": {
+				"query": %s,
+				"type": "phrase",
+				"fields": [
+				  "s_winner.mwinner"
+				]
+			  }
+			}
+		  ]
+		}
+	  },
+	  "aggs": {
+		"my_name": {
+		  "terms": {
+			"field": "s_subscopeclass",
+			"size": 100
+		  }
+		}
+	  },
+	  "size": 0,
+	  "sort": [],
+	  "facets": {}
+	}`
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%200 == 0 {
+			util.Debug("current ---", count)
+		}
+		func(tmp map[string]interface{}) {
+
+			name := util.ObjToString(tmp["company_name"])
+			esquery := fmt.Sprintf(es, `"` + name + `"`)
+
+			searchResult, _ := client.Search().Index("bidding").Type("bidding").Source(esquery).Do()
+			var result map[string]interface{}
+			mapCount := make(map[string]int)
+			b, _ := json.Marshal(searchResult.Aggregations)
+			_ = json.Unmarshal(b, &result)
+			bk, _ := result["my_name"].(map[string]interface{})
+			if arr, ok := bk["buckets"].([]interface{}); ok {
+				for _, v := range arr{
+					v1 := v.(map[string]interface{})
+					key := strings.Split(util.ObjToString(v1["key"]), "_")[0]
+					mapCount[key] += util.IntAll(v1["doc_count"])
+				}
+			}
+			maxCount := 0
+			maxStr := ""
+			for k, v := range mapCount{
+				if v > maxCount {
+					maxCount = v
+					maxStr = k
+				}else if v == maxCount {
+					if maxStr == "" {
+						maxStr = k
+					}else {
+						maxStr = maxStr + "," + k
+					}
+				}
+			}
+			if maxCount != 0 {
+				updatePool <- []map[string]interface{}{
+					{"_id": tmp["_id"]},
+					{"$set": map[string]interface{}{"tag_business": maxStr}},
+				}
+			}
+		}(tmp)
+		tmp = map[string]interface{}{}
+	}
+	util.Debug("over ---", count)
+
+	c := make(chan bool, 1)
+	<-c
+}
+
+func updateMethod() {
+	arru := make([][]map[string]interface{}, 200)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					Mgo.UpSertBulk("winner_enterprise", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					Mgo.UpSertBulk("winner_enterprise", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		}
+	}
+}

qyxy/winner_tag/tools.go

@@ -0,0 +1 @@
+package main

src/jy/util/config.go

@@ -11,11 +11,12 @@ import (
 //全局配置
 var FormatTextMap map[string][]map[string]interface{}
 var JsonData []string
+
 func init() {
 	loadFormatText()
 	//LoadTagDb("./res/tagdb")
 	LoadTagDb("./res/blocktagdb")
-	util.ReadConfig("./res/jsondata.json",&JsonData)
+	util.ReadConfig("./res/jsondata.json", &JsonData)
 }
 
 //加载格式化正文配置

udpcreateindex/src/biddingall.go

@@ -1,9 +1,10 @@
 package main
 
 import (
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
 	"mongodb"
 	"qfw/util/elastic"
-	"qfw/util/redis"
 	"regexp"
 	"time"
 	//"fmt"
@@ -13,8 +14,6 @@ import (
 	"reflect"
 	"strings"
 	"sync"
-
-	"gopkg.in/mgo.v2/bson"
 )
 
 //对字段处理 bidamount  budget
@@ -76,7 +75,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 	query := session.DB(db).C(c).Find(q).Select(bson.M{
 		"projectinfo.attachment": 0,
 		"contenthtml":            0,
-		"publishdept":			  0,		// 6.30		迭代报错，字段值乱码
+		"publishdept":            0, // 6.30		迭代报错，字段值乱码
 	}).Sort("_id").Iter()
 	//查询抽取结果
 	extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Sort("_id").Iter()
@@ -173,7 +172,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 			defer func() {
 				<-mpool
 			}()
-			if !bnil && compare != nil {
+			if !bnil && compare != nil && len(compare) > 0 {
 				subscopeclass, _ := compare["subscopeclass"].([]interface{}) //subscopeclass
 				if subscopeclass != nil {
 					m1 := map[string]bool{}
@@ -208,17 +207,20 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				//				if winner != "" {
 				//					m1[winner] = true
 				//				}
-				//				package1 := compare["package"]
-				//				if package1 != nil {
-				//					packageM, _ := package1.(map[string]interface{})
-				//					for _, p := range packageM {
-				//						pm, _ := p.(map[string]interface{})
-				//						pw, _ := pm["winner"].(string)
-				//						if pw != "" {
-				//							m1[pw] = true
-				//						}
-				//					}
-				//				}
+				// 多包标记
+				if package1 := compare["package"]; package1 != nil {
+					packageM, _ := package1.(map[string]interface{})
+					for _, p := range packageM {
+						pm, _ := p.(map[string]interface{})
+						if qutil.ObjToString(pm["winner"]) != "" || qutil.Float64All(pm["budget"]) > 0 ||
+							qutil.Float64All(pm["bidamount"]) > 0 {
+							update["multipackage"] = int32(1)
+							break
+						}
+					}
+				} else {
+					update["multipackage"] = int32(0)
+				}
 				compare = nil
 				//				if len(m1) > 0 {
 				//					//str := ","
@@ -229,46 +231,64 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				//					}
 				//					update["s_winner"] = strings.Join(winnerarr, ",")
 				//				}
+			} else {
+				area := qutil.ObjToString(tmp["area"])
+				city := qutil.ObjToString(tmp["city"])
+				district := qutil.ObjToString(tmp["district"])
+				UpdatesLock.Lock()
+				rdata := standardCheckCity(area, city, district)
+				UpdatesLock.Unlock()
+				if len(rdata) > 0 {
+					for k, v := range rdata {
+						update[k] = v
+					}
+				}
 			}
 			//------------------对比结束
 			//同时保存到elastic
 			for tk, tv := range update {
 				tmp[tk] = tv
 			}
+			tmp_up := []map[string]interface{}{}
+			extupdate := make(map[string]interface{})
 			if tmp["s_winner"] != "" {
-				sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
+				//sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
 				var cid []string
-				for _, w := range sWinnerarr {
-					if w != "" {
-						id := redis.GetStr("qyxy_id", w)
-						if id == "" {
-							ents, _ := mgostandard.Find("qyxy_std", bson.M{"company_name": w}, bson.M{"updatetime": -1}, nil, false, -1, -1)
-							if len(*ents) > 0 {
-								id = qutil.ObjToString((*ents)[0]["_id"])
-								redis.PutCKV("qyxy_id", w, id)
-							}else {
-								ent, _ := qyxydb.FindOne("company_history_name", bson.M{"company_name": w})
-								if len(*ent) > 0 {
-									id = qutil.ObjToString((*ent)["company_id"])
-									redis.PutCKV("qyxy_id", w, id)
-								}
-							}
-						}
-						if id == "" {
-							id = "-"
-						}
-						cid = append(cid, id)
-					}
-				}
+				//for _, w := range sWinnerarr {
+				//	if w != "" {
+				//		id := redis.GetStr("qyxy_id", w)
+				//		if id == "" {
+				//			ents, _ := mgostandard.Find("qyxy_std", map[string]interface{}{"company_name": w}, map[string]interface{}{"updatetime": -1}, nil, false, -1, -1)
+				//			if len(*ents) > 0 {
+				//				id = qutil.ObjToString((*ents)[0]["_id"])
+				//			}else {
+				//				ent, _ := qyxydb.FindOne("company_history_name", map[string]interface{}{"history_name": w})
+				//				if len(*ent) > 0 {
+				//					id = qutil.ObjToString((*ent)["company_id"])
+				//				}
+				//			}
+				//		}
+				//		if id == "" {
+				//			id = "-"
+				//		}
+				//		cid = append(cid, id)
+				//	}
+				//}
 				if len(cid) > 0 {
 					tmp["entidlist"] = cid
 					update["entidlist"] = cid
-					tmp_up := []map[string]interface{}{}
-					tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
-					tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
-					UpdataMgoCache <- tmp_up
+					extupdate["entidlist"] = cid
 				}
 			}
+			// 临时
+			//if tmp["purchasing_tag"] != nil {
+			//	extupdate["purchasing_tag"] = tmp["purchasing_tag"]
+			//}
+			if len(extupdate) > 0 {
+				tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
+				tmp_up = append(tmp_up, map[string]interface{}{"$set": extupdate})
+				UpdataMgoCache <- tmp_up
+			}
 			//对projectscope字段的索引处理
 			ps, _ := tmp["projectscope"].(string)
 			if len(ps) > ESLEN {
@@ -309,8 +329,8 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 			UpdatesLock.Lock()
 			if qutil.IntAll(update["extracttype"]) != -1 {
 				newTmp := map[string]interface{}{}
-				for field, ftype := range biddingIndexFieldsMap { //
-					if tmp[field] != nil && del[field] == nil { //
+				for field, ftype := range biddingIndexFieldsMap {
+					if tmp[field] != nil && del[field] == nil {
 						//qutil.Debug(field, tmp[field], reflect.TypeOf(tmp[field]).String(), ftype)
 						if field == "projectinfo" {
 							mp, _ := tmp[field].(map[string]interface{})
@@ -383,7 +403,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 							}
 						} else if field == "review_experts" {
 							// 评审专家
-							if arr, ok :=tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
+							if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
 								arr1 := qutil.ObjArrToStringArr(arr)
 								newTmp[field] = strings.Join(arr1, ",")
 							}
@@ -392,11 +412,11 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 						} else if field == "bidopentime" {
 							if tmp[field] != nil && tmp["bidendtime"] == nil {
 								newTmp["bidendtime"] = tmp[field]
-								newTmp[field] = tmp["bidendtime"]
-							}else if tmp[field] == nil && tmp["bidendtime"] != nil {
+								newTmp[field] = tmp[field]
+							} else if tmp[field] == nil && tmp["bidendtime"] != nil {
 								newTmp["bidendtime"] = tmp[field]
 								newTmp[field] = tmp["bidendtime"]
-							}else {
+							} else {
 								if tmp["bidopentime"] != nil {
 									newTmp[field] = tmp["bidopentime"]
 								}
@@ -406,11 +426,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 							if len([]rune(detail)) > detailLength {
 								detail = detail[:detailLength]
 							}
-							if strings.Contains(qutil.ObjToString(tmp["title"]), detail) {
-								newTmp[field] = FilterDetail(detail)
-							}else {
-								newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
-							}
+							newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
 						} else if field == "_id" || field == "topscopeclass" { //不做处理
 							newTmp[field] = tmp[field]
 						} else if field == "publishtime" || field == "comeintime" {
@@ -431,7 +447,6 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 						}
 					}
 				}
-
 				YuceEndtime(newTmp)
 				arrEs = append(arrEs, newTmp)
 			}
@@ -532,3 +547,125 @@ func UpdateExtract() {
 		}
 	}
 }
+
+//城市标准校验
+func standardCheckCity(area string, city string, district string) map[string]string {
+	rdata := make(map[string]string)
+	if area == "香港" || area == "澳门" || area == "台湾" || (area == "全国" && (city == "" && district == "")) {
+		return rdata
+	}
+	//第一步：区校验
+	if district != "" {
+		districtArr := DistrictDict[district]
+		if districtArr == nil { //涉及了 个别别名相关的数据
+			trim_arr := aliasDataDistrict(district) //拆分后缀
+			if len(trim_arr) > 0 {
+				for _, alias_district := range trim_arr {
+					alias_districtArr := DistrictDict[alias_district]
+					for _, v := range alias_districtArr {
+						if city == v.C_Name && area == v.P_Name {
+							rdata["district"] = alias_district
+							return rdata
+						}
+					}
+				}
+			}
+			rdata["district"] = ""
+		} else {
+			isTrue := false
+			for _, v := range districtArr {
+				if city == v.C_Name && area == v.P_Name {
+					isTrue = true
+					break
+				}
+			}
+			if isTrue { //完全匹配
+				return rdata
+			} else { //未完全匹配
+				if len(districtArr) == 1 {
+					rdata["area"] = districtArr[0].P_Name
+					rdata["city"] = districtArr[0].C_Name
+					rdata["district"] = districtArr[0].D_Name
+					return rdata
+				} else {
+					rdata["district"] = ""
+				}
+			}
+		}
+	}
+
+	//第二步：区校验-失败   市-校验
+	if city != "" {
+		cityArr := CityDict[city]
+		if cityArr == nil {
+			//把市当成区，匹配三级   - 存在优化空间- city:郑州  别名
+			districtArr := DistrictDict[city]
+			for _, v := range districtArr {
+				if city == v.C_Name && area == v.P_Name {
+					rdata["area"] = districtArr[0].P_Name
+					rdata["city"] = districtArr[0].C_Name
+					rdata["district"] = districtArr[0].D_Name
+					return rdata
+				}
+			}
+			rdata["city"] = ""
+		} else {
+			isTrue := false
+			for _, v := range cityArr {
+				if area == v.P_Name {
+					isTrue = true
+					break
+				}
+			}
+			if isTrue { //完全匹配
+				return rdata
+			} else { //未完全匹配
+				if len(cityArr) == 1 {
+					rdata["area"] = cityArr[0].P_Name
+					rdata["city"] = cityArr[0].C_Name
+					rdata["district"] = ""
+					return rdata
+				} else {
+					rdata["city"] = ""
+				}
+			}
+		}
+	}
+
+	//第三步：省份校验
+	if ProvinceDict[area] == nil {
+		rdata["area"] = "全国"
+		rdata["city"] = ""
+		rdata["district"] = ""
+	}
+
+	return rdata
+}
+
+var cityEndReg = regexp.MustCompile("(区|县|市)$")
+
+//拆分三级县
+func aliasDataDistrict(district string) []string {
+	arr := []string{}
+	if cityEndReg.MatchString(district) {
+		str := cityEndReg.FindString(district)
+		strings.TrimRight(district, str)
+		if str == "县" {
+			arr = append(arr, fmt.Sprintf("%s区", strings.TrimRight(district, str)))
+			arr = append(arr, fmt.Sprintf("%s市", strings.TrimRight(district, str)))
+		} else if str == "区" {
+			arr = append(arr, fmt.Sprintf("%s县", strings.TrimRight(district, str)))
+			arr = append(arr, fmt.Sprintf("%s市", strings.TrimRight(district, str)))
+		} else if str == "市" {
+			arr = append(arr, fmt.Sprintf("%s县", strings.TrimRight(district, str)))
+			arr = append(arr, fmt.Sprintf("%s区", strings.TrimRight(district, str)))
+		} else {
+
+		}
+	} else { //未找到 district- 区县市  例： district : 金水
+		arr = append(arr, fmt.Sprintf("%s区", district))
+		arr = append(arr, fmt.Sprintf("%s县", district))
+		arr = append(arr, fmt.Sprintf("%s市", district))
+	}
+	return arr
+}

udpcreateindex/src/biddingdata.go

@@ -1,62 +1,21 @@
 package main
 
 import (
-	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
 	elastic "qfw/util/elastic"
+	"reflect"
 	"strings"
-	"sync"
 
-	"gopkg.in/mgo.v2/bson"
+	//	"strings"
+	"sync"
 )
 
-//对字段处理 bidamount  budget
-var indexfield = []string{
-	"_id",
-	"s_winner",
-	"winner",
-	"buyerclass",
-	"title",
-	"detail",
-	"area",
-	"site",
-	"bidopendate",
-	"bidopentime",
-	"buyer",
-	"city",
-	"comeintime",
-	"href",
-	"infoformat",
-	"projectcode",
-	"projectname",
-	"publishtime",
-	"s_sha",
-	"spidercode",
-	"subtype",
-	"toptype",
-	"agency",
-	"budget",
-	"bidamount",
-	"s_subscopeclass",
-	"projectscope",
-	"bidstatus",
-	"projectinfo",
-	"buyertel",
-	"buyerperson",
-	"projectid",
-	"buyerclass",
-	"district",
-	"topscopeclass",
-	"attachments",
-}
-
-//招标数据表和抽取表一一对应开始更新
 func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 	defer qutil.Catch()
-	thread := 40
-	var mpool = make(chan bool, thread)
+	qutil.Debug(mapInfo)
 	q, _ := mapInfo["query"].(map[string]interface{})
 	if q == nil {
 		q = map[string]interface{}{
@@ -69,258 +28,180 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 	//bidding库
 	session := mgo.GetMgoConn()
 	defer mgo.DestoryMongoConn(session)
-	//extract库
-	extractsession := extractmgo.GetMgoConn()
-	defer extractmgo.DestoryMongoConn(extractsession)
+	//连接信息
 	//连接信息
 	c, _ := mapInfo["coll"].(string)
 	if c == "" {
 		c, _ = bidding["collect"].(string)
 	}
-
-	extractc, _ := bidding["extractcollect"].(string)
 	db, _ := bidding["db"].(string)
-	extractdb, _ := bidding["extractdb"].(string)
-	index, _ := bidding["index"].(string)
-	itype, _ := bidding["type"].(string)
+	index := "bidding_all"
+	itype := "bidding"
 	count, _ := session.DB(db).C(c).Find(&q).Count()
-	fields := strings.Split(bidding["fields"].(string), ",")
 	//线程池
 	UpdatesLock := sync.Mutex{}
-
-	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	qutil.Debug("查询语句:", q, "同步总数:", count, "elastic库:", index)
 	//查询招标数据
 	query := session.DB(db).C(c).Find(q).Select(bson.M{
 		"projectinfo.attachment": 0,
 		"contenthtml":            0,
+		"publishdept":            0,
+		"detail":                 0,
+		"projectscope":           0,
+		"project_scale":          0,
+		"bidstatus":              0,
+		"china_bidding":          0,
 	}).Sort("_id").Iter()
-	//查询抽取结果
-	extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Sort("_id").Iter()
-
 	n := 0
-
 	//更新数组
-
 	arrEs := []map[string]interface{}{}
-	//对比两张表数据，减少查询次数
-	var compare bson.M
-	bnil := false
+	thread := 10
+	espool := make(chan bool, 5)
+	var mpool = make(chan bool, thread)
 	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
-		update := map[string]interface{}{}
-		//对比方法----------------
-		for {
-			if compare == nil {
-				compare = make(bson.M)
-				if !extractquery.Next(compare) {
-					break
-				}
-			}
-			if compare != nil {
-				//对比
-				cid := qutil.BsonIdToSId(compare["_id"])
-				tid := qutil.BsonIdToSId(tmp["_id"])
-				if cid == tid {
-					bnil = false
-					//更新bidding表，生成索引
-					for _, k := range fields {
-						v1 := compare[k] //extract
-						v2 := tmp[k]     //bidding
-						if v2 == nil && v1 != nil {
-							update[k] = v1
-						} else if v2 != nil && v1 != nil {
-							update[k] = v1
-						} else if v2 != nil && v1 == nil {
-							update[k] = v2
-						}
-					}
-					if qutil.IntAll(compare["repeat"]) == 1 {
-						update["extracttype"] = -1
-						//} else if qutil.IntAll(tmp["extracttype"]) == -1 {
-					} else {
-						update["extracttype"] = 1
-					}
-					break
-				} else {
-					if cid < tid {
-						bnil = false
-						compare = nil
-						continue
-					} else {
-						bnil = true
-						break
-					}
-				}
-			} else {
-				bnil = false
-				break
-			}
+		if n%20000 == 0 {
+			log.Println("current:", n, tmp["_id"])
 		}
-		//下面可以多线程跑的--->
-		//处理分类
 		mpool <- true
-		go func(tmp, update, compare map[string]interface{}, bnil bool) {
+		go func(tmp map[string]interface{}) {
 			defer func() {
 				<-mpool
 			}()
-			if !bnil && compare != nil {
-				subscopeclass, _ := compare["subscopeclass"].([]interface{})
-				if subscopeclass != nil {
-					m1 := map[string]bool{}
-					newclass := []string{}
-					for _, sc := range subscopeclass {
-						sclass, _ := sc.(string)
-						if !m1[sclass] {
-							m1[sclass] = true
-							newclass = append(newclass, sclass)
-						}
-					}
-					update["s_subscopeclass"] = strings.Join(newclass, ",")
-					update["subscopeclass"] = newclass
-				}
-				//处理中标企业
-				//				winner, _ := compare["winner"].(string)
-				//				m1 := map[string]bool{}
-				//				if winner != "" {
-				//					m1[winner] = true
-				//				}
-				//				package1 := compare["package"]
-				//				if package1 != nil {
-				//					packageM, _ := package1.(map[string]interface{})
-				//					for _, p := range packageM {
-				//						pm, _ := p.(map[string]interface{})
-				//						pw, _ := pm["winner"].(string)
-				//						if pw != "" {
-				//							m1[pw] = true
-				//						}
-				//					}
-				//				}
-				compare = nil
-				//				if len(m1) > 0 {
-				//					//str := ","
-				//					winnerarr := []string{}
-				//					for k, _ := range m1 {
-				//						//str += k + ","
-				//						winnerarr = append(winnerarr, k)
-				//					}
-				//					update["s_winner"] = strings.Join(winnerarr, ",")
-				//				}
-			}
-			//------------------对比结束
-
-			//同时保存到elastic
-			for tk, tv := range update {
-				tmp[tk] = tv
-			}
-			if tmp["supervisorrate"] != nil { //临时处理supervisorrate抽取类型为string不生索引
-				if _, ok := tmp["supervisorrate"].(string); ok { //supervisorrate数据为string类型
-					delete(tmp, "supervisorrate")
-				}
-			}
-			//对projectscope字段的索引处理
-			ps, _ := tmp["projectscope"].(string)
-			if len(ps) > ESLEN {
-				tmp["projectscope"] = string(([]rune(ps))[:4000])
-			}
-			//对标的物为空处理
-			if filetext := getFileText(tmp); len(filetext) > 0 { //attach_text
-				tmp["filetext"] = filetext
-			}
 			if purchasing, ok := tmp["purchasing"].(string); ok && purchasing == "" {
 				delete(tmp, "purchasing")
 			}
 			if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok && len(purchasinglist) == 0 {
 				delete(tmp, "purchasinglist")
 			}
-			//			if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
-			//				tmp["budget"] = nil
-			//			} else if sbd, ok := tmp["budget"].(string); ok {
-			//				tmp["budget"] = ObjToMoney([]interface{}{sbd, sbd})[0]
-			//			}
-			//			if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
-			//				tmp["bidamount"] = nil
-			//			} else if sbd, ok := tmp["bidamount"].(string); ok {
-			//				tmp["bidamount"] = ObjToMoney([]interface{}{sbd, sbd})[0]
-			//			}
-
-			if qutil.IntAll(update["extracttype"]) != -1 {
-				newTmp := map[string]interface{}{}
-				for _, v := range biddingIndexFields { // indexfield
-					if tmp[v] != nil {
-						if "projectinfo" == v {
-							mp, _ := tmp[v].(map[string]interface{})
-							if mp != nil {
-								newmap := map[string]interface{}{}
-								for _, v1 := range projectinfoFields {
-									if mp[v1] != nil {
-										newmap[v1] = fmt.Sprint(mp[v1])
+			//数据为空处理
+			for _, f := range []string{"bidstatus", "city", "district", "channel"} {
+				if fVal, ok := tmp[f].(string); ok && fVal == "" {
+					delete(tmp, f)
+				}
+			}
+			UpdatesLock.Lock()
+			newTmp := map[string]interface{}{}
+			for field, ftype := range biddingIndexFieldsMap {
+				if tmp[field] != nil { //
+					if field == "projectinfo" {
+						mp, _ := tmp[field].(map[string]interface{})
+						if mp != nil {
+							newmap := map[string]interface{}{}
+							for k, ktype := range projectinfoFieldsMap {
+								mpv := mp[k]
+								if mpv != nil && reflect.TypeOf(mpv).String() == ktype {
+									newmap[k] = mp[k]
+								}
+							}
+							if len(newmap) > 0 {
+								newTmp[field] = newmap
+							}
+						}
+					} else if field == "purchasinglist" { //标的物处理
+						purchasinglist_new := []map[string]interface{}{}
+						if pcl, _ := tmp[field].([]interface{}); len(pcl) > 0 {
+							for _, ls := range pcl {
+								lsm_new := make(map[string]interface{})
+								lsm := ls.(map[string]interface{})
+								for pf, pftype := range purchasinglistFieldsMap {
+									lsmv := lsm[pf]
+									if lsmv != nil && reflect.TypeOf(lsmv).String() == pftype {
+										lsm_new[pf] = lsm[pf]
 									}
 								}
-								if len(newmap) > 0 {
-									newTmp[v] = newmap
+								if lsm_new != nil && len(lsm_new) > 0 {
+									purchasinglist_new = append(purchasinglist_new, lsm_new)
 								}
-								// attachments := mp["attachments"]
-								// con := ""
-								// if attachments != nil {
-								// 	am, _ := attachments.(map[string]interface{})
-								// 	if am != nil {
-								// 		for _, v1 := range am {
-								// 			vm, _ := v1.(map[string]interface{})
-								// 			if vm != nil {
-								// 				c, _ := vm["content"].(string)
-								// 				con += c
-								// 			}
-								// 		}
-								// 	}
-								// }
-								// if con != "" {
-								// 	con = FilterDetailSpace(con)
-								// 	newTmp["attachments"] = con
-								// }
 							}
-						} else if v == "purchasinglist" { //标的物处理
-							purchasinglist_new := []map[string]interface{}{}
-							if pcl, _ := tmp[v].([]interface{}); len(pcl) > 0 {
-								for _, ls := range pcl {
-									lsm_new := make(map[string]interface{})
-									lsm := ls.(map[string]interface{})
-									for _, pf := range purchasinglistFields {
-										if lsm[pf] != nil {
-											lsm_new[pf] = lsm[pf]
+						}
+						if len(purchasinglist_new) > 0 {
+							newTmp[field] = purchasinglist_new
+						}
+					} else if field == "winnerorder" { //中标候选
+						winnerorder_new := []map[string]interface{}{}
+						if winnerorder, _ := tmp[field].([]interface{}); len(winnerorder) > 0 {
+							for _, win := range winnerorder {
+								winMap_new := make(map[string]interface{})
+								winMap := win.(map[string]interface{})
+								for wf, wftype := range winnerorderlistFieldsMap {
+									wfv := winMap[wf]
+									if wfv != nil && reflect.TypeOf(wfv).String() == wftype {
+										if wf == "sort" && qutil.Int64All(wfv) > 100 {
+											continue
 										}
-									}
-									if lsm_new != nil && len(lsm_new) > 0 {
-										purchasinglist_new = append(purchasinglist_new, lsm_new)
+										winMap_new[wf] = winMap[wf]
 									}
 								}
+								if winMap_new != nil && len(winMap_new) > 0 {
+									winnerorder_new = append(winnerorder_new, winMap_new)
+								}
 							}
-							if len(purchasinglist_new) > 0 {
-								newTmp[v] = purchasinglist_new
+						}
+						if len(winnerorder_new) > 0 {
+							newTmp[field] = winnerorder_new
+						}
+					} else if field == "qualifies" {
+						//项目资质
+						qs := []string{}
+						if q, _ := tmp[field].([]interface{}); len(q) > 0 {
+							for _, v := range q {
+								v1 := v.(map[string]interface{})
+								qs = append(qs, qutil.ObjToString(v1["key"]))
 							}
+						}
+						if len(qs) > 0 {
+							newTmp[field] = strings.Join(qs, ",")
+						}
+					} else if field == "review_experts" {
+						// 评审专家
+						if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
+							arr1 := qutil.ObjArrToStringArr(arr)
+							newTmp[field] = strings.Join(arr1, ",")
+						}
+					} else if field == "entidlist" {
+						newTmp[field] = tmp[field]
+					} else if field == "bidopentime" {
+						if tmp[field] != nil && tmp["bidendtime"] == nil {
+							newTmp["bidendtime"] = tmp[field]
+						}
+						if tmp[field] == nil && tmp["bidendtime"] != nil {
+							newTmp[field] = tmp["bidendtime"]
+						}
+					} else if field == "_id" || field == "topscopeclass" { //不做处理
+						newTmp[field] = tmp[field]
+					} else if field == "publishtime" || field == "comeintime" {
+						//字段类型不正确，特别处理
+						if tmp[field] != nil && qutil.Int64All(tmp[field]) > 0 {
+							newTmp[field] = qutil.Int64All(tmp[field])
+						}
+					} else if field == "s" {
+						newTmp[field] = tmp[field]
+					} else { //其它字段判断数据类型，不正确舍弃
+						if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {
+							continue
 						} else {
-							if v == "detail" {
-								detail, _ := tmp[v].(string)
-								newTmp[v] = FilterDetail(detail)
-							} else {
-								newTmp[v] = tmp[v]
+							if fieldval != "" {
+								newTmp[field] = fieldval
 							}
 						}
 					}
 				}
-				UpdatesLock.Lock()
-				arrEs = append(arrEs, newTmp)
-				UpdatesLock.Unlock()
 			}
-			UpdatesLock.Lock()
-			if len(arrEs) >= BulkSize-1 {
+			newTmp["extracttype"] = qutil.IntAll(tmp["extracttype"])
+			arrEs = append(arrEs, newTmp)
+			if len(arrEs) >= BulkSizeBack {
 				tmps := arrEs
-				elastic.BulkSave(index, itype, &tmps, true)
+				espool <- true
+				go func(tmps []map[string]interface{}) {
+					defer func() {
+						<-espool
+					}()
+					elastic.BulkSave(index, itype, &tmps, true)
+				}(tmps)
 				arrEs = []map[string]interface{}{}
 			}
 			UpdatesLock.Unlock()
-		}(tmp, update, compare, bnil)
-		if n%1000 == 0 {
-			log.Println("current:", n)
-		}
+		}(tmp)
 		tmp = make(map[string]interface{})
 	}
 	for i := 0; i < thread; i++ {
@@ -332,5 +213,5 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 		elastic.BulkSave(index, itype, &tmps, true)
 	}
 	UpdatesLock.Unlock()
-	log.Println(mapInfo, "create bidding index...over", n)
+	log.Println(mapInfo, "create biddingdata index...over", n)
 }

udpcreateindex/src/biddingdeletebyextract.go

@@ -1,13 +1,13 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
 
 	elastic "qfw/util/elastic"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 //根据抽取表repeat=1，删除es中重复数据

udpcreateindex/src/biddingdeletebyextracttype.go

@@ -1,13 +1,13 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
 
 	elastic "qfw/util/elastic"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 //根据bidding表extracttype=-1，删除es中重复数据

udpcreateindex/src/biddingindex.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"github.com/donnie4w/go-logger/logger"
 	"go.mongodb.org/mongo-driver/bson"
 	"mongodb"
 	"qfw/util/redis"
@@ -24,7 +23,7 @@ var date1 = regexp.MustCompile("20[0-2][0-9][年|\\-\\/|.][0-9]{1,2}[月|\\-|\\/
 //对字段处理 bidamount  budget
 //招标数据表和抽取表一一对应开始更新
 
-func biddingTask(data []byte, mapInfo map[string]interface{}) {
+func biddingTask(data []byte, mapInfo map[string]interface{}, tasktype string) {
 	defer qutil.Catch()
 	q, _ := mapInfo["query"].(map[string]interface{})
 	bkey, _ := mapInfo["bkey"].(string)
@@ -37,7 +36,7 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 		}
 	}
 
-	logger.SetRollingDaily("./logs", "id.log")
+	//logger.SetRollingDaily("./logs", "id.log")
 	//连接信息
 	c, _ := bidding["collect"].(string)
 	extractc, _ := bidding["extractcollect"].(string)
@@ -64,7 +63,7 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 	//bidding库
 	session := mgo.GetMgoConn()
 	count, _ := session.DB(db).C(c).Find(&q).Count()
-	log.Println("抽取表 重复数据量：", repeatCount)
+	log.Println("抽取表 重复数据量：", extCount, repeatCount)
 	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
 	n1, n2 := 0, 0
 	if count < 200000 {
@@ -79,7 +78,7 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 			tmp = make(map[string]interface{})
 		}
 		mgo.DestoryMongoConn(session)
-		log.Println("查询结果不一致", "bidding:", count, "抽取:", extCount)
+		log.Println("查询结果", "bidding:", count, "抽取:", extCount)
 		if int64(len(res)) != count {
 			time.Sleep(20 * time.Second)
 			toadd := &net.UDPAddr{
@@ -88,7 +87,7 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 			}
 			udpclient.WriteUdp(data, mu.OP_TYPE_DATA, toadd)
 		}
-		n1, n2 = doIndex(res, eMap, index, itype, db, c, bkey)
+		n1, n2 = doIndex(res, eMap, index, itype, db, c, bkey, tasktype)
 		//if int64(n1 + n2) != count {
 		//	log.Println("任务错误，结果不一致")
 		//}
@@ -96,52 +95,12 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 		log.Println("数据量太大，放弃！", count)
 		mgo.DestoryMongoConn(session)
 	}
-	log.Println(mapInfo, "create bidding index...over", "all:", count, "n1:", n1, "n2:", n2)
+	log.Println(mapInfo, "create bidding index...over", "all:", count, "bidding size:", n1, "，es size:", n2)
 
-	//go delEs(mapInfo, index, itype, db, c) //删除索引
 }
 
-//删除索引
-// func delEs(mapInfo map[string]interface{}, index, itype, db, c string) {
-// 	defer qutil.Catch()
-// 	other_delete := false
-// 	if other_index != "" && other_itype != "" {
-// 		other_delete = true
-// 	}
-// 	ids := qutil.ObjToString(mapInfo["ids"])
-// 	idsarr := strings.Split(ids, ",")
-// 	log.Println("delete ids count:", len(idsarr))
-// 	n1 := 0
-// 	update := [][]map[string]interface{}{} //将bidding表中的extracttype改为-1
-// 	set := map[string]interface{}{
-// 		"$set": map[string]interface{}{"extracttype": -1},
-// 	}
-// 	for _, id := range idsarr {
-// 		if id != "" {
-// 			update = append(update, []map[string]interface{}{ //更新
-// 				map[string]interface{}{
-// 					"_id": mongodb.StringTOBsonId(id),
-// 				},
-// 				set,
-// 			})
-// 			if elastic.DelById(index, itype, id) {
-// 				n1++
-// 			}
-// 			if other_delete {
-// 				bidding_other_es.DelById(other_index, other_itype, id)
-// 			}
-// 		}
-// 	}
-// 	//更新
-// 	if len(update) > 0 {
-// 		mgo.UpdateBulkAll(db, c, update...)
-// 	}
-
-// 	log.Println("result delete bidding index...over", "all:", n1)
-// }
-
-func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interface{}, index, itype, db, c, bkey string) (int, int) {
-	n1, n2 := 0, 0
+func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interface{}, index, itype, db, c, bkey, tasktype string) (int, int) {
+	n1, n2 := 0, 0 //bidding数量，索引数量
 	//线程池
 	UpdatesLock := sync.Mutex{}
 	fields := strings.Split(bidding["fields"].(string), ",")
@@ -157,18 +116,30 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 			tmp = make(map[string]interface{})
 			continue
 		}
+		tid := mongodb.BsonIdToSId(tmp["_id"])
+		//loginfo := make(map[string]interface{}) // 日志
 		update := map[string]interface{}{} //要更新的mongo数据
 		//对比方法----------------
-		tid := mongodb.BsonIdToSId(tmp["_id"])
-
 		if eMap[tid] != nil {
 			compare = eMap[tid]
-			if qutil.IntAll(compare["dataging"]) == 1 { //extract中dataging=1不生索引
-				tmp = make(map[string]interface{})
-				compare = nil
-				continue
+			if tasktype == "bidding" {
+				// 增量id段 正常数据
+				if num := qutil.IntAll(compare["dataging"]); num == 1 { //extract中dataging=1跳过
+					tmp = make(map[string]interface{})
+					compare = nil
+					continue
+				}
+				delete(eMap, tid)
+			}
+			if tasktype == "bidding_history" {
+				//增量id段 历史数据
+				if compare["history_updatetime"] == nil { //extract中history_updatetime不存在跳过
+					tmp = make(map[string]interface{})
+					compare = nil
+					continue
+				}
+				delete(eMap, tid)
 			}
-			delete(eMap, tid)
 			//更新bidding表，生成索引;bidding表modifyinfo中的字段不更新
 			modifyinfo := make(map[string]bool)
 			if tmpmodifyinfo, ok := tmp["modifyinfo"].(map[string]interface{}); ok && tmpmodifyinfo != nil {
@@ -187,9 +158,11 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 					update[k] = v1
 				} else if v2 != nil && v1 == nil {
 					//update[k+"_b"] = v2
+					if k == "area" || k == "city" || k == "district" {
+						update[k] = ""
+					}
 				}
 			}
-			//logger.Debug("_id---", tid, ",repeat---", compare["repeat"], ",dataging---", compare["dataging"])
 			if qutil.IntAll(compare["repeat"]) == 1 {
 				update["extracttype"] = -1
 			} else {
@@ -234,33 +207,30 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 				}
 				update["s_topscopeclass"] = strings.Join(newclass, ",")
 			}
-			//处理中标企业
-			//			winner, _ := compare["winner"].(string)
-			//			m1 := map[string]bool{}
-			//			if winner != "" {
-			//				m1[winner] = true
-			//			}
-			//			package1 := compare["package"]
-			//			if package1 != nil {
-			//				packageM, _ := package1.(map[string]interface{})
-			//				for _, p := range packageM {
-			//					pm, _ := p.(map[string]interface{})
-			//					pw, _ := pm["winner"].(string)
-			//					if pw != "" {
-			//						m1[pw] = true
-			//					}
-			//				}
-			//			}
+			if package1 := compare["package"]; package1 != nil {
+				packageM, _ := package1.(map[string]interface{})
+				for _, p := range packageM {
+					pm, _ := p.(map[string]interface{})
+					if qutil.ObjToString(pm["winner"]) != "" || qutil.Float64All(pm["budget"]) > 0 ||
+						qutil.Float64All(pm["bidamount"]) > 0 {
+						update["multipackage"] = 1
+						break
+					}
+				}
+			} else {
+				update["multipackage"] = 0
+			}
 			compare = nil
-			//			if len(m1) > 0 {
-			//				//str := ","
-			//				winnerarr := []string{}
-			//				for k, _ := range m1 {
-			//					//str += k + ","
-			//					winnerarr = append(winnerarr, k)
-			//				}
-			//				update["s_winner"] = strings.Join(winnerarr, ",")
-			//			}
+		} else {
+			area := qutil.ObjToString(tmp["area"])
+			city := qutil.ObjToString(tmp["city"])
+			district := qutil.ObjToString(tmp["district"])
+			rdata := standardCheckCity(area, city, district)
+			if len(rdata) > 0 {
+				for k, v := range rdata {
+					update[k] = v
+				}
+			}
 		}
 		//------------------对比结束
 
@@ -283,8 +253,8 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 						if len(*ents) > 0 {
 							id = qutil.ObjToString((*ents)[0]["_id"])
 							redis.PutCKV("qyxy_id", w, id)
-						}else {
-							ent, _ := qyxydb.FindOne("company_history_name", bson.M{"company_name": w})
+						} else {
+							ent, _ := qyxydb.FindOne("company_history_name", bson.M{"history_name": w})
 							if len(*ent) > 0 {
 								id = qutil.ObjToString((*ent)["company_id"])
 								redis.PutCKV("qyxy_id", w, id)
@@ -349,6 +319,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		//		}
 		go IS.Add("bidding")
 		if qutil.IntAll(update["extracttype"]) != -1 {
+			n2++
 			newTmp := map[string]interface{}{}                //最终生索引的数据
 			for field, ftype := range biddingIndexFieldsMap { //
 				if tmp[field] != nil { //
@@ -436,7 +407,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 						}
 					} else if field == "review_experts" {
 						// 评审专家
-						if arr, ok :=tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
+						if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
 							arr1 := qutil.ObjArrToStringArr(arr)
 							newTmp[field] = strings.Join(arr1, ",")
 						}
@@ -446,10 +417,10 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 						if tmp[field] != nil && tmp["bidendtime"] == nil {
 							newTmp["bidendtime"] = tmp[field]
 							newTmp[field] = tmp[field]
-						}else if tmp[field] == nil && tmp["bidendtime"] != nil {
+						} else if tmp[field] == nil && tmp["bidendtime"] != nil {
 							newTmp["bidendtime"] = tmp["bidendtime"]
 							newTmp[field] = tmp["bidendtime"]
-						}else {
+						} else {
 							if tmp["bidopentime"] != nil {
 								newTmp[field] = tmp["bidopentime"]
 							}
@@ -465,17 +436,15 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 					}
 				}
 			}
-
-			YuceEndtime(newTmp)		// 预测结果时间
+			YuceEndtime(newTmp)                      // 预测结果时间
+			newTmp["createtime"] = time.Now().Unix() // es库数据创建时间，只有增量数据有
 			arrEs = append(arrEs, newTmp)
 		}
 		if len(update) > 0 {
 			delete(update, "winnerorder") //winnerorder不需要更新到bindding表，删除
 			arr = append(arr, []map[string]interface{}{
-				{
-					"_id": tmp["_id"]},
-				{
-					"$set": update},
+				{"_id": tmp["_id"]},
+				{"$set": update},
 			})
 		}
 		if len(arr) >= BulkSize-1 {
@@ -671,6 +640,10 @@ func YuceEndtime(tmp map[string]interface{}) {
 				}
 			}
 		}
+		if yucestarttime > 0 && yuceendtime > yucestarttime {
+			tmp["yuceendtime"] = yuceendtime
+			return
+		}
 		// 预测开始时间 合同签订日期
 		if yucestarttime == 0 {
 			if qutil.IntAll(tmp["signaturedate"]) <= 0 {
@@ -687,7 +660,7 @@ func YuceEndtime(tmp map[string]interface{}) {
 		if yucestarttime > 0 && yuceendtime == 0 {
 			if qutil.IntAll(tmp["project_duration"]) > 0 && qutil.ObjToString(tmp["project_timeunit"]) != "" {
 				yuceendtime = YcEndTime(yucestarttime, qutil.IntAll(tmp["project_duration"]), qutil.ObjToString(tmp["project_timeunit"]))
-				tmp["yeceendtime"] = yuceendtime
+				tmp["yuceendtime"] = yuceendtime
 			}
 		}
 	}
@@ -696,16 +669,16 @@ func YuceEndtime(tmp map[string]interface{}) {
 func YcEndTime(starttime int64, num int, unit string) int64 {
 	yuceendtime := int64(0)
 	if unit == "日历天" || unit == "天" || unit == "日" {
-		yuceendtime = starttime + int64(num * 86400)
-	}else if unit == "周" {
-		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num * 7).Unix()
-	}else if unit == "月" {
+		yuceendtime = starttime + int64(num*86400)
+	} else if unit == "周" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num*7).Unix()
+	} else if unit == "月" {
 		yuceendtime = time.Unix(starttime, 0).AddDate(0, num, 0).Unix()
-	}else if unit == "年" {
+	} else if unit == "年" {
 		yuceendtime = time.Unix(starttime, 0).AddDate(num, 0, 0).Unix()
-	}else if unit == "工作日" {
-		n := num/7 * 2
-		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num + n).Unix()
+	} else if unit == "工作日" {
+		n := num / 7 * 2
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num+n).Unix()
 	}
 	return yuceendtime
 }
@@ -722,8 +695,7 @@ func FormatDateStr(ds string) int64 {
 	if err != nil {
 		qutil.Debug(err)
 		return 0
-	}else {
+	} else {
 		return location.Unix()
 	}
 }
-

udpcreateindex/src/biddingindexback.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
@@ -8,7 +9,6 @@ import (
 	"reflect"
 	"strings"
 
-	"gopkg.in/mgo.v2/bson"
 	//elastic "qfw/util/elastic_v5"
 	"regexp"
 	//	"strings"
@@ -51,7 +51,7 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 	query := session.DB(db).C(c).Find(q).Select(bson.M{
 		"projectinfo.attachment": 0,
 		"contenthtml":            0,
-		"publishdept":			  0,
+		"publishdept":            0,
 	}).Sort("_id").Iter()
 	//查询抽取结果
 	n := 0
@@ -203,7 +203,7 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 						}
 					} else if field == "review_experts" {
 						// 评审专家
-						if arr, ok :=tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
+						if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
 							arr1 := qutil.ObjArrToStringArr(arr)
 							newTmp[field] = strings.Join(arr1, ",")
 						}
@@ -221,9 +221,9 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 						if len([]rune(detail)) > detailLength {
 							detail = detail[:detailLength]
 						}
-						if strings.Contains(qutil.ObjToString(tmp["title"]), detail) {
+						if strings.Contains(detail, qutil.ObjToString(tmp["title"])) {
 							newTmp[field] = FilterDetail(detail)
-						}else {
+						} else {
 							newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
 						}
 					} else if field == "_id" || field == "topscopeclass" { //不做处理
@@ -278,9 +278,20 @@ var filterReg = regexp.MustCompile("<[^>]+>")
 var filterSpace = regexp.MustCompile("<[^>]*?>|[\\s\u3000\u2003\u00a0]")
 
 func FilterDetail(text string) string {
-	return filterReg.ReplaceAllString(text, "")
+	text = filterReg.ReplaceAllString(text, "")
+	for _, s := range FilterKeyword {
+		reg := regexp.MustCompile(s)
+		text = reg.ReplaceAllString(text, "")
+	}
+	return text
 }
 
 func FilterDetailSpace(text string) string {
 	return filterSpace.ReplaceAllString(text, "")
 }
+
+// 正则判断是否包含
+func checkContains(s, sub string) bool {
+	reg := regexp.MustCompile(`(?i)(^|([\s\t\n]+))(` + sub + `)($|([\s\t\n]+))`)
+	return reg.MatchString(s)
+}

udpcreateindex/src/biddingindexback2.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
@@ -13,7 +14,6 @@ import (
 	"sync"
 	"time"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 func biddingBackTask2(data []byte, mapInfo map[string]interface{}) {

udpcreateindex/src/biddingmerge.go

@@ -2,13 +2,13 @@ package main
 
 import (
 	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
 	"strings"
 	"sync"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 //对字段处理 bidamount  budget

udpcreateindex/src/bidingpurchasing.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"qfw/util"
 	elastic "qfw/util/elastic"
@@ -8,8 +9,6 @@ import (
 	"unicode/utf8"
 
 	u "util"
-
-	"gopkg.in/mgo.v2/bson"
 )
 
 //定时查询bidding中extract_state为2的数据生成索引
@@ -353,6 +352,11 @@ func getFileText(tmp map[string]interface{}) (filetext string) {
 							if utf8.RuneCountInString(filetext+bs) < fileLength {
 								filetext += bs + "\n"
 							} else {
+								if utf8.RuneCountInString(bs) > fileLength {
+									filetext = bs[0:fileLength]
+								} else {
+									filetext = bs
+								}
 								break
 							}
 						}

udpcreateindex/src/buyerindex.go

@@ -1,13 +1,13 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	"qfw/util"
 	elastic "qfw/util/elastic"
 	"sync"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 /*

udpcreateindex/src/buyertask.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"log"
 	"mongodb"
 	qu "qfw/util"
@@ -8,7 +9,6 @@ import (
 	"sync"
 	"time"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 var fieldArr = []string{"institute_type", "fixedphone", "mobilephone", "latestfixedphone", "latestmobilephone", "province", "city"}
@@ -23,8 +23,8 @@ func buyerEsTaskOnce() {
 	now := time.Now()
 	preTime := time.Date(now.Year(), now.Month(), now.Day()-1, now.Hour(), 0, 0, 0, time.Local)
 	curTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.Local)
-	task_sid := mongodb.BsonIdToSId(bson.NewObjectIdWithTime(preTime))
-	task_eid := mongodb.BsonIdToSId(bson.NewObjectIdWithTime(curTime))
+	task_sid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(preTime))
+	task_eid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(curTime))
 	log.Println("buyer 区间id:", task_sid, task_eid)
 	// task_sid = "5e6611b7aec95406dccf7151"
 	// task_eid = "5f7249164bdc0447a6c90fa5"

udpcreateindex/src/config.json

@@ -4,9 +4,9 @@
   "uname": "root",
   "upwd": "root",
   "mongodb": {
-    "addr": "192.168.3.207:27092",
+    "addr": "192.168.3.207:27001",
     "pool": 10,
-    "db": "wjh"
+    "db": "qfw_data"
   },
   "savedb": {
     "addr": "192.168.3.207:27092",
@@ -30,25 +30,25 @@
     "type": "buyer"
   },
   "biddingback": {
-    "db": "qfw",
+    "db": "qfw_data",
     "collect": "bidding",
-    "index": "bidding",
+    "index": "bidding_v2",
     "type": "bidding"
   },
   "bidding": {
-    "db": "wjh",
+    "db": "qfw_data",
     "collect": "bidding",
-    "index": "bidding_v1",
+    "index": "bidding_v2",
     "type": "bidding",
     "extractdb": "wjh",
-    "extractcollect": "",
+    "extractcollect": "extract",
     "indexfields": [
       "buyerzipcode", "winnertel", "winnerperson", "contractcode", "winneraddr", "agencyaddr", "buyeraddr", "signaturedate", "projectperiod", "projectaddr", "agencytel", "agencyperson",
       "buyerperson", "agency", "projectscope", "projectcode", "bidopentime", "supervisorrate", "buyertel", "bidamount", "winner", "buyer", "budget", "projectname", "bidstatus", "buyerclass",
       "topscopeclass", "s_topscopeclass", "s_subscopeclass", "area", "city", "district", "s_winner", "_id", "title", "detail", "site", "comeintime", "href", "infoformat", "publishtime",
       "s_sha", "spidercode", "subtype", "toptype", "projectinfo", "purchasing", "purchasinglist", "filetext", "channel", "winnerorder", "project_scale", "project_duration", "project_timeunit",
       "project_startdate", "project_completedate", "payway", "contract_guarantee", "bid_guarantee", "qualifies", "entidlist", "funds", "review_experts", "bidmethod", "bidendtime", "bidopenaddress",
-      "docamount", "agencyrate", "agencyfee", "getdocmethod", "china_bidding", "bidway"
+      "docamount", "agencyrate", "agencyfee", "getdocmethod", "china_bidding", "bidway", "purchasing_tag", "multipackage"
     ],
     "indexfieldsmap": {
       "buyerzipcode": "string", "winnertel": "string", "winnerperson": "string", "contractcode": "string", "winneraddr": "string", "agencyaddr": "string", "buyeraddr": "string", "signaturedate": "int64",
@@ -59,16 +59,17 @@
       "projectinfo": "", "purchasing": "string", "purchasinglist": "", "filetext": "string", "channel": "string", "winnerorder": "", "project_scale": "string", "project_duration": "int32", "project_timeunit": "string",
       "project_startdate": "int64", "project_completedate": "int64", "payway": "string", "contract_guarantee": "bool", "bid_guarantee": "bool", "qualifies": "", "entidlist": "[]string", "funds": "string",
       "review_experts": "", "bidmethod": "string", "bidendtime": "int64", "bidopenaddress": "string", "docamount": "float64", "agencyrate": "float64", "agencyfee": "float64", "bidway": "string",
-      "getdocmethod": "string", "china_bidding": "string"
+      "getdocmethod": "string", "china_bidding": "string", "purchasing_tag": "string", "multipackage": "int32"
     },
-    "fields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,s_topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass,dataging,winnerorder,project_scale,project_duration,project_timeunit,project_startdate,project_completedate, payway,contract_guarantee,bid_guarantee,qualifies,funds,review_experts,bidmethod,bidendtime,bidopenaddress,docamount,bidway,agencyrate,agencyfee,getdocmethod",
+    "fields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,s_topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass,dataging,winnerorder,project_scale,project_duration,project_timeunit,project_startdate,project_completedate, payway,contract_guarantee,bid_guarantee,qualifies,funds,review_experts,bidmethod,bidendtime,bidopenaddress,docamount,bidway,agencyrate,agencyfee,getdocmethod,purchasing_tag",
     "projectinfo": "approvecode,approvecontent,approvestatus,approvetime,approvedept,approvenumber,projecttype,approvecity",
     "projectinfomap": {
       "approvecode": "string", "approvecontent": "string", "approvestatus": "string", "approvetime": "string", "approvedept": "string", "approvenumber": "string", "projecttype": "string", "approvecity": "string"
     },
-    "purchasinglist": "itemname,brandname,model,unitname,number,unitprice,totalprice",
+    "purchasinglist": "itemname,brandname,model,unitname,number,unitprice,totalprice,projectname,buyer,item,projectscope,expurasingtime,reserved_amount",
     "purchasinglistmap": {
-      "itemname": "string", "brandname": "string", "model": "string", "unitname": "string", "number": "float64", "unitprice": "float64", "totalprice": "float64"
+      "itemname": "string", "brandname": "string", "model": "string", "unitname": "string", "number": "float64", "unitprice": "float64", "totalprice": "float64", "projectname": "string", "buyer": "string", "item": "string",
+      "projectscope": "string", "expurasingtime": "string", "reserved_amount": "string"
     },
     "winnerorder": "sort,sortstr,entname",
     "winnerordermap": {
@@ -98,13 +99,14 @@
     "db": "mixdata"
   },
   "standard": {
-    "addr": "192.168.3.207:27092",
+    "addr": "192.168.3.207:27001",
     "pool": 10,
     "db": "mixdata",
+    "coll_area": "address_jy_2021",
     "winnerent": {
       "collect1": "winner_enterprise",
       "collect2": "winner_err",
-      "index": "winner_v3",
+      "index": "winner_v1",
       "type": "winner"
     },
     "buyerent": {
@@ -120,8 +122,15 @@
     }
   },
   "elastic": {
-    "addr": "http://192.168.3.11:9800",
+    "addr": "http://192.168.3.206:9800",
     "pool": 12,
     "node": "4q7v7e6mQ5aeCwjUgM6HcA"
-  }
+  },
+  "filter-keyword": ["(招标网|千里马|采招网|招标采购导航网|招标与采购网|中国招投标网|中国采购与招标网|中国采购与招标|优质采)[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}(qianlima|zhaobiao|okcis|zbytb|infobidding|bidcenter|youzhicai|chinabidding|Chinabidding｜CHINABIDDING)[a-z0-9.\\/\\/]{0,40}",
+    "招标网[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}zhaobiao[a-z0-9.\\/\\/]{0,40}",
+    "千里马[\\w\\W]{0,15}[a-z0-9:\\/\\/.]{0,20}qianlima[a-z0-9.\\/\\/]{0,10}",
+    "[\\(（]?(网址)?[:：；;]?(http|https|htpps)*[:：]?(\\/\\/)?(www|jinan|WWW)?.(zhaobiao|chinabidding|Chinabidding|CHINABIDDING|infobidding|zbytb|okcis|qianlima|youzhicai).(com|cn|COM|CN)?(.cn|.CN)?\\/?[\\)）]?",
+    "[\\(（]?(网址)?(:：)?(http|https|htpps)*(:|：)?\\/\\/www.bidcenter.com.cn\\/",
+      "千里马(平台|网站)+", "[“\"]?优质采(平台|电子交易平台|云采购平台|交易平台)?[”\"]?", "《?(中国采购与|中国)?招(投)?标(与采购|采购导航)?网》?",
+    "《?元博网(采购与招标网)?》?", "《?(中国)?招标采购导航网》?", "中\\W{0,3}国采\\W{0,3}招\\W{0,3}网\\W*[(（]?(bidcenter.com.cn)?[)）]?", "已方宝", "中国招标与采购"]
 }

udpcreateindex/src/default.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"gopkg.in/mgo.v2/bson"
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	"qfw/util"

udpcreateindex/src/main.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"log"
 	mu "mfw/util"
 	"mongodb"
 	"net"
 	"net/http"
+	_ "net/http/pprof"
 	"qfw/util"
 	elastic "qfw/util/elastic"
 	"qfw/util/redis"
@@ -16,6 +18,19 @@ import (
 	u "util"
 )
 
+type Province struct {
+	P_Name string
+}
+type City struct {
+	P_Name string
+	C_Name string
+}
+type District struct {
+	P_Name string
+	C_Name string
+	D_Name string
+}
+
 var (
 	Sysconfig                map[string]interface{} //配置文件
 	mgo                      *mongodb.MongodbSim    //mongodb操作对象
@@ -43,15 +58,22 @@ var (
 	other_index      string
 	other_itype      string
 
-	esAddr 			 string
-	esNode			 string
+	esAddr string
+	esNode string
+
+	FilterKeyword []string              //正文竟品关键词过滤
+	ProvinceDict  map[string][]Province //省份-map
+	CityDict      map[string][]City     //城市-map
+	DistrictDict  map[string][]District //区县-map
 
 	winner, bidding, biddingback, project, project2, buyer, standard, qyxy_ent map[string]interface{}
 )
 var UpdataMgoCache = make(chan []map[string]interface{}, 1000)
 var SP = make(chan bool, 5)
+var SaveLogChan = make(chan []map[string]interface{}, 1000)
+var SaveSp = make(chan bool, 5)
 
-var StopFlag = false		// 程序生索引停止标志
+var StopFlag = false // 程序生索引停止标志
 
 func init() {
 	util.ReadConfig(&Sysconfig)
@@ -76,9 +98,8 @@ func init() {
 		MongodbAddr: mconf["addr"].(string),
 		Size:        util.IntAllDef(mconf["pool"], 5),
 		DbName:      mconf["db"].(string),
-		UserName:	 Sysconfig["uname"].(string),
+		UserName:    Sysconfig["uname"].(string),
 		Password:    Sysconfig["upwd"].(string),
-		ReplSet: 	 "bidding",
 	}
 	mgo.InitPool()
 	project2db = &mongodb.MongodbSim{
@@ -194,6 +215,10 @@ func init() {
 	}
 	log.Println(projectinfoFields)
 	log.Println(purchasinglistFields)
+
+	initCheckCity()
+
+	FilterKeyword = util.ObjArrToStringArr(Sysconfig["filter-keyword"].([]interface{}))
 	//初始化oss
 	u.InitOss()
 
@@ -212,23 +237,23 @@ func main() {
 }
 
 /**
-	检查es查询队列  10s查询一次
- */
-func inspectQuery()  {
+检查es查询队列  10s查询一次
+*/
+func inspectQuery() {
 	ticker := time.NewTicker(time.Second * 10)
 	url := esAddr + "/_nodes/stats/thread_pool"
 	for range ticker.C {
 		resp, _ := http.Get(url)
-		if resp != nil && resp.Body != nil{
+		if resp != nil && resp.Body != nil {
 			defer resp.Body.Close()
 		}
-		body,_ := ioutil.ReadAll(resp.Body)
+		body, _ := ioutil.ReadAll(resp.Body)
 		respMap := make(map[string]interface{})
 		err := json.Unmarshal(body, &respMap)
 		if err == nil {
 			if data, o1 := respMap["nodes"].(map[string]interface{}); o1 {
 				if nodes, o2 := data[esNode].(map[string]interface{}); o2 {
-					if pool,  o3 := nodes["thread_pool"].(map[string]interface{}); o3 {
+					if pool, o3 := nodes["thread_pool"].(map[string]interface{}); o3 {
 						index, _ := pool["index"].(map[string]interface{})
 						search, _ := pool["search"].(map[string]interface{})
 						bulk, _ := pool["bulk"].(map[string]interface{})
@@ -237,7 +262,7 @@ func inspectQuery()  {
 							util.Debug("es thread_pool search queue---", search["queue"])
 							util.Debug("es thread_pool bulk queue---", bulk["queue"])
 							StopFlag = true
-						}else {
+						} else {
 							StopFlag = false
 						}
 					}
@@ -248,6 +273,7 @@ func inspectQuery()  {
 }
 
 var pool = make(chan bool, 20)
+
 func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 	switch act {
 	case mu.OP_TYPE_DATA: //上个节点的数据
@@ -280,7 +306,15 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					defer func() {
 						<-pool
 					}()
-					biddingTask(data, mapInfo)
+					biddingTask(data, mapInfo, tasktype)
+				}()
+			case "bidding_history": //增量id段历史数据
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					biddingTask(data, mapInfo, tasktype)
 				}()
 			case "project":
 				pool <- true
@@ -306,7 +340,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					}()
 					biddingAllTask(data, mapInfo)
 				}()
-			case "biddingdata": //联表生成索引不合并，不生成关键词
+			case "biddingdata": //bidding全量数据
 				pool <- true
 				go func() {
 					defer func() {
@@ -384,3 +418,101 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 		log.Println("发送成功", string(data))
 	}
 }
+
+//初始化城市
+func initCheckCity() {
+	//初始化-城市配置
+	ProvinceDict = make(map[string][]Province, 0)
+	CityDict = make(map[string][]City, 0)
+	DistrictDict = make(map[string][]District, 0)
+
+	q := map[string]interface{}{
+		"town_code": map[string]interface{}{
+			"$exists": 0,
+		},
+	}
+	sess := mgostandard.GetMgoConn()
+	defer mgostandard.DestoryMongoConn(sess)
+	it := sess.DB("mixdata").C(util.ObjToString(standard["coll_area"])).Find(&q).Iter()
+	total := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%1000 == 0 {
+			log.Println("当前数量:", total)
+		}
+		district_code := util.IntAll(tmp["district_code"])
+		city_code := util.IntAll(tmp["city_code"])
+		if district_code > 0 {
+			province := util.ObjToString(tmp["province"])
+			city := util.ObjToString(tmp["city"])
+			district := util.ObjToString(tmp["district"])
+			data := District{province, city, district}
+			if DistrictDict[district] == nil {
+				DistrictDict[district] = []District{data}
+			} else {
+				arr := DistrictDict[district]
+				arr = append(arr, data)
+				DistrictDict[district] = arr
+			}
+		} else {
+			if city_code > 0 {
+				province := util.ObjToString(tmp["province"])
+				city := util.ObjToString(tmp["city"])
+				data := City{province, city}
+				if CityDict[city] == nil {
+					CityDict[city] = []City{data}
+				} else {
+					arr := CityDict[city]
+					arr = append(arr, data)
+					CityDict[city] = arr
+				}
+			} else {
+				province := util.ObjToString(tmp["province"])
+				data := Province{province}
+				if ProvinceDict[province] == nil {
+					ProvinceDict[province] = []Province{data}
+				} else {
+					arr := ProvinceDict[province]
+					arr = append(arr, data)
+					ProvinceDict[province] = arr
+				}
+			}
+		}
+		tmp = make(map[string]interface{})
+	}
+	util.Debug(fmt.Sprintf("城市配置加载完毕...省~%d 市~%d 区~%d", len(ProvinceDict), len(CityDict), len(DistrictDict)))
+}
+
+func saveLog() {
+	arru := make([][]map[string]interface{}, 200)
+	indexu := 0
+	for {
+		select {
+		case v := <-SaveLogChan:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				SaveSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-SaveSp
+					}()
+					extractmgo.UpSertBulk("createIndex_log", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				SaveSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-SaveSp
+					}()
+					extractmgo.UpSertBulk("createIndex_log", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		}
+	}
+}

udpcreateindex/src/projectindex.go

@@ -55,7 +55,10 @@ func projectTask(data []byte, project, mapInfo map[string]interface{}) {
 	arr := make([]map[string]interface{}, savesizei)
 	var n int
 	i := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+	for tmp := make(map[string]interface{}); query.Next(tmp); i++ {
+		if n%10000 == 0 {
+			log.Println("current---------", n)
+		}
 		pp := map[string]map[string]interface{}{}
 		if packages, ok := tmp["package"].(map[string]interface{}); ok {
 			for _, pks := range packages {

udpcreateindex/src/standardata.go

@@ -1,12 +1,12 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	mgov "mongodb"
 	"qfw/util"
 	elastic "qfw/util/elastic"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 func standardTask(stype string, mapInfo map[string]interface{}) {

udpcreateindex/src/task.go

@@ -2,6 +2,7 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"log"
 	"mongodb"
 	qutil "qfw/util"
@@ -16,19 +17,32 @@ func task_index() {
 	//c.AddFunc("0 30 * * * *", func() { task_biddingfile() }) //每30分钟执行一次
 	//c.AddFunc("0 22 14 * * *", func() { task_qyxyindex() })
 
-	c.AddFunc("0 0 0 * * ?", func() { task_winneres() }) //每天凌晨执行一次winner生索引
-	c.AddFunc("0 0 1 * * ?", func() { task_buyeres() })  //每天1点执行一次buyer生索引
+	_ = c.AddFunc("0 0 0 * * ?", func() { task_winneres() })   //每天凌晨执行一次winner生索引
+	_ = c.AddFunc("0 0 1 * * ?", func() { task_buyeres() })    //每天1点执行一次buyer生索引
+	_ = c.AddFunc("0 0 2 * * ?", func() { task_biddingAll() }) //每天2点执行 前一天的所有招标数据
 	c.Start()
 }
 func task_winneres() {
-	log.Println("定时任务，winneres")
+	qutil.Debug("定时任务，winneres")
 	winnerEsTaskOnce()
 }
 func task_buyeres() {
-	log.Println("定时任务，buyeres")
+	qutil.Debug("定时任务，buyeres")
 	buyerEsTaskOnce()
 }
 
+func task_biddingAll() {
+	qutil.Debug("定时任务，bidding_all")
+	t := time.Now()
+	currenttime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
+	endtime := time.Unix(currenttime.Unix(), 0).AddDate(0, 0, -1)
+
+	sid := primitive.NewObjectIDFromTimestamp(endtime)
+	eid := primitive.NewObjectIDFromTimestamp(currenttime)
+
+	biddingDataTask(nil, map[string]interface{}{"gtid": mongodb.BsonIdToSId(sid), "lteid": mongodb.BsonIdToSId(eid)})
+}
+
 //招标附件、标的物,临时用
 func task_biddingfile() {
 	defer qutil.Catch()

udpcreateindex/src/util/ossclient.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	ossEndpoint        = "oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
+	ossEndpoint        = "oss-cn-beijing.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
 	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
 	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
 	ossBucketName      = "topjy"

udpcreateindex/src/winnerindex.go

@@ -1,12 +1,12 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson"
 	"log"
 	"mongodb"
 	"qfw/util"
 	elastic "qfw/util/elastic"
 
-	"gopkg.in/mgo.v2/bson"
 )
 
 func winnerTask(data []byte, mapInfo map[string]interface{}) {

udpcreateindex/src/winnertask.go

@@ -1,14 +1,13 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"log"
 	"mongodb"
 	qu "qfw/util"
 	elastic "qfw/util/elastic"
 	"sync"
 	"time"
-
-	"gopkg.in/mgo.v2/bson"
 )
 
 func winnerEsTaskOnce() {
@@ -21,10 +20,10 @@ func winnerEsTaskOnce() {
 	now := time.Now()
 	preTime := time.Date(now.Year(), now.Month(), now.Day()-1, now.Hour(), 0, 0, 0, time.Local)
 	curTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.Local)
-	task_sid := mongodb.BsonIdToSId(bson.NewObjectIdWithTime(preTime))
-	task_eid := mongodb.BsonIdToSId(bson.NewObjectIdWithTime(curTime))
-	// task_sid = "5e6598f82c27dc56292158da"
-	// task_eid = "5f80c8f89a0c261af872294c"
+	task_sid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(preTime))
+	task_eid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(curTime))
+	//task_sid = "5e6598f82c27dc56292158da"
+	//task_eid = "620576342566c40049f26155"
 	log.Println("winner 区间id:", task_sid, task_eid)
 	//区间id
 	q := map[string]interface{}{
@@ -65,11 +64,12 @@ func winnerEsTaskOnce() {
 			savetmp["pici"] = tmp["updatetime"]
 			if province := qu.ObjToString(tmp["province"]); province != "" {
 				savetmp["province"] = province
-
 			}
 			if city := qu.ObjToString(tmp["city"]); city != "" {
 				savetmp["city"] = city
-
+			}
+			if text := qu.ObjToString(tmp["tag_business"]); text != "" {
+				savetmp["tag_business"] = text
 			}
 
 			winerEsLock.Lock()

udps/main.go

@@ -22,10 +22,8 @@ func main() {
 	flag.IntVar(&p, "p", 6601, "端口")
 	flag.IntVar(&tmptime, "tmptime", 0, "时间查询")
 	flag.StringVar(&tmpkey, "tmpkey", "", "时间字段")
-
-	flag.StringVar(&id1, "gtid", "114168ea1a75b8f44678a39b", "gtid")
-	flag.StringVar(&id2, "lteid", "9142e5741a75b8f4467b3276", "lteid")
-
+	flag.StringVar(&id1, "gtid", "", "gtid")
+	flag.StringVar(&id2, "lteid", "", "lteid")
 	flag.StringVar(&ids, "ids", "", "id1,id2")
 	flag.StringVar(&stype, "stype", "biddingall", "stype,传递类型")
 	flag.StringVar(&bkey, "bkey", "", "bkey,加上此参数表示不生关键词和摘要")