Merge branch 'dev3.4.1' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4.1

fullproject/src_v1/config.json

@@ -4,7 +4,7 @@
     "statusdays": 15,
 	"mongodbServers": "192.168.3.207:27092",
     "mongodbPoolSize": 10,
-    "mongodbName": "qfw_data",
+    "mongodbName": "wjh",
 	"hints":"publishtime_1",
     "extractColl": "bidding",
     "extractColl1": "bidding",

fullproject/src_v1/load_data.go

@@ -35,7 +35,7 @@ func (p *ProjectTask) loadData(starttime int64) {
 					id := tmp.Id.Hex()
 					for _, v := range append([]string{tmp.ProjectName}, tmp.MPN...) {
 						if v != "" {
-							v = pcReplace.ReplaceAllString(v, "")
+							//v = pcReplace.ReplaceAllString(v, "")
 							if v != "" {
 								k := p.mapPn[v]
 								if k == nil {
@@ -49,7 +49,7 @@ func (p *ProjectTask) loadData(starttime int64) {
 					}
 					for _, v := range append([]string{tmp.ProjectCode}, tmp.MPC...) {
 						if v != "" {
-							v = pcReplace.ReplaceAllString(v, "")
+							//v = pcReplace.ReplaceAllString(v, "")
 							if v != "" {
 								k := p.mapPc[v]
 								if k == nil {

fullproject/src_v1/main.go

@@ -91,8 +91,8 @@ func main() {
 
 //测试组人员使用
 func mainT() {
-	sid = "600f86a17cdc9beb563d9e33"
-	eid = "600f86a17cdc9beb563d9e34"
+	sid = "6062826adeed5af79ca930c9"
+	eid = "6062826adeed5af79ca930ca"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()

fullproject/src_v1/project.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"log"
 	"math"
+	"mongodb"
 	qu "qfw/util"
 	"sort"
 	"strings"
@@ -95,7 +96,7 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 	p.findLock.Lock()
 	defer p.findLock.Unlock()
 	// 3.18 isfow=0数据不参与项目合并
-	code := qu.ObjToString(tmp["spidercode"])
+	code := strings.ReplaceAll(qu.ObjToString(tmp["spidercode"]), " ", "")
 	p.mapSpiderLock.Lock()
 	isflow := p.mapSpider[code]
 	p.mapSpiderLock.Unlock()
@@ -420,28 +421,10 @@ func compareResult(resVal, pjVal, score2 int, comStr, compareBuyer, compareCity,
 	return eqV
 }
 
-//项目中的字段
-var FIELDS = []string{
-	"area",
-	"city",
-	"district",
-	"projectname",
-	"projectcode",
-	"buyer",
-	"buyerclass",
-	"buyerperson",
-	"buyertel",
-	"winner",
-	"agency",
-	"topscopeclass",
-	"subscopeclass",
-	"package",
-}
-
 //招标时间zbtime、中标时间jgtime、项目状态bidstatus、招标类型bidtype、最后发布时间lasttime、首次发布时间firsttime
-
 func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *ProjectInfo) {
-	pId := primitive.NewObjectID() //NewObjectId()
+	//pId := primitive.NewObjectID() //NewObjectId()
+	pId := mongodb.StringTOBsonId(thisinfo.Id)
 	set := map[string]interface{}{}
 	set["_id"] = pId
 	for _, f := range FIELDS {
@@ -515,10 +498,10 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	}
 	set["ids"] = []string{thisinfo.Id}
 	if thisinfo.TopType == "招标" {
-		if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
+		//if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
 			set["zbtime"] = tmp["publishtime"]
 			p1.Zbtime = qu.Int64All(tmp["publishtime"])
-		}
+		//}
 	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
 		set["jgtime"] = tmp["publishtime"]
 		p1.Jgtime = thisinfo.Publishtime
@@ -669,49 +652,6 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	return pId.Hex(), &p1
 }
 
-//招标信息字段
-var INFOFIELDS = []string{
-	"projectname",
-	"projectcode",
-	"projectscope",
-	"contractcode",
-	"title",
-	"href",
-	"publishtime",
-	"comeintime",
-	"bidopentime",
-	"toptype",
-	"subtype",
-	"buyer",
-	"buyerclass",
-	"agency",
-	"winner",
-	"budget",
-	"bidamount",
-	"topscopeclass",
-	"subscopclass",
-	"infoformat",
-	"buyerperson",
-	"buyertel",
-	"area",
-	"city",
-	"district",
-	"spidercode",
-	"site",
-	"review_experts",
-	"purchasing",
-	"project_scale",
-	"project_duration",
-	"project_timeunit",
-	"project_startdate",
-	"project_completedate",
-	"payway",
-	"contract_guarantee",
-	"bid_guarantee",
-	"qualifies",
-	"entidlist",
-}
-
 //项目中list的信息
 func (p *ProjectTask) PushListInfo(tmp map[string]interface{}, infoid string) bson.M {
 	res := bson.M{
@@ -761,7 +701,7 @@ func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidty
 	return p1
 }
 
-//更新项目
+//更新项目	全量合并
 func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string, ex int) {
 	if p.currentType != "updateInfo" {
 		if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
@@ -780,7 +720,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	pInfo.LastTime = thisinfo.Publishtime
 	set["lasttime"] = thisinfo.Publishtime
 	if thisinfo.TopType == "招标" {
-		if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" && pInfo.Zbtime <= 0 {
+		if pInfo.Zbtime <= 0 {
 			set["zbtime"] = tmp["publishtime"]
 		}
 		if pInfo.Jgtime > 0 {
@@ -791,19 +731,18 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
 			if pInfo.Jgtime > 0 {
 				//jg1 := int64(math.Abs(float64(pInfo.Jgtime - thisinfo.Publishtime)))
-				// 4.12	直接更新jgtime
 				//公告状态和项目状态同样都是中标或者成交，
-				if (thisinfo.SubType == "中标" || thisinfo.SubType == "成交") && (pInfo.Bidstatus == "中标" || pInfo.Bidstatus == "成交") {
-					//if jg1 > p.jgTime {
+				if thisinfo.SubType == "成交" && pInfo.Bidstatus == "中标" {
+					if p.jgTime < thisinfo.Publishtime {
 						set["jgtime"] = tmp["publishtime"]
 						pInfo.Jgtime = thisinfo.Publishtime
-					//}
+					}
 					//公告状态和项目状态同样是流标或者废标
 				} else if (thisinfo.SubType == "流标" || thisinfo.SubType == "废标") && (pInfo.Bidstatus == "流标" || pInfo.Bidstatus == "废标") {
-					//if jg1 > p.jgTime {
+					if p.jgTime < thisinfo.Publishtime {
 						set["jgtime"] = tmp["publishtime"]
 						pInfo.Jgtime = thisinfo.Publishtime
-					//}
+					}
 				}
 			} else {
 				set["jgtime"] = tmp["publishtime"]
@@ -934,7 +873,12 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		if pInfo.ContractCode == "" {
 			set["contractcode"] = thisinfo.ContractCode
 		}else {
-			set["contractcode"] = pInfo.ContractCode + "," + thisinfo.ContractCode
+			list := strings.Split(pInfo.ContractCode, ",")
+			if BinarySearch(list, thisinfo.ContractCode) == -1 {
+				list = append(list, thisinfo.ContractCode)
+				sort.Strings(list)
+			}
+			set["contractcode"] = strings.Join(list, ",")
 		}
 	}
 
@@ -1180,6 +1124,10 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	}
 }
 
+func (p *ProjectTask) x() {
+	
+}
+
 /**
  *	更新项目时，项目状态的处理
  *	返回是否新增项目，异常标记
@@ -1244,18 +1192,6 @@ func ComparePlace(project *ProjectInfo, info *Info) bool {
 	return true
 }
 
-var PackageEle = []string{
-	"origin",
-	"name",
-	//"text",
-	"budget",
-	"winner",
-	"bidamount",
-	"bidamounttype",
-	"currency",
-	"bidstatus",
-}
-
 func packageEle(map1 map[string]interface{}, id string) map[string]interface{} {
 	p2 := map[string]interface{}{}
 	for _, k := range PackageEle {
@@ -1495,24 +1431,6 @@ func IsCreatePro(info *Info) (bol bool) {
 	return bol
 }
 
-var bidtype = map[string]string{
-	"招标": "招标",
-	"邀标": "邀标",
-	"询价": "询价",
-	"单一": "单一",
-	"竞价": "竞价",
-	"竞谈": "竞谈",
-}
-
-var bidstatus = map[string]string{
-	"预告": "预告",
-	"中标": "中标",
-	"成交": "成交",
-	"废标": "废标",
-	"流标": "流标",
-	"合同": "合同",
-}
-
 func (p *ProjectTask) GetBidTypeAndBidStatus(info *Info) (string, string) {
 	p.mapBidLock.Lock()
 	defer p.mapBidLock.Unlock()
@@ -1536,7 +1454,3 @@ func (p *ProjectTask) GetBidTypeAndBidStatus(info *Info) (string, string) {
 	return typeStr, statusStr
 
 }
-
-func ListSort(tmp map[string]interface{}) {
-
-}

fullproject/src_v1/project_tool.go

@@ -0,0 +1,94 @@
+package main
+
+//项目中的字段
+var FIELDS = []string{
+	"area",
+	"city",
+	"district",
+	"projectname",
+	"projectcode",
+	"buyer",
+	"buyerclass",
+	"buyerperson",
+	"buyertel",
+	"winner",
+	"agency",
+	"topscopeclass",
+	"subscopeclass",
+	"package",
+}
+
+//招标信息字段
+var INFOFIELDS = []string{
+	"projectname",
+	"projectcode",
+	"projectscope",
+	"contractcode",
+	"title",
+	"href",
+	"publishtime",
+	"comeintime",
+	"bidopentime",
+	"toptype",
+	"subtype",
+	"buyer",
+	"buyerclass",
+	"agency",
+	"winner",
+	"budget",
+	"bidamount",
+	"topscopeclass",
+	"subscopclass",
+	"infoformat",
+	"buyerperson",
+	"buyertel",
+	"area",
+	"city",
+	"district",
+	"spidercode",
+	"site",
+	"review_experts",
+	"purchasing",
+	"project_scale",
+	"project_duration",
+	"project_timeunit",
+	"project_startdate",
+	"project_completedate",
+	"payway",
+	"contract_guarantee",
+	"bid_guarantee",
+	"qualifies",
+	"entidlist",
+}
+
+// 包信息
+var PackageEle = []string{
+	"origin",
+	"name",
+	//"text",
+	"budget",
+	"winner",
+	"bidamount",
+	"bidamounttype",
+	"currency",
+	"bidstatus",
+}
+
+var bidstatus = map[string]string{
+	"预告": "预告",
+	"中标": "中标",
+	"成交": "成交",
+	"废标": "废标",
+	"流标": "流标",
+	"合同": "合同",
+}
+
+var bidtype = map[string]string{
+	"招标": "招标",
+	"邀标": "邀标",
+	"询价": "询价",
+	"单一": "单一",
+	"竞价": "竞价",
+	"竞谈": "竞谈",
+}
+

fullproject/src_v1/task.go

@@ -455,7 +455,7 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 	countRepeat := 0
 
 	pool := make(chan bool, p.thread)
-	log.Println("start project", q, p.pici)
+	util.Debug("start project", q, p.pici)
 	sess := MongoTool.GetMgoConn()
 	defer MongoTool.DestoryMongoConn(sess)
 	infoPool := make(chan map[string]interface{}, 2000)
@@ -476,8 +476,23 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 					//普通合并
 					p.CommonMerge(tmp, info)
 				}(tmp)
-			case <-over:
-				break L
+			default:
+				select {
+				case tmp := <-infoPool:
+					pool <- true
+					go func(tmp map[string]interface{}) {
+						defer func() {
+							<-pool
+						}()
+						p.fillInPlace(tmp)
+						info := ParseInfo(tmp)
+						p.currentTime = info.Publishtime
+						//普通合并
+						p.CommonMerge(tmp, info)
+					}(tmp)
+				case <- over:
+					break L
+				}
 			}
 		}
 	}()
@@ -517,16 +532,18 @@ L:
 					}
 				}
 				if util.IntAll(tmp["repeat"]) == 0 {
-					if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {
+					if P_QL.currentType == "ql" {
+						infoPool <- tmp
+					}else if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {
 						infoPool <- tmp
 					}else {
 						util.Debug("增量   dataging == 1 ", tmp["_id"])
 					}
 				}else {
 					countRepeat++
-					if P_QL.currentType == "project" {
-						util.Debug("repeat err---", tmp["_id"])
-					}
+					//if P_QL.currentType == "project" {
+					//	util.Debug("repeat err---", tmp["_id"])
+					//}
 				}
 				count++
 			} else {
@@ -615,20 +632,20 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	}
 
 	if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
-		thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
-		if thisinfo.ProjectName != "" {
+		//thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
+		//if thisinfo.ProjectName != "" {
 			thisinfo.pnbval++
-		}
+		//}
 	}
 
 	if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
 		if thisinfo.ProjectCode != "" {
-			thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
+			//thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
 			if thisinfo.pnbval == 0 && len([]rune(thisinfo.ProjectCode)) < 5 {
 				thisinfo.ProjectCode = StrOrNum.ReplaceAllString(thisinfo.ProjectCode, "")
 			}
 		} else {
-			thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
+			//thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
 			if thisinfo.pnbval == 0 && len([]rune(thisinfo.PTC)) < 5 {
 				thisinfo.PTC = StrOrNum.ReplaceAllString(thisinfo.PTC, "")
 			}

monitor/config.json

@@ -20,6 +20,6 @@
   },
   "jkmail": {
     "to": "wangjianghan@topnet.net.cn",
-    "api": "http://10.171.112.160:19281/_send/_mail"
+    "api": "http://172.17.145.179:19281/_send/_mail"
   }
 }

qyxy/src/main.go

@@ -42,8 +42,8 @@ func init() {
 		MongodbAddr: Sysconfig["mgodb"].(string),
 		Size:        qu.IntAllDef(Sysconfig["dbsize"], 5),
 		DbName:      Dbname,
-		//UserName: 	 Sysconfig["uname"].(string),
-		//Password: 	 Sysconfig["upwd"].(string),
+		UserName: 	 Sysconfig["uname"].(string),
+		Password: 	 Sysconfig["upwd"].(string),
 	}
 	Mgo.InitPool()
 	//es
@@ -61,7 +61,7 @@ func init() {
 	EsFields = qu.ObjArrToStringArr(econf["esfields"].([]interface{}))
 	//启动es保存
 	go SaveEs()    //过滤后数据
-	go SaveAllEs() //所有数据
+	//go SaveAllEs() //所有数据
 	//初始化其他信息
 	TaskTime = qu.IntAll(Sysconfig["tasktime"])
 	//Updatetime = qu.Int64All(Sysconfig["updatetime"])
@@ -71,9 +71,9 @@ func init() {
 }
 
 func main() {
-	//go TimeTask()
+	go TimeTask()
 	//QyxyStandard()
-	HistoryQyxyStandard()
+	//HistoryQyxyStandard()
 	ch := make(chan bool, 1)
 	<-ch
 }

qyxy/src/task.go

@@ -60,7 +60,6 @@ type City struct {
 
 //定时任务
 func TimeTask() {
-	StartTask()
 	c := cron.New()
 	cronstr := "0 0 15 ? * Tue" //每周二15点执行
 	//cronstr := "0 */" + fmt.Sprint(TaskTime) + " * * * ?" //每TaskTime小时执行一次
@@ -392,7 +391,7 @@ func QyxyStandard() bool {
 				mgoMap[k] = v
 			}
 			//es数据过滤
-			EsSaveFlag := true
+			//EsSaveFlag := true
 			company_type := qu.ObjToString(esMap["company_type"])
 			company_name := qu.ObjToString(esMap["company_name"])
 			if company_type == "个体工商户" {
@@ -417,21 +416,19 @@ func QyxyStandard() bool {
 				}
 			}
 			lock.Lock()
-			if EsSaveFlag {
-				if esMap["history_name"] != nil {
-						var nameArr []string
-						for _, v := range strings.Split(qu.ObjToString(esMap["history_name"]), "；") {
-							if v != "" {
-								nameArr = append(nameArr, v)
-							}
-						}
-						if len(nameArr) > 0 {
-							esMap["history_name"] = nameArr
-						}
+			if esMap["history_name"] != nil {
+				var nameArr []string
+				for _, v := range strings.Split(qu.ObjToString(esMap["history_name"]), "；") {
+					if v != "" {
+						nameArr = append(nameArr, v)
+					}
+				}
+				if len(nameArr) > 0 {
+					esMap["history_name"] = nameArr
 				}
-				EsSaveCache <- esMap //过滤后数据保存
 			}
-			EsSaveAllCache <- esMap //所有数据保存
+			EsSaveCache <- esMap //过滤后数据保存
+			//EsSaveAllCache <- esMap //所有数据保存
 			update = append(update, map[string]interface{}{"$set": mgoMap})
 			SaveHistoryName(tmp)		//保存曾用名
 			if len(update) == 2 {
@@ -846,7 +843,6 @@ func SaveEs() {
 					defer func() {
 						<-SP
 					}()
-					qu.Debug(Index, Itype, arru)
 					Es.BulkSave(Index, Itype, &arru, true)
 				}(arru[:indexu])
 				arru = make([]map[string]interface{}, 500)

qyxy_change/qy_baidu/main.go

@@ -37,8 +37,8 @@ func init() {
 		MongodbAddr: util.ObjToString((*pingan)["dbServer"]),
 		Size:        util.IntAll((*pingan)["dbSize"]),
 		DbName:      Dbname_pa,
-		UserName: 	 util.ObjToString((*pingan)["uname"]),
-		Password: 	 util.ObjToString((*pingan)["upwd"]),
+		//UserName: 	 util.ObjToString((*pingan)["uname"]),
+		//Password: 	 util.ObjToString((*pingan)["upwd"]),
 	}
 	MgoMix.InitPool()
 

qyxy_change/qy_baidu/task.go

@@ -14,7 +14,7 @@ import (
 )
 
 func TimeTask() {
-	//GetPaData()
+	GetPaData()
 	c := cron.New()
 	cronstrBd := "0 0 */" + fmt.Sprint(BdTaskTime) + " * * ?" 		//每TaskTime小时执行一次
 	//cronstr := "0 0 " + fmt.Sprint(TaskTime) + " * * ?"			//每天TaskTime跑一次
@@ -24,7 +24,7 @@ func TimeTask() {
 	c.Start()
 }
 
-func GetBdData()  {
+func GetBdData() {
 	count := 0
 	lastid := ""
 	sess := MgoBd.GetMgoConn()
@@ -46,7 +46,6 @@ func GetBdData()  {
 
 func GetPaData() {
 	count := 0
-	lastid := ""
 	sess := MgoMix.GetMgoConn()
 	defer MgoMix.DestoryMongoConn(sess)
 	fields := map[string]interface{}{"changes": 1, "company_id": 1, "company_name": 1, "company_type": 1, "establish_date": 1, "create_time": 1}
@@ -55,9 +54,8 @@ func GetPaData() {
 	util.Debug("ping an count ------", c)
 	tmp := make(map[string]interface{})
 	for query.Next(&tmp) {
-		lastid = mongodb.BsonIdToSId(tmp["company_id"])
 		if count%1000 == 0 {
-			util.Debug("ping an ----current-----", count, lastid)
+			util.Debug("ping an ----current-----", count)
 		}
 		if strings.Contains(util.ObjToString(tmp["company_type"]), "个体") {
 			continue

udpcreateindex/src/biddingall.go

@@ -97,10 +97,10 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 		//	}
 		//}
 
-		// if sensitive := qutil.ObjToString(tmp["sensitive"]); sensitive != "" { //bidding中有敏感词，不生索引
-		// 	tmp = make(map[string]interface{})
-		// 	continue
-		// }
+		if sensitive := qutil.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
+			tmp = make(map[string]interface{})
+			continue
+		}
 		update := map[string]interface{}{}
 		del := map[string]interface{}{} //记录extract没有值而bidding中有值的字段
 		//对比方法----------------
@@ -232,31 +232,26 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 			for tk, tv := range update {
 				tmp[tk] = tv
 			}
-			if tmp["s_winner"] != "" {
-				sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
-				var cid []string
-				for _, w := range sWinnerarr {
-					if w != "" {
-						ent, _ := mgostandard.FindOne("qyxy_historyname", map[string]interface{}{"company_name": w})
-						if len(*ent) > 0 {
-							cid = append(cid, qutil.ObjToString((*ent)["company_id"]))
-						}else {
-							ent, _ = mgostandard.FindOne("qyxy_std", map[string]interface{}{"company_name": w})
-							if len(*ent) > 0 {
-								cid = append(cid, qutil.ObjToString((*ent)["_id"]))
-							}
-						}
-					}
-				}
-				if len(cid) > 0 {
-					tmp["entidlist"] = cid
-					update["entidlist"] = cid
-					tmp_up := []map[string]interface{}{}
-					tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
-					tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
-					UpdataMgoCache <- tmp_up
-				}
-			}
+			//if tmp["s_winner"] != "" {
+			//	sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
+			//	var cid []string
+			//	for _, w := range sWinnerarr {
+			//		if w != "" {
+			//			ent, _ := mgostandard.FindOne("qyxy_historyname", map[string]interface{}{"company_name": w})
+			//			if len(*ent) > 0 {
+			//				cid = append(cid, qutil.ObjToString((*ent)["company_id"]))
+			//			}
+			//		}
+			//	}
+			//	if len(cid) > 0 {
+			//		tmp["entidlist"] = cid
+			//		update["entidlist"] = cid
+			//		tmp_up := []map[string]interface{}{}
+			//		tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
+			//		tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
+			//		UpdataMgoCache <- tmp_up
+			//	}
+			//}
 			//对projectscope字段的索引处理
 			ps, _ := tmp["projectscope"].(string)
 			if len(ps) > ESLEN {
@@ -382,7 +377,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 							if tmp[field] != nil && qutil.Int64All(tmp[field]) > 0 {
 								newTmp[field] = qutil.Int64All(tmp[field])
 							}
-						} else if field == "entidlist" {
+						} else if field == "s" {
 							newTmp[field] = tmp[field]
 						} else { //其它字段判断数据类型，不正确舍弃
 							if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {

udpcreateindex/src/biddingindex.go

@@ -54,7 +54,6 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 
 	//bidding库
 	session := mgo.GetMgoConn()
-	qutil.Debug(db, c)
 	count, _ := session.DB(db).C(c).Find(&q).Count()
 	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
 	n1, n2 := 0, 0
@@ -144,10 +143,10 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 	log.Println("开始迭代..")
 	for n, tmp := range infos {
 		n1++
-		// if sensitive := qutil.ObjToString(tmp["sensitive"]); sensitive != "" { //bidding中有敏感词，不生索引
-		// 	tmp = make(map[string]interface{})
-		// 	continue
-		// }
+		if sensitive := qutil.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
+			tmp = make(map[string]interface{})
+			continue
+		}
 		update := map[string]interface{}{} //要更新的mongo数据
 		//对比方法----------------
 		tid := mongodb.BsonIdToSId(tmp["_id"])
@@ -546,6 +545,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 	}
 	if len(arrEs) > 0 {
 		tmps := arrEs
+		log.Println("---", tmps[0])
 		elastic.BulkSave(index, itype, &tmps, true)
 		if other_index != "" && other_itype != "" {
 			bidding_other_es.BulkSave(other_index, other_itype, &tmps, true)

udpcreateindex/src/config.json

@@ -6,7 +6,7 @@
   "mongodb": {
     "addr": "192.168.3.207:27092",
     "pool": 10,
-    "db": "wjh"
+    "db": "qfw_data"
   },
   "savedb": {
     "addr": "192.168.3.207:27092",
@@ -36,7 +36,7 @@
     "type": "bidding"
   },
   "bidding": {
-    "db": "wjh",
+    "db": "qfw_data",
     "collect": "bidding",
     "index": "bidding",
     "type": "bidding",
@@ -98,7 +98,7 @@
     "type": "qyxy_ent"
   },
   "standard": {
-    "addr": "192.168.3.205:27082,192.168.3.205:27083",
+    "addr": "192.168.3.207:27092",
     "pool": 10,
     "db": "mixdata",
     "winnerent": {

udpcreateindex/src/main.go

@@ -9,8 +9,6 @@ import (
 	"qfw/util"
 	elastic "qfw/util/elastic"
 	"strings"
-
-	//"time"
 	u "util"
 )
 
@@ -49,7 +47,7 @@ var SP = make(chan bool, 5)
 func init() {
 	util.ReadConfig(&Sysconfig)
 	//inits()
-	go checkMapJob()
+	//go checkMapJob()
 	detailLength = util.IntAllDef(Sysconfig["detaillength"], 50000)
 	fileLength = util.IntAllDef(Sysconfig["filelength"], 50000)
 	updport, _ = Sysconfig["updport"].(string)
@@ -106,11 +104,11 @@ func init() {
 		MongodbAddr: standard["addr"].(string),
 		Size:        util.IntAllDef(standard["pool"], 5),
 		DbName:      standard["db"].(string),
-		UserName:    Sysconfig["uname"].(string),
-		Password:    Sysconfig["upwd"].(string),
-		ReplSet: 	 "bidding",
+		//UserName:    Sysconfig["uname"].(string),
+		//Password:    Sysconfig["upwd"].(string),
 	}
 	mgostandard.InitPool()
+
 	//初始化es
 	//bidding
 	econf := Sysconfig["elastic"].(map[string]interface{})
@@ -184,6 +182,7 @@ func init() {
 	log.Println(purchasinglistFields)
 	//初始化oss
 	u.InitOss()
+
 }
 
 func main() {

udpcreateindex/src/util/ossclient.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	ossEndpoint        = "oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
+	ossEndpoint        = "oss-cn-beijing.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
 	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
 	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
 	ossBucketName      = "topjy"