xg

fullproject/src_v1/init.go

@@ -17,6 +17,7 @@ var (
 	Sysconfig                                      map[string]interface{} //读取配置文件
 	MongoTool, MgoBidding                          *MongodbSim            //mongodb连接
 	ExtractColl, ProjectColl, BackupColl, SiteColl string                 //抽取表、项目表、项目快照表、站点表
+	ExtractColl1                                   string
 	Thread                                         int                    //配置项线程数
 	BlackList                                      []interface{}
 	BlaskListMap                                   map[string]bool
@@ -66,6 +67,7 @@ func init() {
 	MgoBidding.InitPool()
 
 	ExtractColl = Sysconfig["extractColl"].(string)
+	ExtractColl1 = Sysconfig["extractColl1"].(string)
 	ProjectColl = Sysconfig["projectColl"].(string)
 	BackupColl = Sysconfig["projectColl"].(string) + "_back"
 	SiteColl = Sysconfig["siteColl"].(string)

fullproject/src_v1/mgotool.go

@@ -124,7 +124,10 @@ func (ms *MgoSess) Iter() *MgoIter {
 }
 
 func (ms *MgoSess) Count() (int64, error) {
-	return ms.M.C.Database(ms.Db).Collection(ms.Coll).CountDocuments(ms.M.Ctx, ms.Query)
+	if ms.Query != nil {
+		return ms.M.C.Database(ms.Db).Collection(ms.Coll).CountDocuments(ms.M.Ctx, ms.Query)
+	}
+	return ms.M.C.Database(ms.Db).Collection(ms.Coll).EstimatedDocumentCount(ms.M.Ctx)
 }
 
 type MongodbSim struct {
@@ -189,9 +192,9 @@ func (m *MongodbSim) UpSertBulk(c string, doc ...[]map[string]interface{}) bool
 		write.SetUpsert(true)
 		writes = append(writes, write)
 	}
-	_, e := coll.BulkWrite(m.Ctx, writes)
+	r, e := coll.BulkWrite(m.Ctx, writes)
 	if e != nil {
-		log.Println("mgo upsert error:", e.Error())
+		log.Println("mgo upsert error:", e.Error(), r)
 		return false
 	}
 	return true

fullproject/src_v1/project.go

@@ -601,6 +601,9 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 			set["qualifies"] = strings.Join(str, ",")
 		}
 	}
+	if len(p1.EntIdList) > 0 {
+		set["entidlist"] = p1.EntIdList
+	}
 	p1.InfoFiled = make(map[string]InfoField)
 	infofield := InfoField{
 		Budget:       thisinfo.Budget,

fullproject/src_v1/task.go

@@ -422,7 +422,7 @@ func (p *ProjectTask) delInfoPro(udpInfo map[string]interface{}) {
 	}
 	client := Es.GetEsConn()
 	defer Es.DestoryEsConn(client)
-	esquery := `{"query": {"bool": {"must": [{"match": {"ids": "`+infoid+`"}}]}}}`
+	esquery := `{"query": {"bool": {"must": [{"term": {"ids": "`+infoid+`"}}]}}}`
 	data := Es.Get(Index, Itype, esquery)
 	if len(*data) > 0 {
 		pid := util.ObjToString(((*data)[0])["_id"])
@@ -490,7 +490,9 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 						p.CommonMerge(tmp, info)
 					} else {
 						//信息错误,进行更新
+						p.mapBidLock.Lock()
 						countRepeat++
+						p.mapBidLock.Unlock()
 					}
 				}(tmp)
 			case <-over:

fullproject/src_v1/update.go

@@ -88,6 +88,13 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 			for _, v := range proList{
 				v1 := v.(map[string]interface{})
 				temp := MongoTool.FindById(ExtractColl, qu.ObjToString(v1["infoid"]))
+				if len(temp) == 0 {
+					temp = MongoTool.FindById(ExtractColl1, qu.ObjToString(v1["infoid"]))
+					if len(temp) == 0 {
+						qu.Debug("extract not find id...", v1["infoid"])
+						continue
+					}
+				}
 				tempInfo := ParseInfo(temp)
 				if flag {
 					merge := p.ReMerge(tempInfo, temp, tmpPro)
@@ -197,9 +204,9 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 		backupPro(tmpPro)
 		c := MongoTool.Delete(ProjectColl, pid)
 		if c > 0 {
-			//client := Es.GetEsConn()
-			//defer Es.DestoryEsConn(client)
-			//Es.DelById(Itype, Index, pid)
+			client := Es.GetEsConn()
+			defer Es.DestoryEsConn(client)
+			Es.DelById(Itype, Index, pid)
 		}
 		return
 	}
@@ -225,6 +232,13 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 		for _, v := range proList{
 			v1 := v.(map[string]interface{})
 			temp := MongoTool.FindById(ExtractColl, qu.ObjToString(v1["infoid"]))
+			if len(temp) == 0 {
+				temp = MongoTool.FindById(ExtractColl1, qu.ObjToString(v1["infoid"]))
+				if len(temp) == 0 {
+					qu.Debug("extract not find id...", v1["infoid"])
+					continue
+				}
+			}
 			tempInfo := ParseInfo(temp)
 			if flag {
 				merge := p.ReMerge(tempInfo, temp, tmpPro)
@@ -447,6 +461,13 @@ func (p *ProjectTask) innerMerge(infoList []interface{}, tmpPro map[string]inter
 	for k, m := range infoList{
 		m1 := m.(map[string]interface{})
 		temp := MongoTool.FindById(ExtractColl, qu.ObjToString(m1["infoid"]))
+		if len(temp) == 0 {
+			temp = MongoTool.FindById(ExtractColl1, qu.ObjToString(m1["infoid"]))
+			if len(temp) == 0 {
+				qu.Debug("extract not find id...", m1["infoid"])
+				continue
+			}
+		}
 		tempInfo := ParseInfo(temp)
 		if k == 0 {
 			p1 = p.newPro(temp, newP, tempInfo)
@@ -467,6 +488,13 @@ func (p *ProjectTask) innerMerge1(infoList []interface{}, infoid string, tmpPro
 			continue
 		}
 		temp := MongoTool.FindById(ExtractColl, qu.ObjToString(m1["infoid"]))
+		if len(temp) == 0 {
+			temp = MongoTool.FindById(ExtractColl1, qu.ObjToString(m1["infoid"]))
+			if len(temp) == 0 {
+				qu.Debug("extract not find id...", m1["infoid"])
+				continue
+			}
+		}
 		tempInfo := ParseInfo(temp)
 		if k == 0 {
 			p1 = p.newPro(temp, newP, tempInfo)

qyxy/src/main.go

@@ -70,8 +70,9 @@ func init() {
 }
 
 func main() {
-	//go TimeTask()
-	QyxyStandard()
+	go TimeTask()
+	//QyxyStandard()
+	//HistoryQyxyStandard()
 	ch := make(chan bool, 1)
 	<-ch
 }

qyxy/src/task.go

@@ -97,6 +97,9 @@ func QyxyStandard() bool {
 	wg := &sync.WaitGroup{}
 	lock := &sync.Mutex{} //控制读写
 	arr := [][]map[string]interface{}{}
+	//q := map[string]interface{}{
+	//	"company_name" : "北京教培师训网络科技股份有限公司",
+	//}
 	count, _ := sess.DB(Dbname).C(Dbcoll).Find(nil).Count()
 	log.Println("共查询:", count, "条")
 	if count == 0 {
@@ -377,23 +380,17 @@ func QyxyStandard() bool {
 			}
 			//es数据过滤
 			EsSaveFlag := true
-			company_name := qu.ObjToString(esMap["company_name"])
-			if len([]rune(company_name)) < 8 {
-				EsSaveFlag = false
-			}
-			if EsSaveFlag {
-				company_type := qu.ObjToString(esMap["company_type"])
-				if company_type == "" || company_type == "个体工商户" {
-					EsSaveFlag = false
-				}
-			}
-			if EsSaveFlag {
-				status := qu.ObjToString(esMap["company_status"])
-				if status != "正常" {
+			company_type := qu.ObjToString(esMap["company_type"])
+			if company_type == "个体工商户" {
+				esMap["company_type_int"] = 1
+				company_name := qu.ObjToString(esMap["company_name"])
+				if len([]rune(company_name)) < 5 {
 					EsSaveFlag = false
 				}
-			}
-			if EsSaveFlag {
+			}else if company_type == "" {
+				EsSaveFlag = false
+			}else {
+				esMap["company_type_int"] = 0
 				credit_no := strings.TrimSpace(qu.ObjToString(esMap["credit_no"]))
 				company_code := strings.TrimSpace(qu.ObjToString(esMap["company_code"]))
 				if credit_no == "" && company_code == "" {
@@ -417,7 +414,7 @@ func QyxyStandard() bool {
 			}
 			EsSaveAllCache <- esMap //所有数据保存
 			update = append(update, map[string]interface{}{"$set": mgoMap})
-			SaveHistoryName(tmp)
+			SaveHistoryName(tmp)		//保存曾用名
 			if len(update) == 2 {
 				arr = append(arr, update)
 			}
@@ -728,19 +725,20 @@ func HistoryQyxyStandard() {
 			}
 			//es数据过滤
 			EsSaveFlag := true
-			company_name := qu.ObjToString(esMap["company_name"])
-			if len([]rune(company_name)) < 8 {
-				EsSaveFlag = false
-			}
-			if EsSaveFlag {
-				company_type := qu.ObjToString(esMap["company_type"])
-				if company_type == "" || company_type == "个体工商户" {
+			company_type := qu.ObjToString(esMap["company_type"])
+			if company_type == "个体工商户" {
+				esMap["company_type_int"] = 1
+				company_name := qu.ObjToString(esMap["company_name"])
+				if len([]rune(company_name)) < 5 {
 					EsSaveFlag = false
 				}
-			}
-			if EsSaveFlag {
-				status := qu.ObjToString(esMap["company_status"])
-				if status != "正常" {
+			}else if company_type == "" {
+				EsSaveFlag = false
+			}else {
+				esMap["company_type_int"] = 0
+				credit_no := strings.TrimSpace(qu.ObjToString(esMap["credit_no"]))
+				company_code := strings.TrimSpace(qu.ObjToString(esMap["company_code"]))
+				if credit_no == "" && company_code == "" {
 					EsSaveFlag = false
 				}
 			}
@@ -766,9 +764,9 @@ func HistoryQyxyStandard() {
 				}
 				EsSaveCache <- esMap //过滤后数据保存
 			}
-			EsSaveAllCache <- esMap //所有数据保存
-			SaveHistoryName(tmp)
-			update = append(update, map[string]interface{}{"$set": mgoMap})
+			//EsSaveAllCache <- esMap //所有数据保存
+			//SaveHistoryName(tmp)
+			//update = append(update, map[string]interface{}{"$set": mgoMap})
 			if len(update) == 2 {
 				arr = append(arr, update)
 			}
@@ -896,10 +894,17 @@ func InitAddress() {
 	log.Println("Init Address...")
 	AddressMap = map[string]*City{}
 	AddressOldMap = map[string]*City{}
-	address, _ := Mgo.Find("address_new_2020", nil, nil, nil, false, -1, -1)
-	for _, tmp := range *address {
+
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+	result := sess.DB(Dbname).C("address_new_2020").Find(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); result.Next(&tmp); count++ {
 		code := qu.ObjToString(tmp["code"])
 		codeLen := len(code)
+		if codeLen > 6 {
+			continue
+		}
 		if t_code := CodeMap[codeLen]; t_code != "" { //新的address表补齐code
 			code = code + t_code
 		}
@@ -915,6 +920,7 @@ func InitAddress() {
 			AddressMap[code] = city
 		}
 	}
+	qu.Debug("Init Address end...", len(AddressMap), len(AddressOldMap))
 }
 
 func InitQyStype() {

udpcreateindex/src/biddingall.go

@@ -90,6 +90,13 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 		// 	tmp = make(map[string]interface{})
 		// 	continue
 		// }
+		//if qutil.IntAll(tmp["repeat"]) != 0 {
+		//	esQ := `{"query": {"bool": {"must": [{"term": {"id": "`+ mongodb.BsonIdToSId(tmp["_id"]) +`"}}]}}}`
+		//	esData := elastic.Get(index, itype, esQ)
+		//	if len(*esData) > 0 {
+		//		elastic.DelById(index, itype, mongodb.BsonIdToSId(tmp["_id"]))
+		//	}
+		//}
 
 		// if sensitive := qutil.ObjToString(tmp["sensitive"]); sensitive != "" { //bidding中有敏感词，不生索引
 		// 	tmp = make(map[string]interface{})
@@ -483,75 +490,6 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 	UpdatesLock.Unlock()
 	log.Println(mapInfo, "create bidding index...over", n)
 }
-func preNUm(data byte) int {
-	var mask byte = 0x80
-	var num int = 0
-	//8bit中首个0bit前有多少个1bits
-	for i := 0; i < 8; i++ {
-		if (data & mask) == mask {
-			num++
-			mask = mask >> 1
-		} else {
-			break
-		}
-	}
-	return num
-}
-
-func isGBK(data []byte) bool {
-	length := len(data)
-	var i int = 0
-	for i < length {
-		if data[i] <= 0x7f {
-			//编码0~127,只有一个字节的编码，兼容ASCII码
-			i++
-			continue
-		} else {
-			//大于127的使用双字节编码，落在gbk编码范围内的字符
-			if data[i] >= 0x81 &&
-				data[i] <= 0xfe &&
-				data[i+1] >= 0x40 &&
-				data[i+1] <= 0xfe &&
-				data[i+1] != 0xf7 {
-				i += 2
-				continue
-			} else {
-				return false
-			}
-		}
-	}
-	return true
-}
-
-func isUtf8(data []byte) bool {
-	i := 0
-	for i < len(data) {
-		if (data[i] & 0x80) == 0x00 {
-			// 0XXX_XXXX
-			i++
-			continue
-		} else if num := preNUm(data[i]); num > 2 {
-			// 110X_XXXX 10XX_XXXX
-			// 1110_XXXX 10XX_XXXX 10XX_XXXX
-			// 1111_0XXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
-			// 1111_10XX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
-			// 1111_110X 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
-			// preNUm() 返回首个字节的8个bits中首个0bit前面1bit的个数，该数量也是该字符所使用的字节数
-			i++
-			for j := 0; j < num-1; j++ {
-				//判断后面的 num - 1 个字节是不是都是10开头
-				if (data[i] & 0xc0) != 0x80 {
-					return false
-				}
-				i++
-			}
-		} else {
-			//其他情况说明不是utf-8
-			return false
-		}
-	}
-	return true
-}
 
 //更新extract表
 func UpdateExtract() {

udpcreateindex/src/config.json

@@ -61,7 +61,7 @@
     "projectinfomap": {
       "approvecode": "string", "approvecontent": "string", "approvestatus": "string", "approvetime": "string", "approvedept": "string", "approvenumber": "string", "projecttype": "string", "approvecity": "string"
     },
-    "purchasinglist": "itemname,brandname,model,unitname,number",
+    "purchasinglist": "itemname,brandname,model,unitname,number,unitprice,totalprice",
     "purchasinglistmap": {
       "itemname": "string", "brandname": "string", "model": "string", "unitname": "string", "number": "float64", "unitprice": "float64", "totalprice": "float64"
     },