1.14 代码修改

fullproject/src_v1/config.json

@@ -2,12 +2,12 @@
     "loadStart": 0,
 	"validdays":150,
     "statusdays": 7,
-	"mongodbServers": "192.168.3.207:27092",
+	"mongodbServers": "192.168.3.166:27082",
     "mongodbPoolSize": 10,
-    "mongodbName": "extract_kf",
+    "mongodbName": "zhaolongyue",
 	"hints":"publishtime_1",
-    "extractColl": "ceshi_info",
-    "projectColl": "jh_project",
+    "extractColl": "huawei_bidding_all_0110_v2",
+    "projectColl": "huawei_project_0113_v2",
     "backupFlag": false,
     "siteColl": "site",
     "thread": 1,
@@ -15,7 +15,7 @@
         "to": "wangjianghan@topnet.net.cn",
         "api": "http://10.171.112.160:19281/_send/_mail"
     },
-    "udpport": ":1182",
+    "udpport": "1182",
     "nextNode": [
     ]
 }

fullproject/src_v1/init.go

@@ -64,7 +64,6 @@ func init() {
 	ExtractColl = Sysconfig["extractColl"].(string)
 	ProjectColl = Sysconfig["projectColl"].(string)
 	BackupColl = Sysconfig["projectColl"].(string)+"_back"
-	log.Println(BackupColl)
 	SiteColl = Sysconfig["siteColl"].(string)
 	Thread = util.IntAll(Sysconfig["thread"])
 	//NextNode = Sysconfig["nextNode"].([]interface{})
@@ -275,7 +274,7 @@ type ProjectInfo struct {
 	Ids           []string           `json:"ids,omitempty"`
 	Topscopeclass []string           `json:"topscopeclass,omitempty"`
 	Subscopeclass []string           `json:"subscopeclass,omitempty"` //子行业分类
-	Winners       []string           `json:"winners,omitempty"`       //中标人
+	Winners       []string           `json:"s_winner,omitempty"`       //中标人
 	ProjectName   string             `json:"projectname,omitempty"`   //项目名称
 	ProjectCode   string             `json:"projectcode,omitempty"`   //项目代码唯一（纯数字的权重低）
 	ContractCode  string			 `json:"contractcode,omitempty"`  //项目编号

fullproject/src_v1/main.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"os/signal"
 	"qfw/util"
-	qu "qfw/util"
 	"syscall"
 	"time"
 )
@@ -20,7 +19,7 @@ var (
 	toaddr       = []*net.UDPAddr{} //下节点对象
 	ChSign       = make(chan os.Signal)
 
-	sid, eid string //测试使用
+	sid, eid string 	//测试使用
 )
 
 func init() {
@@ -59,7 +58,7 @@ func DealSign() {
 	}
 }
 
-func main() {
+func mainT() {
 	//udp跑增量  id段   project
 	//udp跑全量			ql
 	//udp跑历史数据  信息id1,id2/或id段  ls
@@ -78,9 +77,9 @@ func main() {
 }
 
 //测试组人员使用
-func mainT() {
-	//sid = "5dfbd43ce9d1f601e43fa402"
-	//eid = "5e0954b30cf41612e061d0c8"
+func main() {
+	//sid = "5649a0fcaf5374672e005704"
+	//eid = "5e169e5250b5ea296ec896f0"
 
 	flag.StringVar(&sid, "sid", "", "开始id")
 	flag.StringVar(&eid, "eid", "", "结束id")
@@ -91,8 +90,8 @@ func mainT() {
 		log.Println("sid, eid参数不能为空")
 		os.Exit(0)
 	}
-	mapinfo["gtid"] = qu.StringTOBsonId(sid)
-	mapinfo["lteid"] = qu.StringTOBsonId(eid)
+	mapinfo["gtid"] = sid
+	mapinfo["lteid"] = eid
 	mapinfo["stype"] = "ql"
 	mapinfo["ip"] = "127.0.0.1"
 	mapinfo["port"] = Sysconfig["udpport"]
@@ -103,52 +102,12 @@ func mainT() {
 		}
 	}
 	P_QL.loadSite()
-	task(mapinfo)
-	SingleThread <- true
+	P_QL.currentType = mapinfo["stype"].(string)
+	P_QL.pici = time.Now().Unix()
+	P_QL.taskQl(mapinfo)
 	time.Sleep(20 * time.Second)
 }
 
-func task(mapInfo map[string]interface{}) {
-	SingleThread <- true
-	tasktype, _ := mapInfo["stype"].(string)
-	log.Println("tasktype:", tasktype)
-	switch tasktype {
-	case "ql": //全量合并
-		go func() {
-			defer func() {
-				<-SingleThread
-			}()
-			P_QL.currentType = tasktype
-			P_QL.pici = time.Now().Unix()
-			P_QL.taskQl(mapInfo)
-		}()
-	case "project": //增量合并，未抽取到项目名称或项目编号的不合并  bidding中mergestatus 1已合并 2字段问题不合并 3历史待合并
-		//合同、验收公告在6个月内查询不到可扩展到两年
-		go func() {
-			defer func() {
-				<-SingleThread
-			}()
-			P_QL.currentType = tasktype
-			P_QL.pici = time.Now().Unix()
-			P_QL.taskZl(mapInfo)
-		}()
-	case "updateInfo": //招标字段变更
-		go func() {
-			defer func() {
-				<-SingleThread
-			}()
-			P_QL.currentType = tasktype
-			P_QL.pici = time.Now().Unix()
-			P_QL.taskUpdateInfo(mapInfo)
-		}()
-	case "history": //历史数据合并,暂时不写
-		go func() {
-			defer func() {
-				<-SingleThread
-			}()
-		}()
-	}
-}
 
 //udp调用信号
 func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
@@ -188,7 +147,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					P_QL.pici = time.Now().Unix()
 					P_QL.taskZl(mapInfo)
 				}()
-			case "updateInfo": //招标字段变更
+			case "updateInfo":		//招标字段变更
 				go func() {
 					defer func() {
 						<-SingleThread

fullproject/src_v1/project.go

@@ -176,8 +176,8 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 
 			ex := 0
 			resArr := []*ProjectInfo{}
-			for _, res := range resN {
-				choose, e := p.CompareStatus(resN[0], info)
+			for i, res := range resN {
+				choose, e := p.CompareStatus(resN[i], info)
 				if !choose {
 					ex = e
 					resArr = append(resArr, res)
@@ -186,6 +186,7 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 			if len(resArr) > 0 {
 				bFindProject = true
 				findPid = resArr[0].Id.Hex()
+				p.UpdateProject(tmp, info, resArr[0], kv+1, resArr[0].comStr, ex)
 				for k2, bv := range []int{bpn, bpc, bptc, bpb} {
 					if bv > -1 {
 						pids[bv].Arr = append(pids[bv].Arr, findPid)
@@ -210,7 +211,6 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 									resArr[0].MPC = append(resArr[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
 								}
 							}
-
 						} else {
 							if resArr[0].Buyer == "" {
 								resArr[0].Buyer = info.Buyer
@@ -218,7 +218,6 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 						}
 					}
 				}
-				p.UpdateProject(tmp, info, resArr[0], kv+1, resArr[0].comStr, ex)
 			} else {
 				bFindProject = false
 				findPid = ""
@@ -506,12 +505,11 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	bt := qu.ObjToString(tmp["toptype"])
 	bs := qu.ObjToString(tmp["subtype"])
 	p.mapBidLock.Lock()
+	set["bidtype"] = bidtype[bs]
 	if bt == "招标" {
 		set["projectscope"] = qu.ObjToString(tmp["projectscope"])
-		set["bidtype"] = bidtype[bs]
 		set["bidstatus"] = bs
 	}else {
-		set["bidtype"] = bt
 		if bidstatus[bs] != "" {
 			set["bidstatus"] = thisinfo.SubType
 		} else if tmp["infoformat"] == 2 {
@@ -522,7 +520,8 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	}
 	p.mapBidLock.Unlock()
 
-	p1, pkg := p.NewCachePinfo(pId, thisinfo, bt)
+	pkg := PackageFormat(thisinfo, nil)
+	p1 := p.NewCachePinfo(pId, thisinfo, bt, pkg)
 	if len(thisinfo.Subscopeclass) > 0 {
 		s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
 		set["s_subscopeclass"] = s_subscopeclass
@@ -567,6 +566,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 			"$set": set,
 		},
 	}
+	//log.Println(set)
 	return pId.Hex(), &p1
 }
 
@@ -614,11 +614,7 @@ func (p *ProjectTask) PushListInfo(tmp map[string]interface{}, infoid string) bs
 }
 
 //生成存放在内存中的对象
-func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidtype string) (ProjectInfo, map[string]interface{}) {
-	pkg := map[string]interface{}{}
-	if thisinfo.HasPackage {
-		pkg = PackageFormat(thisinfo, nil)
-	}
+func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidtype string, pkg map[string]interface{}) ProjectInfo {
 	p1 := ProjectInfo{
 		Id:            id,
 		Ids:           []string{thisinfo.Id},
@@ -649,7 +645,7 @@ func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidty
 	if thisinfo.LenPTC > 5 {
 		p1.MPC = append(p1.MPC, thisinfo.PTC)
 	}
-	return p1, pkg
+	return p1
 }
 
 //更新项目
@@ -683,15 +679,16 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	bt := qu.ObjToString(tmp["toptype"])
 	bs := qu.ObjToString(tmp["subtype"])
 	p.mapBidLock.Lock()
+	if bidtype[bs] != "" {
+		set["bidtype"] = bidtype[bs]
+	}
 	if bt == "招标" {
 		//招标状态，更新projectscope
 		if tmp["projectscope"] != nil {
 			set["projectscope"] = qu.ObjToString(tmp["projectscope"])
 		}
-		set["bidtype"] = bidtype[bs]
 		set["bidstatus"] = bs
 	}else {
-		set["bidtype"] = bt
 		if bidstatus[bs] != "" {
 			set["bidstatus"] = thisinfo.SubType
 		} else if tmp["infoformat"] == 2 {
@@ -807,6 +804,10 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	}
 	//winner
 	if len(thisinfo.Winners) > 0 {
+		if len(pInfo.Winners) <= 0 {
+			set["winner"] = tmp["winner"].(string)
+		}
+
 		sort.Strings(pInfo.Winners)
 		for _, k := range thisinfo.Winners {
 			if BinarySearch(pInfo.Winners, k) == -1 {
@@ -944,7 +945,7 @@ func ComparePlace(project *ProjectInfo, info *Info) bool {
 var PackageEle = []string{
 	"origin",
 	"name",
-	"text",
+	//"text",
 	"budget",
 	"winner",
 	"bidamount",
@@ -969,11 +970,8 @@ func packageEle(map1 map[string]interface{}, id string) map[string]interface{} {
 
 func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
 	p1 := map[string]interface{}{}
-	packageCopy := make(map[string]interface{})
-	if project != nil && project.Package != nil {
-		packageCopy = Copy(project.Package).(map[string]interface{})
-	}
-	if packageCopy != nil && len(packageCopy) > 0 {
+	if project != nil && project.Package != nil && len(project.Package) > 0 {
+		packageCopy := Copy(project.Package).(map[string]interface{})
 		p1 = packageCopy
 		for k, v := range info.Package {
 			if v1, ok := v.(map[string]interface{}); ok {
@@ -985,7 +983,8 @@ func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
 				addFlag := false
 				for k1, v3 := range p1 {
 					if v4, ok := v3.([]map[string]interface{}); ok {
-						if qu.ObjToString(v4[0]["origin"]) == qu.ObjToString(v2["origin"]) && qu.ObjToString(v4[0]["name"]) == qu.ObjToString(v2["name"]) {
+						//if qu.ObjToString(v4[0]["origin"]) == qu.ObjToString(v2["origin"]) && qu.ObjToString(v4[0]["name"]) == qu.ObjToString(v2["name"]) {
+						if k1 == k {
 							v4 = append(v4, v2)
 							p1[k1] = v4
 							addFlag = true
@@ -998,8 +997,9 @@ func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
 				}
 			}
 		}
+		p1 = packageCopy
 	} else {
-		for k, v := range packageCopy {
+		for k, v := range info.Package {
 			v1, _ := v.(map[string]interface{})
 			p2 := map[string]interface{}{}
 			p2 = packageEle(v1, info.Id)
@@ -1009,7 +1009,6 @@ func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
 			p1[k] = []map[string]interface{}{p2}
 		}
 	}
-	p1 = packageCopy
 	return p1
 }
 
@@ -1027,7 +1026,12 @@ func CountAmount(project *ProjectInfo, info *Info) {
 				}
 			}
 		}
-		project.Budget = budget
+		if budget == 0 && info.Budget > 0 {
+			budget = info.Budget
+		}
+		if budget > 0 {
+			project.Budget = budget
+		}
 	}else {
 		//招标没有多包
 		k := KeyPackage.FindStringSubmatch(project.ProjectName)

fullproject/src_v1/task.go

@@ -83,7 +83,7 @@ func NewPT() *ProjectTask {
 		mapPc:     make(map[string]*Key, 5000000),
 		mapHref:   make(map[string]string, 1500000),
 		mapSite:   make(map[string]*Site, 1000000),
-		saveSize:  400,
+		saveSize:  100,
 
 		//saveSign:   make(chan bool, 1),
 		//updateSign: make(chan bool, 1),
@@ -377,7 +377,6 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 	log.Println("start project", q)
 	sess := MongoTool.GetMgoConn()
 	defer MongoTool.DestoryMongoConn(sess)
-
 	infoPool := make(chan map[string]interface{}, 2000)
 	over := make(chan bool)
 	go func() {