增加全量项目合并

udpprojectset/src/cleareids.go

@@ -100,7 +100,7 @@ func clearIdsKeys(pKey *KeyMap, nowtime int64) []string {
 }
 
 func deleteSliceId(a []string, id string) *[]string {
-	ret := make([]string, 0, len(a))
+	ret := make([]string, 0)
 	for _, val := range a {
 		if val != id {
 			ret = append(ret, val)

udpprojectset/src/config.json

@@ -23,7 +23,8 @@
     },
     "taskstock": {
         "open": true,
-        "endate": "2019-06-30"
+        "endate": "2019-06-30",
+		"startTime":1325347200
     },
     "udpport": ":1482",
     "nextNode": [

udpprojectset/src/fulldata.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"log"
-	"strings"
 
 	"qfw/util"
 	"qfw/util/mongodb"
@@ -14,10 +13,13 @@ import (
 
 var FullCount = 0
 
-func RunFullData() {
+func RunFullData(startTime int64) {
+	if startTime < 1325347200 {
+		log.Println("时间错误", startTime)
+	}
 	defer util.Catch()
 	var wg = sync.WaitGroup{}
-	startTime := int64(1325347200) //2012-01-01
+	//startTime := int64(1325347200) //2012-01-01
 	ps := 3
 	pool := make(chan *task, ps)
 	day := 0
@@ -27,7 +29,7 @@ func RunFullData() {
 		bComplete := false
 		for {
 			if startTime > now || bComplete {
-				log.Println("任务结束")
+				log.Println("任务结束", startTime)
 				endChan <- true
 				break
 			}
@@ -126,79 +128,3 @@ func (t *task) query() {
 	log.Println("currentFull", FullCount)
 
 }
-
-//获取对比项目数组
-func getComeperProjects2(p PCBV, thisinfo *Info) (res []interface{}, pncb []*CompareInfo) {
-	newarr := []string{}
-	repeatId := map[string]bool{}
-	if p.PnameLen > 0 {
-		pn := NewCompareInfo("pn", thisinfo.PNKey, PNKey)
-		pncb = append(pncb, pn)
-		thisinfo.AllRelatePNKeyMap = map[string]*Key{}
-		pn.KeyMap.Lock.Lock()
-		for k, v := range pn.KeyMap.Map {
-			if strings.Contains(k, pn.Key) || strings.Contains(pn.Key, k) {
-				thisinfo.AllRelatePNKeyMap[k] = v
-				for _, id := range *v.Arr {
-					if !repeatId[id] {
-						newarr = append(newarr, id)
-						repeatId[id] = true
-					}
-				}
-			}
-		}
-		if thisinfo.AllRelatePNKeyMap[pn.Key] == nil {
-			K := &Key{&[]string{}, &sync.Mutex{}}
-			thisinfo.AllRelatePNKeyMap[pn.Key] = K
-			pn.KeyMap.Map[pn.Key] = K
-		}
-		pn.KeyMap.Lock.Unlock()
-	}
-	if p.PcodeLen > 0 {
-		pc := NewCompareInfo("pc", thisinfo.PCKey, PCKey)
-		pncb = append(pncb, pc)
-		thisinfo.AllRelatePCKeyMap = map[string]*Key{}
-		pc.KeyMap.Lock.Lock()
-		for k, v := range pc.KeyMap.Map {
-			if strings.Contains(k, pc.Key) || strings.Contains(pc.Key, k) {
-				thisinfo.AllRelatePCKeyMap[k] = v
-				for _, id := range *v.Arr {
-					if !repeatId[id] {
-						newarr = append(newarr, id)
-						repeatId[id] = true
-					}
-				}
-			}
-		}
-		if thisinfo.AllRelatePCKeyMap[pc.Key] == nil {
-			K := &Key{&[]string{}, &sync.Mutex{}}
-			thisinfo.AllRelatePCKeyMap[pc.Key] = K
-			pc.KeyMap.Map[pc.Key] = K
-		}
-		pc.KeyMap.Lock.Unlock()
-	}
-
-	if p.BuyerLen > 0 {
-		pb := NewCompareInfo("pb", thisinfo.PBKey, PBKey)
-		pncb = append(pncb, pb)
-		pb.KeyMap.Lock.Lock()
-		K := pb.KeyMap.Map[pb.Key]
-		if K == nil {
-			K = &Key{&[]string{}, &sync.Mutex{}}
-			pb.KeyMap.Map[pb.Key] = K
-		} else {
-			for _, id := range *K.Arr {
-				if !repeatId[id] {
-					newarr = append(newarr, id)
-					repeatId[id] = true
-				}
-			}
-		}
-		pb.KeyMap.Lock.Unlock()
-	}
-
-	if len(newarr) > 0 {
-		res = redis.Mget(REDISIDS, newarr)
-	}
-	return
-}

udpprojectset/src/main.go

@@ -146,11 +146,14 @@ func main() {
 	log.Println("load data from redis finished.", n)
 	//清理redis
 	//clearedis()
-	if taskstock, ok := Sysconfig["taskstock"].(map[string]interface{}); ok { //跑存量数据
-		if b, _ := taskstock["open"].(bool); b {
-			endate, _ := taskstock["endate"].(string)
-			taskStock(endate)
-		}
+	//	if taskstock, ok := Sysconfig["taskstock"].(map[string]interface{}); ok { //跑存量数据
+	//		if b, _ := taskstock["open"].(bool); b {
+	//			endate, _ := taskstock["endate"].(string)
+	//			taskStock(endate)
+	//		}
+	//	}
+	if taskstock, ok := Sysconfig["taskstock"].(map[string]interface{}); ok {
+		RunFullData(util.Int64All(taskstock["startTime"]))
 	}
 	updport := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}

udpprojectset/src/projectmeger.go

@@ -375,30 +375,28 @@ func getComeperProjects(p PCBV, thisinfo *Info) (res []interface{}, pncb []*Comp
 		pncb = append(pncb, pb)
 	}
 	repeatId := map[string]bool{}
-	IdLock.Lock() //此处加id锁，会引进多线程的死锁，对比三个大map数组，找到key相同的项目id数组,并去重
+	//IdLock.Lock() //此处加id锁，会引进多线程的死锁，对比三个大map数组，找到key相同的项目id数组,并去重
 	for _, pv := range pncb {
-		if pv != nil {
-			pv.KeyMap.Lock.Lock()
-			K := pv.KeyMap.Map[pv.Key]
-			if K == nil {
-				K = &Key{&[]string{}, &sync.Mutex{}}
-				pv.KeyMap.Map[pv.Key] = K
-			}
-			pv.K = K
-			pv.K.Lock.Lock()
-			pv.KeyMap.Lock.Unlock()
-			defer pv.K.Lock.Unlock()
-			newarr := []string{}
-			for _, id := range *K.Arr {
-				if !repeatId[id] {
-					newarr = append(newarr, id)
-					repeatId[id] = true
-				}
+		pv.KeyMap.Lock.Lock()
+		K := pv.KeyMap.Map[pv.Key]
+		if K == nil {
+			K = &Key{&[]string{}, &sync.Mutex{}}
+			pv.KeyMap.Map[pv.Key] = K
+		}
+		pv.K = K
+		pv.K.Lock.Lock()
+		pv.KeyMap.Lock.Unlock()
+		defer pv.K.Lock.Unlock()
+		newarr := []string{}
+		for _, id := range *K.Arr {
+			if !repeatId[id] {
+				newarr = append(newarr, id)
+				repeatId[id] = true
 			}
-			pv.IdArr = newarr
 		}
+		pv.IdArr = newarr
 	}
-	IdLock.Unlock()
+	//IdLock.Unlock()
 	for _, pv := range pncb {
 		if len(pv.IdArr) > 0 {
 			res = append(res, redis.Mget(REDISIDS, pv.IdArr))