xg

udpcreateindex/src/biddingdata.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	//	"fmt"
+	"fmt"
 	"log"
 	qutil "qfw/util"
 	elastic "qfw/util/elastic"
@@ -122,12 +122,11 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 					bnil = false
 					//更新bidding表，生成索引
 					for _, k := range fields {
-						v1 := compare[k]
-						v2 := tmp[k]
+						v1 := compare[k] //extract
+						v2 := tmp[k]     //bidding
 						if v2 == nil && v1 != nil {
 							update[k] = v1
 						} else if v2 != nil && v1 != nil {
-							//update[k+"_b"] = v2
 							update[k] = v1
 						} else if v2 != nil && v1 == nil {
 							update[k] = v2
@@ -165,14 +164,12 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 			if !bnil && compare != nil {
 				subscopeclass, _ := compare["subscopeclass"].([]interface{})
 				if subscopeclass != nil {
-					//str := ","
 					m1 := map[string]bool{}
 					newclass := []string{}
 					for _, sc := range subscopeclass {
 						sclass, _ := sc.(string)
 						if !m1[sclass] {
 							m1[sclass] = true
-							//str += sclass + ","
 							newclass = append(newclass, sclass)
 						}
 					}
@@ -213,14 +210,26 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 			for tk, tv := range update {
 				tmp[tk] = tv
 			}
+			if tmp["supervisorrate"] != nil { //临时处理supervisorrate抽取类型为string不生索引
+				if _, ok := tmp["supervisorrate"].(string); ok { //supervisorrate数据为string类型
+					delete(tmp, "supervisorrate")
+				}
+			}
 			//对projectscope字段的索引处理
 			ps, _ := tmp["projectscope"].(string)
-			if ps == "" {
-				tmp["projectscope"] = "" //= tmp["detail"]
-			}
 			if len(ps) > ESLEN {
 				tmp["projectscope"] = string(([]rune(ps))[:4000])
 			}
+			//对标的物为空处理
+			if filetext := getFileText(tmp); len(filetext) > 0 { //attach_text
+				tmp["filetext"] = filetext
+			}
+			if purchasing, ok := tmp["purchasing"].(string); ok && purchasing == "" {
+				delete(tmp, "purchasing")
+			}
+			if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok && len(purchasinglist) == 0 {
+				delete(tmp, "purchasinglist")
+			}
 			//			if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
 			//				tmp["budget"] = nil
 			//			} else if sbd, ok := tmp["budget"].(string); ok {
@@ -234,7 +243,7 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 
 			if qutil.IntAll(update["extracttype"]) != -1 {
 				newTmp := map[string]interface{}{}
-				for _, v := range indexfield {
+				for _, v := range biddingIndexFields { // indexfield
 					if tmp[v] != nil {
 						if "projectinfo" == v {
 							mp, _ := tmp[v].(map[string]interface{})
@@ -242,10 +251,12 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 								newmap := map[string]interface{}{}
 								for _, v1 := range projectinfoFields {
 									if mp[v1] != nil {
-										newmap[v1] = mp[v1]
+										newmap[v1] = fmt.Sprint(mp[v1])
 									}
 								}
-								newTmp[v] = newmap
+								if len(newmap) > 0 {
+									newTmp[v] = newmap
+								}
 								// attachments := mp["attachments"]
 								// con := ""
 								// if attachments != nil {
@@ -265,6 +276,25 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 								// 	newTmp["attachments"] = con
 								// }
 							}
+						} else if v == "purchasinglist" { //标的物处理
+							purchasinglist_new := []map[string]interface{}{}
+							if pcl, _ := tmp[v].([]interface{}); len(pcl) > 0 {
+								for _, ls := range pcl {
+									lsm_new := make(map[string]interface{})
+									lsm := ls.(map[string]interface{})
+									for _, pf := range purchasinglistFields {
+										if lsm[pf] != nil {
+											lsm_new[pf] = lsm[pf]
+										}
+									}
+									if lsm_new != nil && len(lsm_new) > 0 {
+										purchasinglist_new = append(purchasinglist_new, lsm_new)
+									}
+								}
+							}
+							if len(purchasinglist_new) > 0 {
+								newTmp[v] = purchasinglist_new
+							}
 						} else {
 							if v == "detail" {
 								detail, _ := tmp[v].(string)
@@ -273,8 +303,6 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 								newTmp[v] = tmp[v]
 							}
 						}
-					} else if v == "budget" || v == "bidamount" {
-						newTmp[v] = nil
 					}
 				}
 				UpdatesLock.Lock()
@@ -284,7 +312,7 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 			UpdatesLock.Lock()
 			if len(arrEs) >= BulkSize-1 {
 				tmps := arrEs
-				elastic.BulkSave(index, itype, &tmps, false)
+				elastic.BulkSave(index, itype, &tmps, true)
 				arrEs = []map[string]interface{}{}
 			}
 			UpdatesLock.Unlock()
@@ -300,8 +328,7 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 	UpdatesLock.Lock()
 	if len(arrEs) > 0 {
 		tmps := arrEs
-		log.Println(tmps[0])
-		elastic.BulkSave(index, itype, &tmps, false)
+		elastic.BulkSave(index, itype, &tmps, true)
 	}
 	UpdatesLock.Unlock()
 	log.Println(mapInfo, "create bidding index...over", n)