Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

fullproject/src_v1/config.json

@@ -1,13 +1,13 @@
 {
     "loadStart": 0,
 	"validdays":150,
-    "statusdays": 7,
-	"mongodbServers": "192.168.3.166:27082",
+    "statusdays": 15,
+	"mongodbServers": "192.168.3.207:27092",
     "mongodbPoolSize": 10,
-    "mongodbName": "zhaolongyue",
+    "mongodbName": "extract_kf",
 	"hints":"publishtime_1",
-    "extractColl": "huawei_bidding_all_0110_v2",
-    "projectColl": "huawei_project_0113_v2",
+    "extractColl": "jh_info",
+    "projectColl": "jh_project",
     "backupFlag": false,
     "siteColl": "site",
     "thread": 1,

fullproject/src_v1/init.go

@@ -325,6 +325,7 @@ type Site struct {
 	City     string `json:"city"`     //市
 	District string `json:"district"` //区、县
 	Domain   string `json:"domain"`   //地址
+	Status	 int	`json:"status"`	 //
 }
 
 //二分字符串查找

fullproject/src_v1/load_data.go

@@ -115,7 +115,8 @@ func (p *ProjectTask) loadSite() {
 			select {
 			case tmp := <-pool:
 				n++
-				if tmp != nil {
+				//站点有效标记state
+				if tmp != nil && tmp.Status == 5 {
 					p.mapSite[tmp.Site] = tmp
 				}
 			case <-over:

fullproject/src_v1/main.go

@@ -58,9 +58,9 @@ func DealSign() {
 	}
 }
 
-func main() {
+func mainT() {
 	//udp跑增量  id段   project
-	//udp跑全量			ql
+	//udp跑全量			qlT
 	//udp跑历史数据  信息id1,id2/或id段  ls
 	//udp强制合并  信息id1,id2,id3 [项目id] 不存在时新建  qzhb
 	//udp强制拆分  项目id,信息id1,id2          qzcf
@@ -77,9 +77,9 @@ func main() {
 }
 
 //测试组人员使用
-func mainT() {
-	sid = "5d18eca4a5cb26b9b7c7f587"
-	eid = "5e381b7650b5ea296ed16e51"
+func main() {
+	sid = "5c90370ca5cb26b9b72b3d0a"
+	eid = "5d3a88ffa5cb26b9b7755564"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()

fullproject/src_v1/project.go

@@ -5,14 +5,12 @@ import (
 	"log"
 	"time"
 
-	//	"log"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"math"
 	qu "qfw/util"
 	"sort"
 	"strings"
-	//"gopkg.in/mgo.v2/bson"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
 /**

src/main_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"jy/admin/track"
+	"jy/clear"
 	"jy/extract"
 	. "jy/mongodbutil"
 	"log"
@@ -133,3 +134,8 @@ func Test_buyer(t *testing.T) {
 		}
 	}
 }
+
+func Test_util1(t *testing.T) {
+	data := clear.CutSymbol([]interface{}{"----------123123", "-----123123"})
+	fmt.Println(data)
+}

udpcreateindex/src/biddingall.go

@@ -100,15 +100,21 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				tid := qutil.BsonIdToSId(tmp["_id"])
 				if cid == tid {
 					bnil = false
-					//更新bidding表，生成索引
+					//更新bidding表，生成索引;bidding表中modifyinfo中的字段不更新
+					modifyinfo := make(map[string]bool)
+					if tmpmodifyinfo, ok := tmp["modifyinfo"].(map[string]interface{}); ok && tmpmodifyinfo != nil {
+						for k, v := range tmpmodifyinfo {
+							modifyinfo[k] = v.(bool)
+						}
+					}
 					for _, k := range fields { //fields更新到mongo的字段
 						v1 := compare[k] //extract
 						v2 := tmp[k]     //bidding
 						if v2 == nil && v1 != nil {
 							update[k] = v1
-						} else if v2 != nil && v1 != nil {
+						} else if v2 != nil && v1 != nil && !modifyinfo[k] {
 							update[k] = v1
-						} else if v2 != nil && v1 == nil { //
+						} else if v2 != nil && v1 == nil && !modifyinfo[k] { //
 							if k == "s_subscopeclass" && del["subscopeclass"] == nil {
 								continue
 							} else if k == "s_topscopeclass" && del["topscopeclass"] == nil {

udpcreateindex/src/bidingpurchasing.go

@@ -7,7 +7,7 @@ import (
 	"sync"
 	"unicode/utf8"
 
-	u "util"
+	u "./util"
 
 	"gopkg.in/mgo.v2/bson"
 )

udpcreateindex/src/config.json

@@ -30,8 +30,8 @@
     },
     "bidding": {
         "db": "mxs",
-        "collect": "test",
-        "index": "bidding_v2",
+        "collect": "test1",
+        "index": "bidding_v1",
         "type": "bidding",
         "extractdb": "mxs",
         "extractcollect": "extract",
@@ -95,11 +95,5 @@
     "elastic": {
         "addr": "http://192.168.3.128:9800",
         "pool": 12
-    },
-    "elastic_other": {
-        "addr": "http://127.0.0.1:9800",
-        "pool": 12,
-        "index": "bidding_v2",
-        "type": "bidding"
     }
 }

udpcreateindex/src/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	u "./util"
 	"encoding/json"
 	"log"
 	mu "mfw/util"
@@ -10,7 +11,6 @@ import (
 	"qfw/util/mongodb"
 	"strings"
 	"time"
-	u "util"
 )
 
 var (
@@ -40,8 +40,8 @@ var (
 
 func init() {
 	util.ReadConfig(&Sysconfig)
-	inits()
-	go checkMapJob()
+	//inits()
+	//go checkMapJob()
 	detailLength = util.IntAllDef(Sysconfig["detaillength"], 50000)
 	fileLength = util.IntAllDef(Sysconfig["filelength"], 50000)
 	updport, _ = Sysconfig["updport"].(string)
@@ -141,7 +141,7 @@ func init() {
 }
 
 func main() {
-	go task_index()
+	//go task_index()
 	//task_qyxyindex()
 	updport := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}

udpcreateindex/src/util/ossclient.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	ossEndpoint        = "http://oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
+	ossEndpoint        = "http://oss-cn-beijing.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
 	ossAccessKeyId     = "LTAI4FvLSWN3Wz9F6dUxQGMR"
 	ossAccessKeySecret = "WnQpnNVEiRfZsz5hIqFSr0phayMo3U"
 	ossBucketName      = "topjy"

udps/main.go

@@ -21,13 +21,13 @@ func main() {
 	flag.StringVar(&startDate, "start", "", "开始日期2006-01-02")
 	flag.StringVar(&endDate, "end", "", "结束日期2006-01-02")
 	flag.StringVar(&ip, "ip", "127.0.0.1", "ip")
-	flag.IntVar(&p, "p", 0, "端口")
+	flag.IntVar(&p, "p", 1483, "端口")
 	flag.IntVar(&tmptime, "tmptime", 0, "时间查询")
 	flag.StringVar(&tmpkey, "tmpkey", "", "时间字段")
-	flag.StringVar(&id1, "gtid", "", "gtid")
-	flag.StringVar(&id2, "lteid", "", "lteid")
+	flag.StringVar(&id1, "gtid", "5ed869759e628c5991636bca", "gtid")
+	flag.StringVar(&id2, "lteid", "5eec14bd4c87bb08581c03cb", "lteid")
 	flag.StringVar(&ids, "ids", "", "id1,id2")
-	flag.StringVar(&stype, "stype", "", "stype,传递类型")
+	flag.StringVar(&stype, "stype", "biddingall", "stype,传递类型")
 	flag.StringVar(&bkey, "bkey", "", "bkey,加上此参数表示不生关键词和摘要")
 	flag.StringVar(&q, "q", "", "q查询语句\"{'':''}\",有q就不要gtid,lteid")
 	flag.StringVar(&param, "param", "", "param,生信息发布或其他索引时用双引号套单引号\"{'mgoaddr':'','d':'','c':'','index':'','type':''}\"")