Эх сурвалжийг харах

Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

fengweiqiang 5 жил өмнө
parent
commit
1225ffc03a

+ 1 - 1
src/config.json

@@ -30,7 +30,7 @@
     "brandgoods": false,
     "pricenumber":true,
     "udptaskid": "5cdd3025698414032c8322b1",
-    "udpport": "1483",
+    "udpport": "1484",
     "nextNode": [
         {
             "addr": "127.0.0.1",

+ 1 - 1
src/jy/clear/cutspace.go

@@ -16,7 +16,7 @@ var (
 	endSymblo      *regexp.Regexp
 )
 
-var spaces = []string{"\u3000", "\u2003", "\u00a0", "\t", "\r", "\n"}
+var spaces = []string{"\u3000", "\u2003", "\u00a0", "\t", "\r", "\n", "\u0001"}
 
 func init() {
 	cutSpace, _ = regexp.Compile(`^\s*|\s*$`)

+ 12 - 4
src/res/fieldscore.json

@@ -155,20 +155,20 @@
         "positivewords": [
             {
                 "describe": "以*结尾",
-                "regstr": ".{2,100}(委员会|管委会|医院|卫计委|机关|社区|中心|中心校|分校|办公室|学校|幼儿园|动物园|管理站|馆|基地|青年宫|少年宫|艺术宫|电视台|协会|政府|[初高]中|集团|银行|[大中小]学|院|厂|店|段|场|社|室|部|厅|局|处|所|队|公司|监狱|监测站|血站|检查站|工作站)$",
+                "regstr": ".{2,100}(委员会|管委会|联合会|联合体|医院|卫计委|机关|社区|中心|中心校|分校|办公室|学校|幼儿园|动物园|管理站|馆|基地|青年宫|少年宫|艺术宫|电视台|协会|政府|[初高]中|集团|银行|[大中小]学|院|厂|店|段|场|社|室|部|厅|局|处|所|队|公司|监狱|监测站|血站|检查站|工作站|供应站)$",
                 "score": 10
             }
         ],
         "negativewords": [
             {
                 "describe": "包含负分",
-                "regstr": "(标人|附件|委托|认证|代理|咨询|顾问|管理顾问|招标失败|交易中心|不足|公告|变更|招标|废标|废止|流标|中标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注:|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|:|;|;|?|¥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
+                "regstr": "(标人|附件|委托|认证|代理|咨询|管理顾问|招标失败|不足|公告|变更|废标|废止|流标|中标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注:|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|:|;|;|?|¥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
                 "score": -15
             },
             {
                 "describe": "包含负分",
-                "regstr": "(代表)",
-                "score": -10
+                "regstr": "(代表|招标|交易中心|顾问)",
+                "score": -5
             },
             {
                 "describe": "包含负分不再展示",
@@ -194,6 +194,14 @@
                 "describe": "[gt,lte,score]",
                 "range": [
                     3,
+                    4,
+                    -1
+                ]
+            },
+            {
+                "describe": "[gt,lte,score]",
+                "range": [
+                    4,
                     25,
                     3
                 ]

+ 1 - 1
src/res/formattext.json

@@ -77,7 +77,7 @@
             "desc": ""
 		},
 		{
-			"reg": "([^((,,。、.;;::\r\n]{0,8})(联系人|地址)[::]([^\\s\u3000\u2003\u00a0,,]+?)(联系)?(电话(/传真)?|手机|传真|邮编)[::](.+)",
+			"reg": "([^((,,。、.;;::\r\n公司]{0,8})(联系人|地址)[::]([^\\s\u3000\u2003\u00a0,,]+?)(联系)?(电话(/传真)?|手机|传真|邮编)[::](.+)",
             "separator": "\n${1}${2}:${3}\n${1}${5}:${7}",
             "desc": ""
 		},

+ 1 - 1
src/web/templates/admin/result_list.html

@@ -154,7 +154,7 @@ $(function () {
 				}
 				return "<a style='color:#333' target='_blank' title='"+val+"'>"+tmpval+"</a>"
 			}},
-			{ "data": "_id",render:function(val,a,row){
+			{ "data": "_id","width":"14%",render:function(val,a,row){
 				rowdata[val]= row;
 				var url = row["url"];
 				return  "<div class='operate'><a class='btn btn-primary' onclick='trackShow(\""+val+"\")'>详细</a>&nbsp&nbsp&nbsp"+

BIN
standardata/src/sss


+ 19 - 19
standardata/src/standaragency.go

@@ -19,11 +19,11 @@ func agencyStandarData(db string, query map[string]interface{}) {
 	defer qu.Catch()
 	sess := MongoFrom.GetMgoConn()
 	defer MongoFrom.Close()
-	it := sess.DB(db).C(extractcoll).Find(query).Select(bson.M{"repeat": 1,"agency": 1, "agencytel": 1, "agencyperson": 1, "topscopeclass": 1,
+	it := sess.DB(db).C(extractcoll).Find(query).Select(bson.M{"repeat": 1, "agency": 1, "agencytel": 1, "agencyperson": 1, "topscopeclass": 1,
 		"agencyaddr": 1}).Sort("_id").Iter()
 	index := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
-		if qu.IntAll(tmp["repeat"]) > 0  { //重复数据跳过
+		if qu.IntAll(tmp["repeat"]) > 0 { //重复数据跳过
 			continue
 		}
 		agency := qu.ObjToString(tmp["agency"])
@@ -36,7 +36,7 @@ func agencyStandarData(db string, query map[string]interface{}) {
 		ps := []map[string]interface{}{}
 		agencyperson := qu.ObjToString(tmp["agencyperson"])
 		agencytel := qu.ObjToString(tmp["agencytel"])
-		if entid == "" {//redis 未存
+		if entid == "" { //redis 未存
 			savetoerr := true
 			if agencytel != "" {
 				v := map[string]interface{}{
@@ -65,14 +65,14 @@ func agencyStandarData(db string, query map[string]interface{}) {
 			}
 		} else {
 			if agencytel != "" {
-				is_exist:=false //电话是否存在
-				for _,v := range ps{
-					if v["phone"]==agencytel{
+				is_exist := false //电话是否存在
+				for _, v := range ps {
+					if v["phone"] == agencytel {
 						is_exist = true
-						if agencyperson!=""&&v["contact_person"]!=agencyperson {
-							v["contact_person"]=agencyperson
+						if agencyperson != "" && v["contact_person"] != agencyperson {
+							v["contact_person"] = agencyperson
 							v["infoid"] = infoid
-							bs, _ := json.Marshal(ps)//替换数据,更新
+							bs, _ := json.Marshal(ps) //替换数据,更新
 							redis.PutRedis("agency", agencybd, agency, bs, -1)
 						}
 						continue
@@ -87,7 +87,7 @@ func agencyStandarData(db string, query map[string]interface{}) {
 					}
 					MongoTo.UpdateById(agencyent, entid,
 						map[string]interface{}{
-							"$set":  v,
+							"$set":  map[string]interface{}{"updatetime": time.Now().Unix()},
 							"$push": map[string]interface{}{"contact": v},
 						},
 					)
@@ -108,7 +108,7 @@ func historyagency(db, fromcoll string) {
 	log.Println("history  start")
 	sess := MongoFrom.GetMgoConn()
 	defer MongoFrom.Close()
-	it := sess.DB(db).C(fromcoll).Find(map[string]interface{}{}).Select(bson.M{"repeat": 1,"agency": 1, "agencytel": 1, "agencyperson": 1, "topscopeclass": 1,
+	it := sess.DB(db).C(fromcoll).Find(map[string]interface{}{}).Select(bson.M{"repeat": 1, "agency": 1, "agencytel": 1, "agencyperson": 1, "topscopeclass": 1,
 		"agencyaddr": 1}).Sort("_id").Iter()
 	index := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
@@ -127,20 +127,20 @@ func historyagency(db, fromcoll string) {
 				agencyperson := qu.ObjToString(tmp["agencyperson"])
 				agencytel := qu.ObjToString(tmp["agencytel"])
 				b, _ := redis.ExistRedis("agency", agencybd, agency)
-				if b {//redis 存在
-					if  agencytel != "" {
+				if b { //redis 存在
+					if agencytel != "" {
 						strs, _ := redis.GetRedisStr("agency", agencybd, agency)
 						ps := []map[string]interface{}{}
 						err := json.Unmarshal([]byte(strs), &ps)
 						if err == nil {
-							is_exist:=false //电话是否存在
-							for _,v := range ps{
-								if v["phone"]==agencytel{
+							is_exist := false //电话是否存在
+							for _, v := range ps {
+								if v["phone"] == agencytel {
 									is_exist = true
-									if agencyperson!=""&&v["contact_person"]!=agencyperson {
-										v["contact_person"]=agencyperson
+									if agencyperson != "" && v["contact_person"] != agencyperson {
+										v["contact_person"] = agencyperson
 										v["infoid"] = _id
-										bs, _ := json.Marshal(ps)//替换数据,更新
+										bs, _ := json.Marshal(ps) //替换数据,更新
 										redis.PutRedis("agency", agencybd, agency, bs, -1)
 									}
 									continue

+ 1 - 0
standardata/src/standarbuyer.go

@@ -232,5 +232,6 @@ func buyerMegerBuyerclass(id string, ps map[string]interface{}) map[string]inter
 		newbuyerclass = append(newbuyerclass, k)
 	}
 	data["buyerclass"] = newbuyerclass
+	data["updatetime"] = time.Now().Unix()
 	return data
 }

+ 1 - 0
standardata/src/standarwinner.go

@@ -253,6 +253,7 @@ func winMegerIndustry(id string, ps map[string]interface{}) map[string]interface
 		newindustry = append(newindustry, k)
 	}
 	data["industry"] = newindustry
+	data["updatetime"] = time.Now().Unix()
 	return data
 }
 

+ 1 - 0
udpcreateindex/src/bidingpurchasing.go

@@ -23,6 +23,7 @@ func biddingPurchaingTask(q map[string]interface{}) {
 	itype, _ := bidding["type"].(string)
 	//
 	session := mgo.GetMgoConn(86400)
+	defer mgo.DestoryMongoConn(session)
 	count, _ := session.DB(db).C(c).Find(&q).Count()
 	log.Println("biddingPurchaingTask:	", db, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)