修改-抽取-判重

data_monitoring/listen_data/src/config.json

@@ -22,10 +22,10 @@
     "from": "zhengkun@topnet.net.cn",
     "to": "zhengkun@topnet.net.cn",
     "cc": "zhengkun@topnet.net.cn",
-    "smtpHost": "smtp.qq.com",
+    "smtpHost": "smtp.exmail.qq.com",
     "smtpPort": "465",
-    "user":     "920032221@qq.com",
-    "pwd":      "lktqxssmdkebbcbj"
+    "user":     "zhengkun@topnet.net.cn",
+    "pwd":      "Zheng123456"
   },
   "xlsx_name" : "site_data.xlsx"
 }

data_monitoring/listen_data/src/main.go

@@ -18,7 +18,7 @@ var (
 	save_mgo,site_mgo,python_mgo        	*MongodbSim            //mongodb操作对象
 	save_c_name,site_c_name,python_c_name,xlsx_name,site_unique_name	string
 	prepareData								[]map[string]interface{}
-	startRun	string
+	startRun,startSend								string
 )
 
 func initMgo()  {
@@ -68,7 +68,9 @@ func init() {
 	return
 	//加载配置文件
 	qu.ReadConfig(&sysconfig)
-	flag.StringVar(&startRun, "s", "", "是否启动") //增量
+	flag.StringVar(&startRun, "sr", "", "是否启动") //增量
+	flag.StringVar(&startSend, "ss", "", "是否发送") //增量
+
 	flag.Parse()
 	initMgo()
 }
@@ -76,50 +78,38 @@ func init() {
 
 func main()  {
 
-	save_mgo = &MongodbSim{
-		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",
-		DbName:      "qfw",
-		Size:        10,
-		UserName: "zhengkun",
-		Password: "zk@123123",
-	}
-	save_mgo.InitPool()
-
 	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.4.85:27080",
+	//	MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
 	//	DbName:      "qfw",
 	//	Size:        10,
-	//	UserName: "",
-	//	Password: "",
+	//	UserName: "zhengkun",
+	//	Password: "zk@123123",
 	//}
 	//save_mgo.InitPool()
 
 	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "127.0.0.1:27017",
-	//	DbName:      "zhengkun",
+	//	MongodbAddr: "172.17.4.85:27080",
+	//	DbName:      "qfw",
 	//	Size:        10,
 	//	UserName: "",
 	//	Password: "",
 	//}
 	//save_mgo.InitPool()
 
-	//reg := regexp.MustCompile(`^.{2}[大|小|中]学$`)
-	//bool := reg.MatchString("中山中学")
-	//log.Println(bool)
-	//pattern := `(中标金额[:：])([0-9.]*)`
-	//text := `中标金额:1111.00`
-	//reg := regexp.MustCompile(pattern)
-	//apos := reg.FindAllStringSubmatchIndex(text, -1)
-	//log.Println(apos)
-	//for i, _ := range apos {
-	//	pos := apos[i]
-	//	log.Println(text[pos[0]:pos[1]])
-	//	log.Println(text[pos[2]:pos[3]])
-	//	log.Println(text[pos[4]:pos[5]])
-	//}
-	decodeJyUrl()
+	save_mgo = &MongodbSim{
+		MongodbAddr: "127.0.0.1:27017",
+		DbName:      "zhengkun",
+		Size:        10,
+		UserName: "",
+		Password: "",
+	}
+	save_mgo.InitPool()
+
+	exportSpecifiedTimeData()
+
 	return
 
+
 	c := cron.New()
 	c.AddFunc("0 50 8 ? * *", func() { dealWithSiteData() })
 	c.Start()
@@ -311,7 +301,7 @@ func dealWithSiteData()  {
 	time.Sleep(5*time.Second)
 
 	//发送邮件
-	if startRun!="" {
+	if startRun!="" && startSend==""  {
 		log.Println("调试过程...不发邮件")
 	}else {
 		sendErrMailSmtp("主要站点最近五个工作日相关统计","附件")

src/jy/clear/tonumber.go

@@ -58,7 +58,7 @@ func ObjToInt(data []interface{}, spidercode ...string) []interface{} {
 	ch_num := map[string]string{
 		"一": "1", "二": "2", "三": "3", "四": "4", "五": "5", "六": "6", "七": "7", "八": "8", "九": "9", "十": "10",
 		"壹": "1", "贰": "2", "叁": "3", "肆": "4", "伍": "5", "陆": "6", "柒": "7", "捌": "8", "玖": "9", "拾": "10",
-		"两": "2",
+		"两": "2", "俩": "2",
 	}
 	tmp_value := fmt.Sprint(data[0])
 	for k,v:=range ch_num {
@@ -187,6 +187,7 @@ var moneyUnitRegBool = regexp.MustCompile(`(中标金额|成交金额|合同金
 //数字金额转换
 func numMoney(data []interface{}) ([]interface{}, bool) {
 	tmp := fmt.Sprintf("%f",data[0])
+	tmp = strings.ReplaceAll(tmp,"（不含税）","")
 	//费率转换% ‰
 	flv := float64(1)
 	if strings.HasSuffix(tmp, "%") {
@@ -420,6 +421,13 @@ func ClearMaxAmount(data []interface{}, spidercode ...string) []interface{} {
 		}
 	}
 	if value >= 50000000000 {
+		if len(spidercode)>0 {
+			code := util.ObjToString(spidercode[0])
+			if code == "xz_xzzzqjzscjgycxxxpt_zbtzs"||
+				code == "js_jsszbtbw_zbhxrgs"{
+				return data
+			}
+		}
 		data[0] = float64(0)
 		//data[1] = false
 		data[1] = true

src/jy/extract/extract.go

@@ -672,6 +672,10 @@ func (e *ExtractTask) ExtractDetail(j *ju.Job, isSite bool, codeSite string) {
 				if !ju.Logic(vc.LuaLogic, tmp) {
 					continue
 				}
+
+				if vc.Field =="winner" {
+					log.Debug("调试抽取字段")
+				}
 				////抽取-前置规则
 				//for _, v := range vc.RulePres {
 				//	tmp = ExtRegPre(tmp, j, v, e.TaskInfo)
@@ -728,9 +732,6 @@ func (e *ExtractTask) ExtractDetail(j *ju.Job, isSite bool, codeSite string) {
 		}
 		//函数清理
 		for key, val := range j.Result {
-			if key =="budget" {
-				//log.Debug(111)
-			}
 			for i, v := range val {
 				if v.Field == "projectname" && v.Type == "table" {
 					break
@@ -2282,12 +2283,33 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 func checkFields(tmp map[string]interface{}) map[string]interface{} {
 	delete(tmp, "contenthtml")
 	delete(tmp, "detail")
-	//delete(tmp, "toptype")
-	//delete(tmp, "subtype")
+	tmp["repeat"] = 0
+
+	//指定爬虫-金额处理-预算-中标金额异常
+	if qu.ObjToString(tmp["spidercode"])=="xz_xzzzqjzscjgycxxxpt_zbtzs" {
+		if budget, ok := tmp["budget"].(float64); ok && budget>0 && budget < 1000000{
+			tmp["budget"] = budget*10000.0
+		}
+		if bidamount, ok := tmp["bidamount"].(float64); ok && bidamount>0 && bidamount > 1000000000{
+			tmp["bidamount"] = bidamount/10000.0
+		}
+	}
+	if qu.ObjToString(tmp["spidercode"])=="js_jsszbtbw_zbhxrgs" {
+		if bidamount, ok := tmp["bidamount"].(float64); ok && bidamount>0 && bidamount > 1000000000{
+			tmp["bidamount"] = bidamount/10000.0
+		}
+	}
+
+
+
+
+
+
 	if _, ok := tmp["bidamount"].(string); ok {
 		delete(tmp, "bidamount")
 	} else if fb, ok := tmp["bidamount"].(float64); ok && fb > 0 && qu.Float64All(tmp["budget"]) > 0 && (fb/5 > qu.Float64All(tmp["budget"]) || qu.Float64All(tmp["budget"])/1000 > fb) {
 		delete(tmp, "bidamount")
+
 	}
 	if _, ok := tmp["budget"].(string); ok {
 		delete(tmp, "budget")
@@ -2320,23 +2342,28 @@ func checkFields(tmp map[string]interface{}) map[string]interface{} {
 		}
 	}
 
-	//工期单位-清理
-	//if tmp["project_timeunit"] == "年" && tmp["project_duration"] == nil {
-	//	delete(tmp, "project_timeunit")
-	//}
+	//项目周期-有效值
+	projectperiod := qu.ObjToString(tmp["projectperiod"])
+	if projectperiod !="" {
+		//项目周期包含日期，数字及日期单位可保留，其余可清洗
+		isNeedValueReg := regexp.MustCompile(`([0-9俩两一二三四五六七八九年月日天周]|合同)`)
+		if !isNeedValueReg.MatchString(projectperiod) {
+			delete(tmp, "projectperiod")
+		}
+	}
+
 	//工期单位是否有效-清理
 	if project_timeunit, ok := tmp["project_timeunit"].(string); ok {
 		dateReg := regexp.MustCompile(`[年|月|日|天|周]`)
-		if !dateReg.MatchString(project_timeunit) {
+		if !dateReg.MatchString(project_timeunit) || utf8.RuneCountInString(project_timeunit)>4 {
+			delete(tmp, "project_timeunit")
+		}
+		//年-0 >5 删除
+		if project_timeunit == "年" && (qu.Int64All(tmp["project_duration"])==0 || qu.Int64All(tmp["project_duration"])>5 ){
 			delete(tmp, "project_timeunit")
 		}
 	}
-	//if tmp["project_timeunit"] == "年" && tmp["project_duration"] == nil {
-	//	delete(tmp, "project_timeunit")
-	//}
 
-
-	tmp["repeat"] = 0
 	if tmp["winner"] != nil && tmp["s_winner"] != nil {
 		strwin := qu.ObjToString(tmp["winner"])
 		strwin_s := qu.ObjToString(tmp["s_winner"])
@@ -2349,19 +2376,15 @@ func checkFields(tmp map[string]interface{}) map[string]interface{} {
 		if bg >= 50000000000 {
 			tmp["budget_max_err"] = bg
 			delete(tmp, "budget")
-		} /*else if bg > 0 && bg < 1000 {
-			tmp["budget_min_err"] = bg
-			delete(tmp, "budget")
-		}*/
+		}
 	}
 	if bg, ok := tmp["bidamount"].(float64); ok && bg >= 50000000000 {
-		if bg >= 50000000000 {
+		code := qu.ObjToString(tmp["spidercode"])
+		if bg >= 50000000000 && code != "xz_xzzzqjzscjgycxxxpt_zbtzs" &&
+			code != "js_jsszbtbw_zbhxrgs"{
 			tmp["bidamount_max_err"] = bg
 			delete(tmp, "bidamount")
-		} /*else if bg > 0 && bg < 1000 {
-			tmp["bidamount_min_err"] = bg
-			delete(tmp, "bidamount")
-		}*/
+		}
 	}
 	//投标方式-
 	bidway := qu.IntAll(tmp["bidway"])
@@ -2389,7 +2412,7 @@ func checkFields(tmp map[string]interface{}) map[string]interface{} {
 
 	return tmp
 }
-
+//处理折扣系数
 func dealWithDiscountBid(tmp map[string]interface{}) float64 {
 	biddiscount := qu.Float64All(tmp["biddiscount"])
 	biddiscount_up := qu.Float64All(tmp["biddiscount_up"])

src/jy/extract/newextractcity.go

@@ -89,13 +89,13 @@ func (e *ExtractTask) NewExtractCity(j *ju.Job, resulttmp *map[string]interface{
 	}
 	//7.buyeraddr buyer title projectname抽取
 	e.NewGetCityByOthers(j, sm, &pscore, &cscore, &dscore)
-	qu.Debug("全称打分后结果---", j.FullAreaScore, j.FullCityScore, j.FullDistrictScore)
-	qu.Debug("简称打分后结果---", j.SimAreaScore, j.SimCityScore, j.SimDistrictScore)
+	//qu.Debug("全称打分后结果---", j.FullAreaScore, j.FullCityScore, j.FullDistrictScore)
+	//qu.Debug("简称打分后结果---", j.SimAreaScore, j.SimCityScore, j.SimDistrictScore)
 	//全称简称得分合并
 	MergeFullSimScore(j) //合并buyer buyeraddr title projectname全称简称
-	qu.Debug("全称简称合并后---", j.FullAreaScore, j.FullCityScore, j.FullDistrictScore)
+	//qu.Debug("全称简称合并后---", j.FullAreaScore, j.FullCityScore, j.FullDistrictScore)
 	//合并区简称得分
-	qu.Debug("pcd=====", pscore, cscore, dscore)
+	//qu.Debug("pcd=====", pscore, cscore, dscore)
 	MergeScores(j, &pscore, &cscore, &dscore) //合并区简称匹配的pcd
 	//qu.Debug("合并区简称打分后结果---", j.FullAreaScore, j.FullCityScore, j.FullDistrictScore)
 
@@ -107,10 +107,10 @@ func (e *ExtractTask) NewExtractCity(j *ju.Job, resulttmp *map[string]interface{
 	if len(j.FullAreaScore) > 0 && len(j.FullCityScore) > 0 { //以上抽取有省有市再从detail中抽取进行判断
 		e.NewGetCityByDetail(j)
 	}
-	qu.Debug("detail打分后全称---", j.FullAreaScore, j.FullCityScore, j.FullDistrictScore)
-	qu.Debug("detail打分后简称---", j.SimAreaScore, j.SimCityScore, j.SimDistrictScore)
+	//qu.Debug("detail打分后全称---", j.FullAreaScore, j.FullCityScore, j.FullDistrictScore)
+	//qu.Debug("detail打分后简称---", j.SimAreaScore, j.SimCityScore, j.SimDistrictScore)
 	MergeFullSimScore(j) //合并detail的全简称
-	qu.Debug("detail合并后---", j.FullAreaScore, j.FullCityScore, j.FullDistrictScore)
+	//qu.Debug("detail合并后---", j.FullAreaScore, j.FullCityScore, j.FullDistrictScore)
 
 	finishP := HighestScoreArr(j.FullAreaScore) //获取最高分的省
 	e.RemoveCD(finishP, j)                      //将city中所属干扰项省的city去除,同时去除district.5d2bd4aba5cb26b9b769d18e

src/jy/extract/score.go

@@ -191,7 +191,12 @@ func ScoreFields(j *ju.Job, ftag map[string][]*Tag) map[string][]*ju.ExtField {
 				if tmpsvalue.ExtFrom == "title" { //标题打分初始化
 					titlescore = fieldscore["title"]
 				}
-				typescore = fieldscore[tmpsvalue.Type]
+				//抽取类型打分-针对处理-
+				if field == "bidamount" && j.SpiderCode=="a_hcsccz_cjgg" && tmpsvalue.Type=="table" {
+					//不打分+3分
+				}else {
+					typescore = fieldscore[tmpsvalue.Type]
+				}
 			} else { //通用抽取属性打分配置
 				if tmpsvalue.ExtFrom == "title" { //标题打分初始化
 					titlescore = CommonScore["title"]
@@ -256,9 +261,9 @@ func ScoreFields(j *ju.Job, ftag map[string][]*Tag) map[string][]*ju.ExtField {
 								gt := qu.IntAll(ranges[0])
 								lte := qu.IntAll(ranges[1])
 								//针对指定 buyer -长度0-4 不打分
-								if field=="buyer" {
+								if field=="buyer" || field=="winner" {
 									buyerValue := fmt.Sprint(tmpsvalue.Value)
-									reg := regexp.MustCompile(`^.{2}[大|小|中]学$`)
+									reg := regexp.MustCompile(`^.{2}([大|小|中|学][学|院]|公司)$`)
 									if reg.MatchString(buyerValue) && gt==0 && lte==4 {
 										continue
 									}
@@ -282,10 +287,10 @@ func ScoreFields(j *ju.Job, ftag map[string][]*Tag) map[string][]*ju.ExtField {
 						if p, ok := position.(map[string]interface{}); ok {
 							qu.Try(func() {
 								if p["regexp"] != nil {
-									if field=="buyer" {
+									if field=="buyer"|| field=="winner" {
 										//针对指定 buyer -个别  不打分
 										buyerValue := fmt.Sprint(tmpsvalue.Value)
-										reg := regexp.MustCompile(`^.{2}[大|小|中]学$`)
+										reg := regexp.MustCompile(`^.{2}([大|小|中|学][学|院]|公司)$`)
 										if reg.MatchString(buyerValue) && qu.ObjToString(p["describe"])=="黑名单"{
 											return
 										}

src/jy/pretreated/analystep.go

@@ -113,7 +113,7 @@ func AnalyStart(job *util.Job, isSite bool, codeSite string) {
 		bl.ColonKV = GetKVAll(bl.Text, "", nil, 1, isSite, codeSite)
 		//处理空格
 		bl.SpaceKV = SspacekvEntity.Entrance(bl.Text, "", nil, isSite, codeSite)
-		//新加 未分块table中未能解析到中标候选人，从正文中解析
+		//新加 未分块table中未能解析到   中标候选人，从正文中解析
 		if job.Winnerorder == nil || len(job.Winnerorder) == 0 {
 			bl.Winnerorder = winnerOrderEntity.Find(bl.Text, true, 1, isSite, codeSite)
 			job.Winnerorder = bl.Winnerorder

src/jy/pretreated/analytable.go

@@ -17,7 +17,7 @@ import (
 **/
 var (
 	//key 的日期单位
-	dateReg *regexp.Regexp = regexp.MustCompile(`[年|月|日|天|周]`)
+	dateReg *regexp.Regexp = regexp.MustCompile(`[年|月|日|天]`)
 	//清理品目中数字
 	numclear = regexp.MustCompile("^[\\d一二三四五六七八九十.]+")
 	num1     = regexp.MustCompile("(\\d)")
@@ -31,7 +31,7 @@ var (
 	//判断key是金额，对万元的处理
 	moneyreg = regexp.MustCompile("(预算|费|价|额|规模|投资)")
 	//key不需要清理-例如折扣 费率
-	no_clear_key_reg = regexp.MustCompile(`[(（](费率|年|月|日|天)[)）]`)
+	no_clear_key_reg = regexp.MustCompile(`[(（](费率|年|月|日|天|日历天|历天)[)）]`)
 	//根据表格的内容判断是不是表头，如果含有金额则不是表头
 	MoneyReg = regexp.MustCompile("^[\\s 　：:0-9.万元()（）人民币￥$]+$")
 	GSReg    = regexp.MustCompile(".*公司.*")
@@ -122,7 +122,7 @@ var (
 	FilterSerial                = regexp.MustCompile(".+[、．.:：，]")
 	underline                   = regexp.MustCompile("_+$")
 	iswinnertabletag            = regexp.MustCompile("(中标|候选人|成交|结果|磋商情况)")
-	nswinnertabletag            = regexp.MustCompile("评得分估|标的|班子成员|人员")
+	nswinnertabletag            = regexp.MustCompile("评得分估|标的信息|班子成员|人员")
 	jsonReg                     = regexp.MustCompile(`\{.+:[^}]*\} `) //  \{".*\":\".+\"}
 	regHz                       = regexp.MustCompile("[\u4e00-\u9fa5]")
 	winnerOrderAndBidResult     = regexp.MustCompile("((中标)?候选人|(中标|评标)结果)")
@@ -167,6 +167,11 @@ func CommonDataAnaly(k, tabletag, tabledesc string, v interface{}, isSite bool,
 	}
 	//u.Debug(2, k)
 	//取标准key
+	if tabletag == "中标情况" {
+		if k1=="价格" {
+			k1="中标金额"
+		}
+	}
 	res := u.GetTags(k1, isSite, codeSite)
 	if len(res) == 0 && k1 != k {
 		res = u.GetTags(k, isSite, codeSite)
@@ -244,7 +249,7 @@ func (table *Table) KVFilter(isSite bool, codeSite string) {
 		//遍历所有key sort.kv
 		//表格描述处理，对成交结果的处理
 		if k=="服务期（天）" {
-			fmt.Println("标准化key")
+			//fmt.Println("标准化key")
 		}
 		if glRex.MatchString(k) {
 			table.Desc += "成交结果,"
@@ -265,7 +270,7 @@ func (table *Table) KVFilter(isSite bool, codeSite string) {
 			as.AddKey(k, v)
 		}
 	}
-	//处理值是数组的kv放入标准化kv中 standKV //处理table.SortKV.value为数组的情况
+	//处理值是数组的kv   放入标准化kv中 standKV //处理table.SortKV.value为数组的情况
 	table.sortKVArr(as, isSite, codeSite)
 	//
 	if len(table.WinnerOrder) > 0 || !table.BPackage {
@@ -492,6 +497,7 @@ func (table *Table) sortKVArr(as *SortMap, isSite bool, codeSite string) {
 						//} else if len(tmpEntname) > 0 {
 						//fmt.Println("table winnerorder only has entname", tmpEntname)
 						//}
+						qutil.Debug("len-smap_v--", len(smap_v))
 						if len(smap_v) > 2 { //只有排序信息 sort和sortstr
 							newSmap = append(newSmap, smap_v)
 						}
@@ -1509,6 +1515,9 @@ func (table *Table) FindKV(isSite bool, codeSite string) {
 		nextdirect, nextvdirect := 0, 0
 		//开始抽取
 		for _, tr := range table.TRs {
+			//if  kkk==18 || kkk==21 {
+			//	log.Println("调试指定tr")
+			//}
 			bcon = trSingleColumn(tr, bcon, table) //tr单列,是否丢弃内容
 			if bcon {
 				continue
@@ -1928,28 +1937,42 @@ func (table *Table) FindTdVal(td *TD, direct, vdirect int) (b bool) {
 					varrpos = len(vals) - 1
 				}
 			} else if vals, ok := val.(string); ok && vals != "" && td.Val != "" {
-				tmapval := strings.TrimSpace(cleardwReg.ReplaceAllString(vals, ""))
+				tmapval := strings.TrimSpace(cleardwReg.ReplaceAllString(vals, ""))//已存在的kv
 				tmapvaltd := strings.TrimSpace(cleardwReg.ReplaceAllString(td.Val, ""))
 				if bvalfind {
-					if tmapvaltd == "" {
-						val = td.Val //vals + "__" + td.Val
-					} else {
-						val = tmapvaltd
-					}
-				} else {
-					tval := []string{}
-					if tmapval == "" {
-						tval = append(tval, vals)
-					} else {
-						tval = append(tval, tmapval)
+					//if tmapvaltd == "" {
+					//	val = td.Val //vals + "__" + td.Val
+					//} else {
+					//	val = tmapvaltd
+					//}
+					if key=="中标单位" {
+						//不能覆盖---
+					}else {
+						if tmapvaltd == "" {
+							val = td.Val //vals + "__" + td.Val
+						} else {
+							val = tmapvaltd
+						}
 					}
-					if tmapvaltd == "" {
-						tval = append(tval, td.Val)
-					} else {
-						tval = append(tval, tmapvaltd)
+				} else{
+					if key=="中标单位" {
+						//新增不能数组
+					}else {
+						tval := []string{}
+						if tmapval == "" {
+							tval = append(tval, vals)
+						} else {
+							tval = append(tval, tmapval)
+						}
+						if tmapvaltd == "" {
+							tval = append(tval, td.Val)
+						} else {
+							tval = append(tval, tmapvaltd)
+						}
+						val = tval
+						varrpos = 1
 					}
-					val = tval
-					varrpos = 1
+
 				}
 			}
 			barr = true

src/jy/pretreated/colonkv.go

@@ -817,12 +817,24 @@ func MergeKvTags(kvTags_1, kvTags_2 map[string][]*Tag) {
 	for k, v := range kvTags_2 {
 		for _, vv := range v {
 			value_vv := strings.TrimSpace(vv.Value)
+			if k=="项目周期"||k=="工期单位"||k=="工期时长" {
+				dateStr := dateReg.FindString(vv.Key)
+				if dateStr !="" && !strings.Contains(value_vv,dateStr) {
+					vv.Value = value_vv+dateStr
+				}
+			}
 			if value_vv == "" || vv.Key == vv.Value {
 				continue
 			}
 			isExists := false
 			for _, vvv := range kvTags_1[k] {
 				value_vvv := strings.TrimSpace(vvv.Value)
+				if k=="项目周期"||k=="工期单位"||k=="工期时长" {
+					dateStr := dateReg.FindString(vvv.Key)
+					if dateStr !="" && !strings.Contains(value_vvv,dateStr) {
+						vvv.Value = value_vvv+dateStr
+					}
+				}
 				if (value_vvv == value_vv || TimeHM.ReplaceAllString(value_vvv, ReplTimeHM) == value_vv || value_vvv == TimeHM.ReplaceAllString(value_vv, ReplTimeHM)) && vvv.Weight == vv.Weight {
 					isExists = true
 					break

src/jy/pretreated/division.go

@@ -627,7 +627,7 @@ func findWinnerBugetBidmountByKv(v *util.BlockPackage, blockPackage map[string]*
 						blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
 					}
 				}
-			} else if kc == "中标金额" && v.Bidamount <= 0 {
+			} else if (kc == "中标金额"||kc=="各包中标/成交候选供应商及报价") && v.Bidamount <= 0 {
 				moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
 				if len(moneys) > 0 {
 					if vf, ok := moneys[0].(float64); ok {
@@ -638,7 +638,7 @@ func findWinnerBugetBidmountByKv(v *util.BlockPackage, blockPackage map[string]*
 						blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
 					}
 				}
-			} else if kc == "中标单位" && v.Winner == "" {
+			} else if (kc == "中标单位"||kc=="各包中标/成交候选供应商及报价") && v.Winner == "" {
 				blockPackage[k].Winner = cv[0].Value
 			}
 		}

src/jy/pretreated/multipackage.go

@@ -21,7 +21,7 @@ var (
 	//替换容易混淆的词
 	PreCon1 = regexp.MustCompile("(施工)*([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+标段)")
 	//提取分包标识
-	MultiReg = regexp.MustCompile("(([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-])+(包|标段|分标))[:：]?|(?:^|\\n)([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+(包|标段))|([第]?([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+)#?((子|合同|分|施工|监理)?(标段?|合同段|标包)))|(((子|分|合同|施工|监理|标包|标|包)(标|包段|项|组)?)[ 　　  ]*([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+))[:：]?|(子项目[0-9]+)")
+	MultiReg = regexp.MustCompile("(([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-])+(包|标段|分标))[:：]?|(?:^|\\n)([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+(包))|([第]?([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+)#?((子|合同|分|施工|监理)?(标段?|合同段|标包)))|(((子|分|合同|施工|监理|标包|标|包)(标|包段|项|组)?)[ 　　  ]*([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+))[:：]?|(子项目[0-9]+)|(包组)[:：\\s]+?([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+)")
 	PreCon4 = regexp.MustCompile("([一二三四五六七八九十]标段[:：￥0-9.]*(万元)?)[、]?")
 	Precon4dw = regexp.MustCompile("(万元|元)")
 	//匹配到的包格式分类统计
@@ -67,6 +67,7 @@ func CheckMultiPackage(con, title string) (content string, m map[string][]string
 	con = PreCheckMulti.ReplaceAllString(con, "")
 	con = PreCon.ReplaceAllString(con, "\n")
 	con = PreCon2.ReplaceAllString(con, "")
+	//修改 \nX标段
 	res := MultiReg.FindAllStringSubmatch(con, -1)
 	if len(res) > 0 { //5 6
 		mindex := map[string]int{}
@@ -89,6 +90,11 @@ func CheckMultiPackage(con, title string) (content string, m map[string][]string
 				k = v[6]
 				vindex = 8
 			}
+			if k == "" {
+				k = v[0]
+				vindex = 17
+			}
+
 			if k != "" && v[vindex] != ""  {
 				vindex += 1
 			}

src/jy/pretreated/winnerorder.go

@@ -28,17 +28,25 @@ var (
 	numberReg         = regexp.MustCompile("[一二三四五六七八九十0-9]+")
 	numberReg2        = regexp.MustCompile("[\\d一二三四五六七八九十.,，]+")
 	thisNumberReg     = regexp.MustCompile("第" + numberReg.String())
-	winnerReg0        = regexp.MustCompile("(中标候选人第\\d名)")
+	winnerReg0        = regexp.MustCompile("(中标候选人第\\d名|第[0-9一二三四五](中选|中标|成交)候选人|[第|弟][0-9一二三四五]中标人|中标人[1-9])")
 	winnerReg1        = regexp.MustCompile("(^|[^为])(【?(推荐)?第[一二三四五六七八九十1-9]+(合格|名|包|标段)?】?([候|侯]选)?(入围|备选|成交|中[标|选])人?([(（]成交[）)])?([候|侯]选|排序)?(人(单位)?|供[应货]商|单位|机构)(名称)?为?)($|[^，；;。,])")
-	winnerReg2        = regexp.MustCompile("(排名第[一二三四五六七八九十1-9]+|第[一二三四五六七八九十1-9]+(中标)?[候|侯]选人|中标候选人排名[:]\\d)")
+	winnerReg2        = regexp.MustCompile("(排名第[一二三四五六七八九十1-9]+|[第|弟][一二三四五六七八九十1-9]+(中标|中选)?[候|侯]选人|中标候选人排名[:]\\d)")
 	//winnerReg2     = regexp.MustCompile("(第[一二三四五六七八九十1-9]+(候|侯)选人)")
-	winnerReg3     = regexp.MustCompile("((中标候选人)?第[一二三四五六七八九十1-9]+名)")
+	winnerReg3     = regexp.MustCompile("((中标候选人)?第[一二三四五六七八九十1-9]+名)|()")
 	winnerReg4     = regexp.MustCompile("((确认|推荐|评审|排[名|序])[为:：]+|(由高到低排序前.名|公示下列内容|(确定|推荐)的?中[标|选]候选人|\n中[标|选]候选.{1,3}\\s*\n|\n(中[标|选]候选.{1,3}[:：\u3000\u2003\u00a0\\s]|成交候选供应商)|(排[名|序]|公[示|告]|具体|推荐|结果(公示)?|中[标|选]候选人.{0,2})如下|[一二三四五六七八九十\\d]+、(中[标|选]候选[^\n:：]{1,8}|.{0,8}(成交|结果)信息|成交[^\n:：]{2,8}))[为:：]?)")
 	winnerReg5     = regexp.MustCompile("([^，；;。,、\n投标人]+?)(为?)(第[一二三四五六七八九十1-9]+(成交|中标)?([候|侯]选(人|供应商|单位|机构)|名)|排名第[一二三四五六七八九十1-9]+)([，；;。,、]|\\s+\n)")
 	winnerReg6     = regexp.MustCompile("(^(排名)?第[一二三四五六七八九十1-9]+[名中标成交备选候人单位供应商]*)")
 	winnerReg7     = regexp.MustCompile("第[一二三四五六七八九十]{1}标段[:：]")
-	winnerReg8     = regexp.MustCompile("(第[一二三四五六七八九十]中标候选人)[:：]?\n（1）单位名称:(.*)\n（2）投标报价（含税）:(.*)")
-	winnerRegclear = regexp.MustCompile("(买方人员|经评审.*排名第[一二三四五六七八九十1-9]+)")
+	winnerReg8     = regexp.MustCompile("(第[一二三四五六七八九十]中选候选人)[:：\\s]+?[(（]1[)）][\\s]+?(单位名称)[:：]?(.*)[\\s]+?[(（]2[)）][\\s]+(参选报价|投标报价（含税）)[:：]?(.*)")
+	//winnerReg8     = regexp.MustCompile("(第[一二三四五六七八九十]中标候选人)[:：]?\n（1）单位名称:(.*)\n（2）投标报价（含税）:(.*)")
+    winnerReg9     = regexp.MustCompile("(第[一二三四五六七八九十][中选]?候选人|中标人[1-9])[:：\\s]+?([\u4E00-\u9FA5]{4,20})([0-9\\.\\s万元]+)")
+    winnerReg10    = regexp.MustCompile("(第[一二三四五六七八九十]中标人)[:：\\s]+?报价[￥]?([0-9\\.\\s万元]+)[;；]([\u4E00-\u9FA5]{4,20})")
+	winnerReg11     = regexp.MustCompile("([弟|第][一二三四五六七八九十]中[标|选]候选人)[:：\\s]+?(单位名称|投标人名称)[:：]?(.*)[\\s]+?(参选报价|投标报价[(（]含税[)）]|投标报价[(（]元[)）])[:：]?(.*)")
+	winnerReg12     = regexp.MustCompile("(中[标|选]候选人[弟|第][一二三四五六七八九十0-9]名)[:：\\s]+?(.*)[\\s，,]+?(投标报价)[:：]?([0-9\\.\\s万元]+)")
+	winnerReg13     = regexp.MustCompile("([弟|第][一二三四五六七八九十0-9])\n(成交候选人|成交供应商)\n(.*)\n([0-9\\.\\s万元]+)")
+
+
+winnerRegclear = regexp.MustCompile("(买方人员|经评审.*排名第[一二三四五六七八九十1-9]+)")
 	colonEndReg    = regexp.MustCompile("[:：]$")
 	toWarpReg      = regexp.MustCompile("[，。,;；]+")
 	findamountReg  = regexp.MustCompile("[，。,;；\u3000\u2003\u00a0\\s]+")
@@ -61,21 +69,27 @@ var (
 func (wo *WinnerOrderEntity) Find(text string, flag bool, from int, isSite bool, codeSite string) []map[string]interface{} {
 	if clearSpace2.MatchString(text){
 		text = TextAfterRemoveTable(text)
-	}
+	}//评得分估|标的|班子成员|人员
 	text = winnerRegclear.ReplaceAllString(text,"")
 	if nswinnertabletag.MatchString(text) {
 		return []map[string]interface{}{}
 	}
 	text = winnerReg5.ReplaceAllString(text, "\n$3：$1\n")
-	text = winnerReg8.ReplaceAllString(text,"\n${1}:${2}\n中标金额:${3}\n")
-	/*
-		"_id" : ObjectId("5c2c6f60a5cb26b9b7b62cd8")
+	text = winnerReg8.ReplaceAllString(text,"\n${1}:${3}\n中标金额:${5}\n")
+	text = winnerReg9.ReplaceAllString(text,"\n${1}:${2}\n中标金额:${3}\n")
+	text = winnerReg10.ReplaceAllString(text,"\n${1}:${3}\n中标金额:${2}\n")
+	text = winnerReg11.ReplaceAllString(text,"\n${1}:${3}\n中标金额:${5}\n")
+	text = winnerReg12.ReplaceAllString(text,"\n${1}:${2}\n中标金额:${4}\n")
+	text = winnerReg13.ReplaceAllString(text,"\n${1}${2}:${3}\n中标金额:${4}\n")
+
 
-		1 .第一中选候选人：
-		（1）	单位名称：成都维诺信科技有限公司
-		（2）	参选报价：522,00.00元
-		（3）	质量：符合比选文件规定的质量标准
 
+	/*
+	第一候选人：河南冠嘉建设工程有限公司41.9450万元 合格  90日历天  孔祥子
+
+	中标人名称 中标价格 中标份额（%）
+	中标人1 山东益通安装有限公司 0.97元 55%
+	中标人2 山东鸿华建筑安装工程有限公司 0.96元 45%
 	*/
 	text = clearSpace1.ReplaceAllString(text, "") //清理（1）	单位名称：成都维诺信科技有限公司-->单位名称：成都维诺信科技有限公司
 	if strings.TrimSpace(text) == "" {
@@ -93,6 +107,7 @@ func (wo *WinnerOrderEntity) Find(text string, flag bool, from int, isSite bool,
 	}
 	if len(winners) == 0 {
 		if flag {
+			//异常
 			winners = wo.findByReg(text, blocks, winnerReg3, from, isSite, codeSite)
 		} else {
 			indexs_4 := winnerReg4.Split(text, -1)
@@ -139,12 +154,18 @@ func (wo *WinnerOrderEntity) getText(text string, blocks []string, reg_2 *regexp
 		for _, v := range array {
 			var wrfg *WinnerFlag
 			if isWinnerReg1 {
+				if v[4]<0 || v[5]<0 {
+					continue
+				}
 				wrfg = &WinnerFlag{
 					index:     wo.toNumber(b_v[v[4]:v[5]], 0),
 					textStart: v[4],
 					textEnd:   v[5],
 				}
 			} else {
+				if v[2]<0 || v[3]<0 {
+					continue
+				}
 				wrfg = &WinnerFlag{
 					index:     wo.toNumber(b_v[v[2]:v[3]], 0),
 					textStart: v[2],

src/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	log "github.com/donnie4w/go-logger/logger"
 	_ "jy/admin"
 	_ "jy/admin/audit"
 	_ "jy/admin/distribution"
@@ -12,8 +13,6 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	qu "qfw/util"
-
-	log "github.com/donnie4w/go-logger/logger"
 	"qfw/util/elastic"
 	"qfw/util/redis"
 )
@@ -74,6 +73,8 @@ func main() {
 	go extract.Export()
 	go Router.Run(":" + qu.ObjToString(util.Config["port"]))
 	go log.Debug("启动..", qu.ObjToString(util.Config["port"]))
+
+
 	go func() {
 		http.ListenAndServe("localhost:10000", nil)
 	}()

udpdataclear/udpSensitiveWords/main.go

@@ -1,14 +1,18 @@
 package main
 
 import (
+	"log"
 	"sensitiveWords.udp/util"
 )
 
 func init() {
+	log.Println("what init")
+	return
 	util.InitC()
 }
 func main() {
-
+	log.Println("what main")
+	return
 	//util.TestData()
 	//return
 	go util.AddTaskSensitiveWordsData() //增量

udpdataclear/udpSensitiveWords/util/udpdata.go

@@ -241,7 +241,7 @@ func handleData(datas []string) string {
 	return rstr
 }
 
-//定时增量数据处理---冯
+//定时增量数据处理---
 func AddTaskSensitiveWordsData() {
 	defer func() {
 		if err := recover(); err != nil {

udpfilterdup/src/config.json

@@ -2,7 +2,7 @@
     "udpport": ":17859",
     "dupdays": 7,
     "mongodb": {
-        "addr": "192.168.3.207:27092",
+        "addr": "127.0.0.1:27017",
         "pool": 10,
         "db": "zhengkun",
         "extract": "repeat_test",

udpfilterdup/src/dataMethod.go

@@ -74,14 +74,15 @@ func againRepeat(v *Info, info *Info ,site bool) bool {
 	if v.projectcode != "" && info.projectcode != "" && v.projectcode != info.projectcode {
 		return true
 	}
-	if v.projectname != info.projectname && v.projectname != "" && info.projectname != ""{
-		return true
-	}
 	if v.title != info.title && v.title != "" && info.title != ""{
+		if v.projectname != info.projectname && v.projectname != "" && info.projectname != ""{
+			return true
+		}
+	}
+	if v.projectname != info.projectname && v.projectname != "" && info.projectname != ""{
 		return true
 	}
 
-
 	return false
 }
 
@@ -117,8 +118,8 @@ func againContainSpecialWord (v *Info, info *Info) bool {
 //提取标题-标段号处理
 func dealTitleSpecial(title1 string,title2 string) bool{
 
-	regular1 := "(包|标段|标包)[(（]?[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９][)）]?"
-	regular2 := "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９](包|标段|标包)"
+	regular1 := "(包|标段|标包)[(（]?[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９]+[)）]?"
+	regular2 := "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９]+(包|标段|标包)"
 	regx1_1,_ := regexp.Compile(regular1)
 	str1:=regx1_1.FindString(title1)
 	if str1!="" {
@@ -263,6 +264,9 @@ func isTheSameDay(i1 int64 ,i2 int64) bool {
 	if day1==day2 {
 		return true
 	}
+	//if math.Abs(float64(i1-i2)) <=86400.0 {
+	//	return true
+	//}
 	return false
 }
 
@@ -312,7 +316,9 @@ func buyerIsContinue(v *Info, info *Info) bool {
 		return true
 	}
 	if v.title != info.title && v.title != "" && info.title != ""{
-		return true
+		if v.projectname != info.projectname && v.projectname != "" && info.projectname != ""{
+			return true
+		}
 	}
 	if v.projectname != info.projectname && v.projectname != "" && info.projectname != ""{
 		return true

udpfilterdup/src/dataMethodHeavy.go

@@ -1,6 +1,8 @@
 package main
 
-import "strings"
+import (
+	"strings"
+)
 
 //判重方法1
 func quickHeavyMethodOne(v *Info, info *Info, reason string) (bool, string) {

udpfilterdup/src/datamap.go

@@ -251,6 +251,7 @@ func NewInfo(tmp map[string]interface{}) *Info {
 //判重方法
 func (d *datamap) check(info *Info) (b bool, source *Info, reasons string) {
 	reason := ""
+	isTestLog := false
 	keys := []string{}
 	d.lock.Lock()
 	for k, _ := range d.keys { //不同时间段
@@ -277,12 +278,15 @@ L:
 		if len(data) > 0 { //对比v   找到同类型，同省或全国的数据作对比
 			for _, v := range data {
 				reason = ""
+				isTestLog = false
 				if v.id == info.id { //正常重复
 					return false, v, ""
 				}
-
 				//buyer 优先级高，有值且不相等过滤
 				if info.buyer!=""&&v.buyer!=""&&info.buyer!=v.buyer {
+					if v.title != info.title && v.title != "" && info.title != "" {
+						isTestLog = true
+					}
 					if buyerIsContinue(v,info) {
 						continue
 					}
@@ -312,7 +316,7 @@ L:
 						reasons = reason
 						break L
 					}
-					//相同发布时间-标题无包含关系
+					//相同发布时间-标题无包含关系 - 项目名称不等
 					if isTheSameDay(info.publishtime,v.publishtime) &&
 						!(strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title)) {
 						continue
@@ -471,6 +475,9 @@ L:
 		d.lock.Unlock()
 	}
 
+	if isTestLog {
+		reasons = reasons+"-新修改"
+	}
 	return
 }
 

udpfilterdup/src/main.go

@@ -53,7 +53,6 @@ var (
 	updatelock 		sync.Mutex         				 //锁4
 	userName,passWord 	string						 //mongo -用户密码
 	taskList		[]map[string]interface{}		 //任务池
-	isRunningRepeat	bool
 )
 
 //udp通道
@@ -213,7 +212,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 			//插入任务-判断任务-是否存在
 			updatelock.Lock()
 			taskList = append(taskList,mapInfo)
-			log.Println("udp收到任务...数量：",len(taskList))
+			log.Println("udp收到任务...数量：",len(taskList),"具体任务：",taskList)
 			updatelock.Unlock()
 		}
 	case mu.OP_NOOP: //下个节点回应
@@ -283,16 +282,16 @@ func getRepeatTask()  {
 	for  {
 		if len(taskList)>0 {
 			updatelock.Lock()
-			log.Println("准备执行判重任务...")
+			//log.Println("准备执行判重任务...")
 			mapInfo := taskList[0]
 			if mapInfo != nil  {
 				taskRepeat(mapInfo) //判重方法
 			}
 			taskList = taskList[1:]
-			log.Println("当前任务池...",len(taskList),taskList)
+			log.Println("此段落结束当前任务池...",len(taskList),taskList)
 			updatelock.Unlock()
 		}else {
-			log.Println("无任务...睡眠15s")
+			//log.Println("无任务...睡眠15s")
 			time.Sleep(15 * time.Second)
 		}
 	}
@@ -324,7 +323,10 @@ func taskRepeat(mapInfo map[string]interface{}) {
 			},
 		}
 	}
+	//临时赋值
 	log.Println("开始数据判重~查询条件：",mgo.DbName, extract, q)
+
+
 	sess := mgo.GetMgoConn()
 	defer mgo.DestoryMongoConn(sess)
 	it := sess.DB(mgo.DbName).C(extract).Find(&q).Sort("publishtime").Iter()
@@ -367,16 +369,16 @@ func taskRepeat(mapInfo map[string]interface{}) {
 				//替换数据池-更新
 				DM.replacePoolData(source)
 
-				Update.updatePool <- []map[string]interface{}{//原始数据打标签
-					map[string]interface{}{
-						"_id": StringTOBsonId(source.id),
-					},
-					map[string]interface{}{
-						"$set": map[string]interface{}{
-							"repeat_ids": repeat_ids,
-						},
-					},
-				}
+				//Update.updatePool <- []map[string]interface{}{//原始数据打标签
+				//	map[string]interface{}{
+				//		"_id": StringTOBsonId(source.id),
+				//	},
+				//	map[string]interface{}{
+				//		"$set": map[string]interface{}{
+				//			"repeat_ids": repeat_ids,
+				//		},
+				//	},
+				//}
 				Update.updatePool <- []map[string]interface{}{//重复数据打标签
 					updateID,
 					map[string]interface{}{
@@ -400,27 +402,29 @@ func taskRepeat(mapInfo map[string]interface{}) {
 	//睡眠时间30s 目的是让数据池更新所有数据...
 	time.Sleep(15 * time.Second)
 	//更新Ocr的标记
-	updateOcrFileData(mapInfo["lteid"].(string))
-	//任务完成，开始发送广播通知下面节点
-	if n >= repeateN && mapInfo["stop"] == nil {
-		log.Println("判重任务完成发送udp")
-		for _, to := range nextNode {
-			sid, _ := mapInfo["gtid"].(string)
-			eid, _ := mapInfo["lteid"].(string)
-			key := sid + "-" + eid + "-" + util.ObjToString(to["stype"])
-			by, _ := json.Marshal(map[string]interface{}{
-				"gtid":  sid,
-				"lteid": eid,
-				"stype": util.ObjToString(to["stype"]),
-				"key":   key,
-			})
-			addr := &net.UDPAddr{
-				IP:   net.ParseIP(to["addr"].(string)),
-				Port: util.IntAll(to["port"]),
+	if !IsFull {
+		updateOcrFileData(mapInfo["lteid"].(string))
+		//任务完成，开始发送广播通知下面节点
+		if n >= repeateN && mapInfo["stop"] == nil {
+			log.Println("判重任务完成发送udp")
+			for _, to := range nextNode {
+				sid, _ := mapInfo["gtid"].(string)
+				eid, _ := mapInfo["lteid"].(string)
+				key := sid + "-" + eid + "-" + util.ObjToString(to["stype"])
+				by, _ := json.Marshal(map[string]interface{}{
+					"gtid":  sid,
+					"lteid": eid,
+					"stype": util.ObjToString(to["stype"]),
+					"key":   key,
+				})
+				addr := &net.UDPAddr{
+					IP:   net.ParseIP(to["addr"].(string)),
+					Port: util.IntAll(to["port"]),
+				}
+				node := &udpNode{by, addr, time.Now().Unix(), 0}
+				udptaskmap.Store(key, node)
+				udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr)
 			}
-			node := &udpNode{by, addr, time.Now().Unix(), 0}
-			udptaskmap.Store(key, node)
-			udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr)
 		}
 	}