jy-data-extract/udpfilterdup/src3/dataMethod.go

package main

import (
	"math"
	"regexp"
	"strings"
	qutil "qfw/util"
)



//完善判重数据检测-前置条件
func convertArabicNumeralsAndLetters(data string) string {
	newData :=data
	res1, _ := regexp.Compile("[a-zA-Z]+");
	if res1.MatchString(data) {
		newData = res1.ReplaceAllStringFunc(data, strings.ToUpper);
	}
	res2, _ := regexp.Compile("[0-9]+");
	if res2.MatchString(newData) {
		arr1:=[]string {"0","1","2","3","4","5","6","7","8","9"}
		arr2:=[]string {"零","一","二","三","四","五","六","七","八","九"}
		for i:=0 ;i<len(arr1) ;i++  {
			resTemp ,_:=regexp.Compile(arr1[i])
			newData= resTemp.ReplaceAllString(newData, arr2[i]);
		}
	}
	return newData
}

func dealWithSpecialPhrases(str1 string,str2 string) (string,string) {
	newStr1:=str1
	newStr2:=str2
	res, _ := regexp.Compile("重新招标");
	if res.MatchString(newStr1) {
		newStr1 = res.ReplaceAllString(newStr1,"重招");
	}
	if res.MatchString(newStr2) {
		newStr2 = res.ReplaceAllString(newStr2,"重招");
	}
	return newStr1,newStr2
}
//关键词数量v
func dealWithSpecialWordNumber(info*Info,v*Info) int {
	okNum:=0
	if  info.titleSpecialWord || info.specialWord {
		okNum++
	}
	if  v.titleSpecialWord || v.specialWord {
		okNum++
	}
	return okNum
}

//关键词再次判断
func againRepeat(v *Info, info *Info) bool {
	if isBidopentimeInterval(info.bidopentime,v.bidopentime) {
		return true
	}
	if v.budget != info.budget && v.budget != 0 && info.budget != 0 {
		return true
	}
	if isBidWinningAmount(v.bidamount,info.bidamount) && v.bidamount != 0 && info.bidamount != 0{
		return true
	}
	if deleteExtraSpace(v.winner) != deleteExtraSpace(info.winner) && v.winner != "" && info.winner != "" {
		return true
	}
	if v.contractnumber != "" && info.contractnumber != "" && v.contractnumber != info.contractnumber {
		return true
	}
	if v.projectcode != "" && info.projectcode != "" && v.projectcode != info.projectcode {
		return true
	}

	return false
}

////站点再次判断
//func againSite(v *Info, info *Info) bool {
//
//	if v.budget != info.budget && v.budget != 0 && info.budget != 0 {
//		return true
//	}
//	if isBidWinningAmount(v.bidamount,info.bidamount) && v.bidamount != 0 && info.bidamount != 0{
//		return true
//	}
//	if deleteExtraSpace(v.winner) != deleteExtraSpace(info.winner) && v.winner != "" && info.winner != "" {
//		return true
//	}
//	if v.contractnumber != "" && info.contractnumber != "" && v.contractnumber != info.contractnumber {
//		return true
//	}
//	if v.projectcode != "" && info.projectcode != "" && v.projectcode != info.projectcode {
//		return true
//	}
//
//	return false
//}






//删除中标单位字符串中多余的空格(含tab)
func deleteExtraSpace(s string) string {
	//删除字符串中的多余空格，有多个空格时，仅保留一个空格
	s1 := strings.Replace(s, "  ", " ", -1)      //替换tab为空格
	regstr := "\\s{2,}"                          //两个及两个以上空格的正则表达式
	reg, _ := regexp.Compile(regstr)             //编译正则表达式
	s2 := make([]byte, len(s1))                  //定义字符数组切片
	copy(s2, s1)                                 //将字符串复制到切片
	spc_index := reg.FindStringIndex(string(s2)) //在字符串中搜索
	for len(spc_index) > 0 {                     //找到适配项
		s2 = append(s2[:spc_index[0]+1], s2[spc_index[1]:]...) //删除多余空格
		spc_index = reg.FindStringIndex(string(s2))            //继续在字符串中搜索
	}
	return string(s2)
}

//中标金额倍率：10000
func isBidWinningAmount(f1 float64 ,f2 float64) bool {

	if f1==f2||f1*10000==f2||f2*10000==f1 {
		return false
	}
	return true
}


//开标时间区间为一天
func isBidopentimeInterval(i1 int64 ,i2 int64) bool {
	if i1==0||i2==0 {
		return false
	}
	//不在同一天-或者同一天间隔超过六小时，属于不相等返回true
	timeOne,timeTwo:=i1,i2
	day1 := qutil.FormatDateByInt64(&timeOne, qutil.Date_yyyyMMdd)
	day2 := qutil.FormatDateByInt64(&timeTwo, qutil.Date_yyyyMMdd)
	if day1==day2 {
		//是否间隔超过十二小时
		if math.Abs(float64(i1-i2)) >43200.0 {
			return true
		}else {
			return false
		}
	}else {
		return true
	}
}

//开标时间区间为一天
func isTheSameDay(i1 int64 ,i2 int64) bool {
	if i1==0||i2==0 {
		return false
	}
	timeOne,timeTwo:=i1,i2
	day1 := qutil.FormatDateByInt64(&timeOne, qutil.Date_yyyyMMdd)
	day2 := qutil.FormatDateByInt64(&timeTwo, qutil.Date_yyyyMMdd)
	if day1==day2 {
		return true
	}
	return false
}



//前置0 五要素均相等认为重复
func leadingElementSame(v *Info, info *Info) bool {

	isok:= 0
	if info.projectname != "" && v.projectname == info.projectname {
		isok++
	}
	if info.buyer != "" && v.buyer == info.buyer {
		isok++
	}
	if info.subtype == "合同" || info.subtype == "验收" || info.subtype == "违规" {
		if info.contractnumber != "" && v.contractnumber == info.contractnumber {
			isok++
		}
	}else {
		if info.projectcode != "" && v.projectcode == info.projectcode {
			isok++
		}
	}
	if info.title != "" && v.title == info.title {
		isok++
	}
	if v.agency == info.agency &&info.agency != "" {
		isok++
	}
	if v.winner == info.winner&&info.winner != "" {
		isok++
	}

	if isok>=5 {
		return true
	}



	return false
}

//buyer的优先级
func buyerIsContinue(v *Info, info *Info) bool {
	if !isTheSameDay(info.publishtime,v.publishtime) {
		return true
	}
	if v.title != info.title && v.title != "" && info.title != ""{
		return true
	}
	if v.projectname != info.projectname && v.projectname != "" && info.projectname != ""{
		return true
	}
	//if v.budget != info.budget && v.budget != 0 && info.budget != 0 {
	//	return true
	//}
	//if isBidWinningAmount(v.bidamount,info.bidamount) && v.bidamount != 0 && info.bidamount != 0{
	//	return true
	//}
	//if deleteExtraSpace(v.winner) != deleteExtraSpace(info.winner) && v.winner != "" && info.winner != "" {
	//	return true
	//}
	if v.contractnumber != "" && info.contractnumber != "" && v.contractnumber != info.contractnumber {
		return true
	}
	if v.projectcode != "" && info.projectcode != "" && v.projectcode != info.projectcode {
		return true
	}

	return false
}



//无效数据
func invalidData(d1 string, d2 string, d3 string, d4 string) bool {
	var n int
	if d1 != "" {
		n++
	}
	if d2 != "" {
		n++
	}
	if d3 != "" {
		n++
	}
	if d4 != "" {
		n++
	}
	if n == 0 {
		return true
	}
	return false
}