no message

fullproject/src_v1/config.json

@@ -35,7 +35,7 @@
             "memo": "修改项目创建new"
         }
     ],
-    "blacklist": ["采购项目", "项目", "工程", "开标", "建设项目", "改造", "扩容", "中标", "公告", "中标公告", "成交", "入围", "招标", "变更", "开标结果", "入围招标变更", "成交信息", "PPP", "资格预审", "资格", "预审", "采购", "供应商", "工程",
+    "rp_blacklist": ["采购项目", "项目", "工程", "开标", "建设项目", "改造", "扩容", "中标", "公告", "中标公告", "成交", "入围", "招标", "变更", "开标结果", "入围招标变更", "成交信息", "PPP", "资格预审", "资格", "预审", "采购", "供应商", "工程",
         "结果", "购置合同", "购置", "五标段", "六标段", "合同", "竞争性谈判", "竞争性", "谈判", "公示时间", "采购", "公开", "招标公告", "废标", "流标", "废标公告", "流标公告", "磋商", "询价", "项目联系人", "中标工期", "随机抽取", "全过程",
         "中标信息", "定标日期", "单价", "中标标的名称", "单位名称", "经济类", "法律类", "云南", "贵州", "其他事项", "其它事项", "法律", "行政法规", "单位", "学历", "修改", "澄清", "无法律", "法规", "成员", "说明", "公示期", "时至", "时止）",
         "中标候选人", "中标结果", "招标人名称", "项目名称", "业主代表", "中标金额", "本项目", "不接受", "招标人代表", "项目实施", "信誉", "业绩", "合法", "完整", "有效", "在此", "资格标", "商务标", "公告期", "成交价", "检测", "检疫", "经评审",
@@ -50,5 +50,35 @@
         "出厂单价", "接管", "查封", "天线", "开关电源", "无源器件", "海商等)", "第二条", "元整", "生产加工", "管理", "运杂", "税前价格", "税额", "计划", "羊口", "管理费", "税金等", "项目概况", "风险费", "山寨", "都不", "母公司", "但银行", "将以",
         "委托", "第五部分", "平顶山", "禹州", "郏县", "包括晋钢", "计划号", "大件运输", "合资", "商标", "国产", "茂名", "旅工", "燕山", "真实", "完整的", "不围标", "不造假", "双水", "台山", "合山", "大槐", "开平", "阳西", "公司", "通过询价", "税率",
         "计量单位", "包括价格", "华润", "融创", "龙湖", "世茂", "保利", "碧桂园", "中海", "中南置地", "华夏幸福", "招商", "营业执照", "中主", "五环", "以水电", "包件", "九德路", "保管", "包件号", "包件售价", "国标", "类型", "检修", "保养", "保函",
-        "信用证", "化工", "商业秘密", "大宗商品", "应付", "应收", "中共党员", "不可卸载", "商的澄清", "项目编号", "运杂费", "台风", "严重火灾", "初审表", "酒店", "商业"]
+        "信用证", "化工", "商业秘密", "大宗商品", "应付", "应收", "中共党员", "不可卸载", "商的澄清", "项目编号", "运杂费", "台风", "严重火灾", "初审表", "酒店", "商业", "不交货", "不含税", "不通过", "不含税价", "东北区域", "中班台",
+        "中班椅", "书架", "书柜", "休息台", "不争辩", "不准确", "不压缩", "不处于", "不挑剔", "不留密码", "中标候选", "中级", "义务", "不存", "东山", "东莞", "中山", "佛山", "南方基地", "不处", "中材", "中美合资", "五效", "万元整", "不漏项", "东进西移",
+        "业务", "中级职称", "水土", "成都", "税率为", "合肥", "天津", "保养等", "东湖", "万张图片", "仪表", "中标人", "中标内容", "书桌", "不卡顿", "不跳帧", "不完整", "不组织", "之中", "东建业", "东北特钢", "业主", "不锈钢", "中转",
+        "凭借栏杆", "不含", "不含斜", "中杯", "光分纤箱", "光交箱","宜宾", "合柜", "性质", "不及格等", "中央空调", "中检", "乌鲁木齐", "五个环", "乙方", "保密", "保证", "人民币", "无效", "以活动", "从化", "休息室", "位置", "储入", "中间件", "公众",
+        "关键", "化学", "家禽", "水产等", "羊肉", "运行", "金属", "定标", "银行", "本科", "理由", "第五章", "计算机", "通信", "项目管理", "评标", "利益", "香港", "在家", "年度", "不得", "不提供", "中职教材", "以及", "业务", "种类", "从业人员"],
+    "winner": {
+        "pre_regexp": ["及报价\\d拟中标人第一:", "中标金额（元）", "第\\d中标候单位:", "第\\d中标侯选人:", "及报价\\d", "成交金额（元）", "成交金额", "成交金额包\\d", "成交金额(元)", "成交金额", "中标供应商为", "</tr><tr><tdcolspan=\"8\">",
+            "中标供应商如下:", "中标供应商为", "中标供应商联系地址中标金额(万元)\\d", "基本情况名称:", "基本情况中标候选人第\\d+名:", "______", "第一中标侯选人名称: \\d", "第一中标侯选单位:", "第一中标供应商-", "第一中标排序人:", "第一中标排序人",
+            "第一成交侯选人:", "第一成交供应商-", "第一成交候选供应商及报价:", "拟定供应商名称:", "名单及其排序名次第一名单位名称", "名单及其排序名\\d次第一名单位名称", "名单及其排序推荐中标人单位名称", "名单及其排序", "名称:", "是:", "\\(成交\\)", "\\)￥41100.00",
+            "\\)为:", "预成交单位:", "最终报价第(二|三)低的", "同意推荐最终报价最低的", "单位名称", "的供应商", "推荐", "第一名", "\\([A-Z]\\)", "（[A-Z]）", "<[^>]*>", "[^a-zA-Z\\p{Han}]{1,}", "[A-Z](-|、)", "A\\d+", "B\\d+", "?[a-zA-Z]?[包|段]",
+            "①"],
+        "back_regexp": ["为中标人$", "为预中标供应商$", "为成交候选人$", "为该项目的预成交单位$", "为成交单位$", "为本次招标项目的中标单位$", "（公司$", "工期:0质量要求:null保证金金额:null$", ",报价:182391224.2900工期$", "786000元$", "预成交人地址",
+            "为该项目的预成交单位$", "为成交单位$", "预成交单位$", ",?(投标报价|报价)\\d{1,}\\.\\d{1,}(万?)元$", ",?(投标报价|报价)\\d+.?\\d+(万?)元$", ",?(投标报价|报价)\\d+.?\\d+(万?)元/平方米$", ",\\[慧通主要产品报价点击下载\\]$", ",$", ",?中标金额\\d{1,}(万?)元$", ",?中标金额\\d{1,}\\.\\d{1,}(万?)元$",
+            ",_$", "单位名称$", ",为本项目.*$", "[^a-zA-Z\\p{Han}]{1,}$"],
+        "back_rep_regexp": ["(有限公$)#有限公司", "(有限责任公$)#有限责任公司", "(公司公司&)#公司", "(公司等&)#公司"],
+        "blacklist": ["项目废标", "标项内容", "单位名称", "null", "不足3家", "中标单位", "公告时间", "（或印鉴）", "中标金额", "法定代表人", "员会名单", "含全部内", "工期", "报价", "名称得分", "元）备注", "的全部内", "地址", "\\.\\.", "法定家数", "的投标人",
+            "流水号", "到我公司", "投标报价", "工期", "一致确认", "多家中标商", "违法违规行", "中标人名称", "格式一致", "寄送中心", "\\.\\.\\.元", "电话地址", "全部内", "合同包合计", "楼青年公寓装修工程"]
+    },
+    "buyer": {
+        "pre_regexp": ["\\(不见面开标\\)", "\\(买方）", "\\(\\d+个月\\)", "\\d{4}年", "\\d+月", "<[^>]*>", "AB", "CC", "CB", "CA", "[A-Z]?\\d+", "\\(第(\\d+|一|二|三|四|五|六|七)(批|次)?\\)", "\\(\\d+局\\)", "\\d+标段", "\\(二次）", "(重招\\d)", "(\\(|（?)周(五|四|三|二|一|日|六)?(\\)|）)",
+            "(\\(|（)?[a-zA-Z](\\)|）)?", "(\\(|（)?\\d{0,8}.?\\d+(\\)|）)?", "#(~|、)?", "#\\d{1,};?", "(\\+|\\.|、|-|#|%|:|：)+"],
+        "back_regexp": ["采购人$", "[^a-zA-Z\\p{Han}]{1,}$"],
+        "back_rep_regexp": ["(有限公$)#有限公司", "(有限责任公$)#有限责任公司", "(公司公司&)#公司", "(公司等&)#公司"],
+        "blacklist": ["有问题", "个县公司", "T及分公司", "大厦\\d+室", "住宅楼", "购物中心", "盖法人章", "东侧路面拓宽", "、技术研发中心", "地下室", "车库", "二公司", "二类居住", "钢芯铝绞线", "已仔细"]
+    },
+    "agency": {
+        "pre_regexp": ["<[^>]*>", "(\\(|（)?[a-zA-Z](\\)|）)?", "(\\(|（)?\\d{0,8}.?\\d+(\\)|）)?", "#(~|、)?", "#\\d{1,};?", "(\\+|\\.|、|-|#|%|:|：)+"],
+        "back_regexp": ["为其所$"],
+        "back_rep_regexp": ["(有限公$)#有限公司", "(有限责任公$)#有限责任公司", "(公司公司&)#公司", "(公司等&)#公司", "([a-z]限公司$)#有限公司", "(有公司$)#有限公司", "(招标代理有$)#招标代理有限公司"],
+        "blacklist": []
+    }
 }

fullproject/src_v1/init.go

@@ -81,6 +81,9 @@ func init() {
 	for _, v := range BlackList {
 		BlaskListMap[util.ObjToString(v)] = true
 	}
+	initWinnerRegexp()
+	initBuyerRegexp()
+	initAgencyRegexp()
 
 	//加载项目数据
 	//---不能通过
@@ -414,3 +417,129 @@ func CosineSimilar(srcWords1, dstWords1 string) float64 {
 	//	}
 	return v1
 }
+
+
+func initWinnerRegexp() {
+	winRegMap := Sysconfig["winner"].(map[string]interface{})
+	preRegexps := winRegMap["pre_regexp"].([]interface{})
+	backRegexps := winRegMap["back_regexp"].([]interface{})
+	backRepRegexps := winRegMap["back_rep_regexp"].([]interface{})
+	backBlack := winRegMap["blacklist"].([]interface{})
+	var winPreRegexps []*regexp.Regexp
+	for _, v := range preRegexps {
+		reg := regexp.MustCompile("^" + v.(string))
+		winPreRegexps = append(winPreRegexps, reg)
+	}
+	PreRegexp["winner"] = winPreRegexps
+
+	var winBackRegexps []*regexp.Regexp
+	for _, v := range backRegexps {
+		reg := regexp.MustCompile(v.(string))
+		winBackRegexps = append(winBackRegexps, reg)
+	}
+	BackRegexp["winner"] = winBackRegexps
+
+	var winBackRepRegexps []RegexpInfo
+	for _, v := range backRepRegexps {
+		reps := strings.Split(v.(string), "#")
+		if len(reps) > 1 {
+			reg := RegexpInfo{
+				regs:   regexp.MustCompile(reps[0]),
+				repstr: reps[1],
+			}
+			winBackRepRegexps = append(winBackRepRegexps, reg)
+		}
+	}
+	BackRepRegexp["winner"] = winBackRepRegexps
+
+	var winBlackRegexps []*regexp.Regexp
+	for _, v := range backBlack {
+		reg := regexp.MustCompile(v.(string))
+		winBlackRegexps = append(winBlackRegexps, reg)
+	}
+	BlackRegexp["winner"] = winBlackRegexps
+}
+
+
+func initBuyerRegexp() {
+	buyRegMap := Sysconfig["buyer"].(map[string]interface{})
+	preRegexps := buyRegMap["pre_regexp"].([]interface{})
+	backRegexps := buyRegMap["back_regexp"].([]interface{})
+	backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
+	backBlack := buyRegMap["blacklist"].([]interface{})
+	var winPreRegexps []*regexp.Regexp
+	for _, v := range preRegexps {
+		reg := regexp.MustCompile("^" + v.(string))
+		winPreRegexps = append(winPreRegexps, reg)
+	}
+	PreRegexp["buyer"] = winPreRegexps
+
+	var winBackRegexps []*regexp.Regexp
+	for _, v := range backRegexps {
+		reg := regexp.MustCompile(v.(string))
+		winBackRegexps = append(winBackRegexps, reg)
+	}
+	BackRegexp["buyer"] = winBackRegexps
+
+	var winBackRepRegexps []RegexpInfo
+	for _, v := range backRepRegexps {
+		reps := strings.Split(v.(string), "#")
+		if len(reps) > 1 {
+			reg := RegexpInfo{
+				regs:   regexp.MustCompile(reps[0]),
+				repstr: reps[1],
+			}
+			winBackRepRegexps = append(winBackRepRegexps, reg)
+		}
+	}
+	BackRepRegexp["buyer"] = winBackRepRegexps
+
+	var winBlackRegexps []*regexp.Regexp
+	for _, v := range backBlack {
+		reg := regexp.MustCompile(v.(string))
+		winBlackRegexps = append(winBlackRegexps, reg)
+	}
+	BlackRegexp["buyer"] = winBlackRegexps
+}
+
+func initAgencyRegexp() {
+	buyRegMap := Sysconfig["agency"].(map[string]interface{})
+	preRegexps := buyRegMap["pre_regexp"].([]interface{})
+	backRegexps := buyRegMap["back_regexp"].([]interface{})
+	backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
+	backBlack := buyRegMap["blacklist"].([]interface{})
+	var winPreRegexps []*regexp.Regexp
+	for _, v := range preRegexps {
+		reg := regexp.MustCompile("^" + v.(string))
+		winPreRegexps = append(winPreRegexps, reg)
+	}
+	PreRegexp["agency"] = winPreRegexps
+
+	var winBackRegexps []*regexp.Regexp
+	for _, v := range backRegexps {
+		reg := regexp.MustCompile(v.(string))
+		winBackRegexps = append(winBackRegexps, reg)
+	}
+	BackRegexp["agency"] = winBackRegexps
+
+	var winBackRepRegexps []RegexpInfo
+	for _, v := range backRepRegexps {
+		reps := strings.Split(v.(string), "#")
+		if len(reps) > 1 {
+			reg := RegexpInfo{
+				regs:   regexp.MustCompile(reps[0]),
+				repstr: reps[1],
+			}
+			winBackRepRegexps = append(winBackRepRegexps, reg)
+		}
+	}
+	BackRepRegexp["agency"] = winBackRepRegexps
+
+	var winBlackRegexps []*regexp.Regexp
+	for _, v := range backBlack {
+		reg := regexp.MustCompile(v.(string))
+		winBlackRegexps = append(winBlackRegexps, reg)
+	}
+	BlackRegexp["agency"] = winBlackRegexps
+}
+

fullproject/src_v1/main.go

@@ -71,7 +71,7 @@ func DealSign() {
 	}
 }
 
-func main() {
+func mainT() {
 	//udp跑增量  id段   project
 	//udp跑全量			qlT
 	//udp跑历史数据  信息id1,id2/或id段  ls
@@ -90,9 +90,9 @@ func main() {
 }
 
 //测试组人员使用
-func mainT() {
-	sid = "5a4909cf40d2d9bbe8ab328a"
-	eid = "5f512d1fc0145440737580da"
+func main() {
+	sid = "5f50518ec014544073750cb8"
+	eid = "5f50518ec014544073750cb8"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()

fullproject/src_v1/project.go

@@ -5,7 +5,6 @@ import (
 	"log"
 	"math"
 	qu "qfw/util"
-	"regexp"
 	"sort"
 	"strings"
 	"time"
@@ -226,8 +225,7 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 	}
 
 	if !bFindProject {
-		if info.SubType == "" || info.SubType == "变更" || info.SubType == "验收" || info.SubType == "违规" ||
-			info.SubType == "结果变更" || info.SubType == "其它" {
+		if !IsCreatePro(info) {
 			return
 		}
 		id, p1 := p.NewProject(tmp, info)
@@ -575,13 +573,8 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	}
 	//项目评审专家
 	if len(thisinfo.ReviewExperts) > 0 {
-		arr := FormatRp(thisinfo.ReviewExperts)
-		if len(arr) > 0 {
-			set["review_experts"] = thisinfo.ReviewExperts
-			p1.ReviewExperts = thisinfo.ReviewExperts
-		} else {
-			p1.ReviewExperts = arr
-		}
+		set["review_experts"] = thisinfo.ReviewExperts
+		p1.ReviewExperts = thisinfo.ReviewExperts
 	}
 	//标的物
 	if thisinfo.Purchasing != "" {
@@ -893,14 +886,8 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 
 	//项目评审专家
 	if len(thisinfo.ReviewExperts) > 0 {
-		arr := FormatRp(thisinfo.ReviewExperts)
-		if len(arr) > 0 {
-			set["review_experts"] = thisinfo.ReviewExperts
-			pInfo.ReviewExperts = thisinfo.ReviewExperts
-		} else {
-			pInfo.ReviewExperts = arr
-			set["review_experts"] = arr
-		}
+		set["review_experts"] = thisinfo.ReviewExperts
+		pInfo.ReviewExperts = thisinfo.ReviewExperts
 	}
 	if thisinfo.Purchasing != "" {
 		if pInfo.Purchasing == "" {
@@ -1338,19 +1325,17 @@ func ClearData(clearMap, tmp map[string]interface{}) {
 	}
 }
 
-//校验评审专家
-func FormatRp(tmp []string) []string {
-	arrTmp := []string{}
-	for _, v := range tmp {
-		// 汉字过滤(全汉字，2-4个字)
-		if ok, _ := regexp.MatchString("^[\\p{Han}]{2,4}$", v); !ok {
-			continue
-		}
-		//黑名单过滤
-		if BlaskListMap[v] {
-			continue
+func IsCreatePro(info *Info) (bol bool) {
+	bol = true
+	if info.SubType == "" || info.SubType == "变更" || info.SubType == "验收" || info.SubType == "违规" ||
+		info.SubType == "结果变更" || info.SubType == "其它" {
+		if info.ProjectName == "" && info.ProjectCode == "" {
+			bol = false
+		}else if info.ProjectName == "" && info.Buyer == "" {
+			bol = false
+		}else if info.ProjectCode == "" && info.Buyer == "" {
+			bol = false
 		}
-		arrTmp = append(arrTmp, v)
 	}
-	return arrTmp
+	return bol
 }

fullproject/src_v1/task.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"unicode/utf8"
 
 	"github.com/robfig/cron"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -23,6 +24,17 @@ import (
 转换成info对象
 **/
 
+var PreRegexp = map[string][]*regexp.Regexp{}
+var BackRegexp = map[string][]*regexp.Regexp{}
+var BackRepRegexp = map[string][]RegexpInfo{}
+var BlackRegexp = map[string][]*regexp.Regexp{}
+
+type RegexpInfo struct {
+	regs   *regexp.Regexp
+	repstr string
+}
+
+
 //项目合并对象
 type ProjectTask struct {
 	InitMinTime int64 //最小时间，小于0的处理一次
@@ -594,7 +606,7 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 		}
 	}()
 	fields := map[string]interface{} {"area": 1, "city": 1, "district": 1, "comeintime": 1, "publishtime": 1, "bidopentime": 1, "title": 1, "projectname": 1, "href": 1,
-		"projectcode": 1, "buyerclass": 1, "winner": 1, "s_winner": 1, "buyer": 1, "buyerperson": 1, "buyertel": 1, "infoformat": 1, "toptype": 1, "subtype": 1, "spidercode": 1,
+		"projectcode": 1, "buyerclass": 1, "winner": 1, "buyer": 1, "buyerperson": 1, "buyertel": 1, "infoformat": 1, "toptype": 1, "subtype": 1, "spidercode": 1,
 		"site": 1, "topscopeclass": 1, "subscopeclass": 1, "bidamount": 1, "budget": 1, "agency": 1, "package": 1, "jsondata": 1, "review_experts": 1, "purchasing": 1, "winnerorder": 1}
 	ms := sess.DB(db).C(coll).Find(q).Select(fields).Sort("publishtime")
 	if Sysconfig["hints"] != nil {
@@ -752,8 +764,13 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	} else {
 		thisinfo.Buyer = ""
 	}
-	//winners整理
-	winner, _ := tmp["winner"].(string)
+	//清理评审专家名单
+	if len(thisinfo.ReviewExperts) > 0 {
+		thisinfo.ReviewExperts = ClearRp(thisinfo.ReviewExperts)
+	}
+	//winners整理、清理
+	winner := WinnerAndBuyerFilter(tmp["winner"].(string), "winner")
+	tmp["winner"] = winner
 	m1 := map[string]bool{}
 	winners := []string{}
 	if winner != "" {
@@ -765,7 +782,7 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 		thisinfo.HasPackage = true
 		for _, p := range packageM {
 			pm, _ := p.(map[string]interface{})
-			pw, _ := pm["winner"].(string)
+			pw := WinnerAndBuyerFilter(pm["winner"].(string), "winner")
 			if pw != "" && !m1[pw] {
 				m1[pw] = true
 				winners = append(winners, pw)
@@ -773,6 +790,21 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 		}
 	}
 	thisinfo.Winners = winners
+	//清理winnerorder
+	var wins []map[string]interface{}
+	for _, v := range thisinfo.WinnerOrder {
+		w := WinnerAndBuyerFilter(util.ObjToString(v["entname"]), "winner")
+		if w != "" {
+			v["entname"] = w
+			wins = append(wins, v)
+		}
+	}
+	thisinfo.WinnerOrder = wins
+	//清理buyer
+	buyer := WinnerAndBuyerFilter(util.ObjToString(tmp["tmp"]), "buyer")
+	tmp["buyer"] = buyer
+	thisinfo.Buyer = buyer
+
 
 	thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
 	thisinfo.LenPTC = len([]rune(thisinfo.PTC))
@@ -935,3 +967,54 @@ func deleteSlice(arr []string, v, stype string) []string {
 	}
 	return arr
 }
+
+//校验评审专家
+func ClearRp(tmp []string) []string {
+	arrTmp := []string{}
+	for _, v := range tmp {
+		// 汉字过滤(全汉字，2-4个字)
+		if ok, _ := regexp.MatchString("^[\\p{Han}]{2,4}$", v); !ok {
+			continue
+		}
+		//黑名单过滤
+		if BlaskListMap[v] {
+			continue
+		}
+		arrTmp = append(arrTmp, v)
+	}
+	return arrTmp
+}
+
+func WinnerAndBuyerFilter(name, stype string) string {
+	name = strings.ReplaceAll(name, " ", "")
+
+	preReg := PreRegexp[stype]
+	for _, v := range preReg {
+		name = v.ReplaceAllString(name, "")
+	}
+	backReg := BackRegexp[stype]
+	for _, v := range backReg {
+		name = v.ReplaceAllString(name, "")
+	}
+	backRepReg := BackRepRegexp[stype]
+	for _, v := range backRepReg {
+		name = v.regs.ReplaceAllString(name, v.repstr)
+	}
+
+	blackReg := BlackRegexp[stype]
+	for _, v := range blackReg {
+		if v.MatchString(name) {
+			name = ""
+			break
+		}
+	}
+
+	if !regexp.MustCompile("[\\p{Han}]{4,}").MatchString(name) {
+		name = ""
+	}
+	if utf8.RuneCountInString(name) > 60 {
+		name = ""
+	}
+	return name
+}
+

projectforecast/config.json

@@ -1,11 +1,11 @@
 {
-  "mongodbServers": "192.168.3.207:27092",
+  "mongodbServers": "192.168.3.166:27082",
   "mongodbPoolSize": 10,
-  "mongodbName": "mixdata",
-  "mongoColl_pro": "projectinfo_c",
+  "mongodbName": "bqData",
+  "mongoColl_pro": "projectinfo_new",
   "mongoColl_ent": "buyer_enterprise",
   "mongoColl_tag": "project_biaoqian",
-  "mongoColl_save": "project_forecast",
+  "mongoColl_save": "project_new_forecast",
   "udpport": ":1182",
   "spiderCodes": ["jzgc_nxjzscjgfwxt_gcyj", "jzgc_fjsjshyxxgkpt_gcxmcx", "jzgc_ahszfhcxjshyggxxcxzx_qycx", "jzgc_ahszfhcxjshyggxxcxzx_xmcx", "jzgc_bjsjzscxxgkpt_qyxx", "jzgc_bjsjzscxxgkpt_xmxx_htbaxx", "jzgc_bjsjzscxxgkpt_xmxx_jgbaxx",
     "jzgc_bjsjzscxxgkpt_xmxx_lwfblyxx", "jzgc_bjsjzscxxgkpt_xmxx_sgxkxx", "jzgc_bjsjzscxxgkpt_xmxx_yjxx", "jzgc_cqszfhcxjswyh_qyxx", "jzgc_cqszfhcxjswyh_xmxx", "jzgc_fjsjshyxxgkpt_qyxxcx", "jzgc_gdsjshysjkfpt_qyxx", "jzgc_gdsjshysjkfpt_xmxx",
@@ -15,7 +15,7 @@
     "jzgc_sxsjzscjgggfwpt_xmxx", "jzgc_tjszfhcxjswyh_gcxmxx", "jzgc_sxsjzscjgycxxxfbpt_gcxmxxcx", "jzgc_sxsjzscjgycxxxfbpt_qycx", "jzgc_xjgcjsy_gcxm", "jzgc_xjgcjsy_qyxx", "jzgc_xzzzqjzscjgycxxxpt_gcxx", "jzgc_xzzzqjzscjgycxxxpt_qyxx", "jzgc_scszfcxjshysjgxpt_xmxx"
   ],
   "forecast": {
-    "规划科研": ["立项环评", "勘察设计", "建设准备", "前期准备"],
+    "规划可研": ["立项环评", "勘察设计", "建设准备", "前期准备"],
     "立项环评": ["勘察设计", "建设准备", "前期施工"],
     "勘察设计": ["建设准备", "前期施工"],
     "建设准备": ["前期施工"],

projectforecast/task.go

@@ -5,7 +5,6 @@ import (
 	"log"
 	"mongodb"
 	qu "qfw/util"
-	"strconv"
 	"time"
 )
 
@@ -75,12 +74,12 @@ func GetProjectData(t string) {
 			}
 		}
 	}()
-	uptime, err := strconv.ParseInt(t, 10, 64)
-	if err == nil {
-		qu.Debug(err)
-	}
+	//uptime, err := strconv.ParseInt(t, 10, 64)
+	//if err == nil {
+	//	qu.Debug(err)
+	//}
 	query := bson.M{
-		"updatetime": bson.M{"$gt": uptime},
+		//"updatetime": bson.M{"$gt": uptime},
 		"o_projectinfo.nature":     bson.M{"$in": Nature},
 		"spidercode": bson.M{"$in": SpiderCodes},
 		"$or": []bson.M{
@@ -88,8 +87,8 @@ func GetProjectData(t string) {
 			{"category_purpose": bson.M{"$in": Category}},
 		},
 	}
-	qu.Debug("query-----", CollPro, query["updatetime"])
-	filed := map[string]interface{}{"area": 1, "city": 1, "buyer": 1, "projectname": 1, "category": 1, "nature": 1, "category_buyer": 1, "category_purpose": 1, "stage": 1, "o_projectinfo": 1, "title": 1}
+	//qu.Debug("query-----", CollPro, query["updatetime"])
+	filed := map[string]interface{}{"area": 1, "city": 1, "buyer": 1, "projectname": 1, "nature": 1, "category_buyer": 1, "category_purpose": 1, "stage": 1, "o_projectinfo": 1, "title": 1}
 	it := sess.DB(Dbname).C(CollPro).Select(filed).Find(query).Iter()
 	var lastid interface{}
 L:

projectinfo/src/config.json

@@ -1,15 +1,15 @@
 {
-	"mgodb": "192.168.3.207:27092",
+	"mgodb": "192.168.3.166:27082",
 	"dbsize": 10,
-	"dbname": "mxs",
-	"dbcoll": "test",
+	"dbname": "bqData",
+	"dbcoll": "project_nijian",
 	"mixdata":{
 		"addr": "192.168.3.207:27092",
 		"db" : "mixdata",
 		"size": 10,
 		"buyer_ent": "buyer_enterprise",
 		"tagcoll": "project_biaoqian",
-		"savecoll": "project_forecast_new"
+		"savecoll": "project_forecast_28"
 	},
 	"udpport": ":1182",
 	"spidercodes": ["jzgc_nxjzscjgfwxt_gcyj", "jzgc_fjsjshyxxgkpt_gcxmcx", "jzgc_ahszfhcxjshyggxxcxzx_qycx", "jzgc_ahszfhcxjshyggxxcxzx_xmcx", "jzgc_bjsjzscxxgkpt_qyxx", "jzgc_bjsjzscxxgkpt_xmxx_htbaxx", "jzgc_bjsjzscxxgkpt_xmxx_jgbaxx",
@@ -30,6 +30,6 @@
 		"运行维护": ["物品采购"]
 	},
 	"category": ["道路", "学校", "医院"],
-	"nature": ["新建", "扩建", "拆建"],
+	"nature": ["翻建", "重建", "其他", "迁建", "新建", "扩建"],
 	"rate": "60%"
 }

projectinfo/src/main.go

@@ -59,7 +59,7 @@ func init() {
 	for _, nature := range util.ObjArrToStringArr(Sysconfig["nature"].([]interface{})) {
 		NatureMap[nature] = true
 	}
-	log.Println("NatureMap：", NatureMap)
+	//log.Println("NatureMap：", NatureMap)
 	SpiderCodesMap = make(map[string]bool)
 	for _, code := range util.ObjArrToStringArr(Sysconfig["spidercodes"].([]interface{})) {
 		SpiderCodesMap[code] = true
@@ -74,7 +74,7 @@ func init() {
 }
 
 func main() {
-	//GetProjectData("1397387921") //手动任务
+	GetProjectData("1397387921") //手动任务
 	ch := make(chan bool, 1)
 	<-ch
 }

projectinfo/src/task.go

@@ -165,18 +165,19 @@ func GetProjectData(t string) {
 	defer MongoTool.DestoryMongoConn(sess)
 	uptime, err := strconv.ParseInt(t, 10, 64)
 	if err != nil {
-		qu.Debug("时间转换错误：", t)
+		qu.Debug("时间转换错误：", uptime)
 		return
 	}
 	query := bson.M{
-		"updatetime": bson.M{"$gt": uptime},
+		//"updatetime": bson.M{"$gt": uptime},
 		"stage":      bson.M{"$exists": true},
-		// "$or": []bson.M{
-		// 	{"category_buyer": bson.M{"$in": Category}},
-		// 	{"category_purpose": bson.M{"$in": Category}},
-		// },
+		//"$or": []bson.M{
+		//	{"category_buyer": bson.M{"$in": Category}},
+		//	{"category_purpose": bson.M{"$in": Category}},
+		//},
+		//"$in": bson.M{"nature": bson.M{"$in": Sysconfig["nature"].([]interface{})}},
 	}
-	filed := map[string]interface{}{"area": 1, "city": 1, "buyer": 1, "projectname": 1, "category": 1, "nature": 1, "category_buyer": 1, "category_purpose": 1, "stage": 1, "o_projectinfo": 1, "title": 1, "spidercode": 1}
+	filed := map[string]interface{}{"area": 1, "city": 1, "buyer": 1, "projectname": 1, "category": 1, "nature": 1, "category_buyer": 1, "category_purpose": 1, "stage": 1, "title": 1, "spidercode": 1}
 	count, _ := sess.DB(Dbname).C(CollPro).Find(query).Count()
 	log.Println("共查询:", count, "条")
 	if count == 0 {
@@ -204,28 +205,28 @@ func GetProjectData(t string) {
 				return
 			}
 			//spidercode
-			code := qu.ObjToString(pro["spidercode"])
-			if !SpiderCodesMap[code] {
-				return
-			}
+			//code := qu.ObjToString(pro["spidercode"])
+			//if !SpiderCodesMap[code] {
+			//	return
+			//}
 			//category
 			category := GetCategory(pro)
 			if category == "" {
 				return
 			}
 			//o_projectinfo nature buyer
-			o_projectinfo, ok := pro["o_projectinfo"].(map[string]interface{})
-			if !ok || o_projectinfo == nil {
-				return
-			}
-			nature := qu.ObjToString(o_projectinfo["nature"])
+			//o_projectinfo, ok := pro["o_projectinfo"].(map[string]interface{})
+			//if !ok || o_projectinfo == nil {
+			//	return
+			//}
+			nature := qu.ObjToString(pro["nature"])
 			if !NatureMap[nature] {
 				return
 			}
-			buyer := qu.ObjToString(o_projectinfo["buyer"])
-			pro["buyer"] = buyer
-			pro["nature"] = nature
-			delete(pro, "o_projectinfo")
+			buyer := qu.ObjToString(pro["buyer"])
+			//pro["buyer"] = buyer
+			//pro["nature"] = nature
+			//delete(pro, "o_projectinfo")
 			//id
 			if id, ok := pro["_id"].(string); ok && id != "" {
 				pro["infoid"] = id