Browse Source

项目合并

Jianghan 2 years ago
parent
commit
d4be118e2d

+ 4 - 1
README.md

@@ -42,5 +42,8 @@
 
 ## data_fusion 数据融合-基于判重数据进行的信息融合
 
-## bidding_sync 数据清洗同步线上
+## data_clear_sync 数据清洗同步线上
 + 数据清洗平台清洗过的数据同步至bidding表、es表、项目表中
+
+## data_project 项目合并
++  依赖bidding数据进行项目合并

+ 0 - 0
data_sync/common.toml → data_clear_sync/common.toml


+ 0 - 0
data_sync/config/conf.go → data_clear_sync/config/conf.go


BIN
data_clear_sync/data_sync


+ 0 - 0
data_sync/go.mod → data_clear_sync/go.mod


+ 0 - 0
data_sync/go.sum → data_clear_sync/go.sum


+ 1 - 1
data_sync/init.go → data_clear_sync/init.go

@@ -4,7 +4,7 @@ import (
 	"app.yhyue.com/data_processing/common_utils/elastic"
 	"app.yhyue.com/data_processing/common_utils/log"
 	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"data_sync/config"
+	"data_clear_sync/config"
 	"fmt"
 	"os"
 )

+ 1 - 1
data_sync/main.go → data_clear_sync/main.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"app.yhyue.com/data_processing/common_utils/redis"
-	"data_sync/config"
+	"data_clear_sync/config"
 	"flag"
 	"log"
 	"time"

+ 1 - 1
data_sync/task.go → data_clear_sync/task.go

@@ -5,7 +5,7 @@ import (
 	"app.yhyue.com/data_processing/common_utils/log"
 	"app.yhyue.com/data_processing/common_utils/mongodb"
 	"app.yhyue.com/data_processing/common_utils/redis"
-	"data_sync/config"
+	"data_clear_sync/config"
 	"fmt"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"

+ 110 - 0
data_project/config.json

@@ -0,0 +1,110 @@
+{
+    "udpport": ":1782",
+    "loadStart": 0,
+	"validdays":150,
+    "statusdays": 15,
+    "redis-addr": "project=192.168.3.207:1679",
+	"mongodbServers": "192.168.3.207:29099",
+    "mongodbPoolSize": 10,
+    "mongodbName": "wjh",
+	"hints":"_id_1_publishtime_1",
+    "extractColl": "extract",
+    "extractColl1": "extract",
+    "projectColl": "projectset",
+    "backupFlag": true,
+    "siteColl": "site",
+    "thread": 1,
+    "jkmail": {
+        "to": "wangjianghan@topnet.net.cn",
+        "api": "http://172.17.145.179:19281/_send/_mail"
+    },
+    "redis": {
+        "dbname": "qyxy_buyer",
+        "addr": "192.168.3.207:1679",
+        "db": "3"
+    },
+    "bidding": {
+        "addr": "192.168.3.207:29099",
+        "dbname": "qfw",
+        "dbsize": 5,
+        "uname": "",
+        "upwd": ""
+    },
+    "spider": {
+        "addr": "192.168.3.207:29099",
+        "dbname": "wjh",
+        "dbsize": 2
+    },
+    "es": {
+        "addr": "http://127.0.0.1:9801",
+        "index": "projectset",
+        "itype": "projectset",
+        "pool": 10
+    },
+    "nsq_id": {
+        "addr": "192.168.3.166:4150",
+        "topic": "project-id",
+        "channel": "projectset",
+        "concurrent": 1
+    },
+    "nextNode": [
+        {
+            "addr": "192.168.20.104",
+            "port": 1483,
+            "memo": "创建项目索引new"
+        },
+        {
+            "addr": "127.0.0.1",
+            "port": 17833,
+            "memo": "修改项目创建new"
+        }
+    ],
+    "rp_blacklist": ["采购项目", "项目", "工程", "开标", "建设项目", "改造", "扩容", "中标", "公告", "中标公告", "成交", "入围", "招标", "变更", "开标结果", "入围招标变更", "成交信息", "PPP", "资格预审", "资格", "预审", "采购", "供应商", "工程",
+        "结果", "购置合同", "购置", "五标段", "六标段", "合同", "竞争性谈判", "竞争性", "谈判", "公示时间", "采购", "公开", "招标公告", "废标", "流标", "废标公告", "流标公告", "磋商", "询价", "项目联系人", "中标工期", "随机抽取", "全过程",
+        "中标信息", "定标日期", "单价", "中标标的名称", "单位名称", "经济类", "法律类", "云南", "贵州", "其他事项", "其它事项", "法律", "行政法规", "单位", "学历", "修改", "澄清", "无法律", "法规", "成员", "说明", "公示期", "时至", "时止)",
+        "中标候选人", "中标结果", "招标人名称", "项目名称", "业主代表", "中标金额", "本项目", "不接受", "招标人代表", "项目实施", "信誉", "业绩", "合法", "完整", "有效", "在此", "资格标", "商务标", "公告期", "成交价", "检测", "检疫", "经评审",
+        "招标人", "道路", "水利", "田径", "游泳", "羽毛球", "求真", "务实", "无气泡", "毛刺", "进场安排", "合理", "由技术", "公平", "经济", "公正", "但是", "支付条件", "税费", "索赔", "安全", "税费", "管理费用", "中行", "农行", "南京", "平安",
+        "合格", "天内", "高风险", "终止合同", "任何加行", "涂改", "其它费", "改期的", "运输", "税收等", "第一名", "第二名", "第三名", "时间较紧", "能评", "洪评", "包装费", "安装费", "保险费", "运输费", "第400章", "清单", "第200章", "合计",
+        "第100章", "应及", "印存", "大华", "宇视", "法令", "公正廉洁", "不徇私", "法令", "公正廉洁", "但是", "支付条件", "税费", "索赔", "务费", "国内", "年份", "宣传费", "出口", "业务总额", "包装", "包装物料", "商务", "第四部分", "委员会由",
+        "其余", "检验", "空调", "台式电脑", "税金", "安装", "成本", "渠道", "公告期限", "第十八条", "项目用途", "随时", "利润", "年1", "保险", "时3", "包号", "开户行", "户名", "安装调试", "时0", "万元", "运费", "建筑面积", "平方米", "成交金额",
+        "其他", "通过", "支票", "谈判小组", "本票", "业主评委", "年0", "计划员", "大写", "合价", "安全员", "资质", "其他要求", "施工员", "中标单位", "答疑", "方式", "第十条", "性能", "五.", "施工", "洪水", "暴风", "滑坡", "保险负责赔偿", "石油石化",
+        "保险", "万元", "大写", "利润", "节能", "其它费用", "机械", "张**", "第二包", "甲方", "已方", "业主专家", "中国官网", "文件", "检验报告", "劳保", "同时", "第四条", "答复", "王**", "严格", "丰收", "时间", "银行支票", "生产厂家", "资料",
+        "第二章", "权值", "有无发热", "干咳", "姓名", "权值", "在各评委", "首先", "务条款", "应以支票", "本票", "有效性", "符合性", "完整性", "公告期限", "合价", "类别", "市政", "房建", "楼民", "性别", "保险费等", "运费", "以最", "行评审", "经查实",
+        "同时", "文件", "合同包", "来源地", "介绍", "通讯", "答复", "利润", "税收", "支票", "年龄", "本票", "用途", "徐**", "合同内容", "金额为", "建设", "不限", "世达", "陆**", "须在", "范围", "招标方式", "合同履约", "进度", "方法", "包料", "其中",
+        "出厂单价", "接管", "查封", "天线", "开关电源", "无源器件", "海商等)", "第二条", "元整", "生产加工", "管理", "运杂", "税前价格", "税额", "计划", "羊口", "管理费", "税金等", "项目概况", "风险费", "山寨", "都不", "母公司", "但银行", "将以",
+        "委托", "第五部分", "平顶山", "禹州", "郏县", "包括晋钢", "计划号", "大件运输", "合资", "商标", "国产", "茂名", "旅工", "燕山", "真实", "完整的", "不围标", "不造假", "双水", "台山", "合山", "大槐", "开平", "阳西", "公司", "通过询价", "税率",
+        "计量单位", "包括价格", "华润", "融创", "龙湖", "世茂", "保利", "碧桂园", "中海", "中南置地", "华夏幸福", "招商", "营业执照", "中主", "五环", "以水电", "包件", "九德路", "保管", "包件号", "包件售价", "国标", "类型", "检修", "保养", "保函",
+        "信用证", "化工", "商业秘密", "大宗商品", "应付", "应收", "中共党员", "不可卸载", "商的澄清", "项目编号", "运杂费", "台风", "严重火灾", "初审表", "酒店", "商业", "不交货", "不含税", "不通过", "不含税价", "东北区域", "中班台",
+        "中班椅", "书架", "书柜", "休息台", "不争辩", "不准确", "不压缩", "不处于", "不挑剔", "不留密码", "中标候选", "中级", "义务", "不存", "东山", "东莞", "中山", "佛山", "南方基地", "不处", "中材", "中美合资", "五效", "万元整", "不漏项", "东进西移",
+        "业务", "中级职称", "水土", "成都", "税率为", "合肥", "天津", "保养等", "东湖", "万张图片", "仪表", "中标人", "中标内容", "书桌", "不卡顿", "不跳帧", "不完整", "不组织", "之中", "东建业", "东北特钢", "业主", "不锈钢", "中转",
+        "凭借栏杆", "不含", "不含斜", "中杯", "光分纤箱", "光交箱","宜宾", "合柜", "性质", "不及格等", "中央空调", "中检", "乌鲁木齐", "五个环", "乙方", "保密", "保证", "人民币", "无效", "以活动", "从化", "休息室", "位置", "储入", "中间件", "公众",
+        "关键", "化学", "家禽", "水产等", "羊肉", "运行", "金属", "定标", "银行", "本科", "理由", "第五章", "计算机", "通信", "项目管理", "评标", "利益", "香港", "在家", "年度", "不得", "不提供", "中职教材", "以及", "业务", "种类", "从业人员", "公告于"],
+    "winner": {
+        "pre_regexp": ["及报价\\d拟中标人第一:", "中标金额(元)", "第\\d中标候单位:", "第\\d中标侯选人:", "及报价\\d", "成交金额(元)", "成交金额", "成交金额包\\d", "成交金额(元)", "成交金额", "中标供应商为", "</tr><tr><tdcolspan=\"8\">",
+            "中标供应商如下:", "中标供应商为", "中标供应商联系地址中标金额(万元)\\d", "基本情况名称:", "基本情况中标候选人第\\d+名:", "______", "第一中标侯选人名称: \\d", "第一中标侯选单位:", "第一中标供应商-", "第一中标排序人:", "第一中标排序人",
+            "第一成交侯选人:", "第一成交供应商-", "第一成交候选供应商及报价:", "拟定供应商名称:", "名单及其排序名次第一名单位名称", "名单及其排序名\\d次第一名单位名称", "名单及其排序推荐中标人单位名称", "名单及其排序", "名称:", "是:", "\\(成交\\)", "\\)¥41100.00",
+            "\\)为:", "预成交单位:", "最终报价第(二|三)低的", "同意推荐最终报价最低的", "单位名称", "的供应商", "推荐", "第一名", "\\([A-Z]\\)", "([A-Z])", "<[^>]*>", "[^a-zA-Z\\p{Han}]{1,}", "[A-Z](-|、)", "A\\d+", "B\\d+", "?[a-zA-Z]?[段]",
+            "①", "投标人被人民法院[,]{0,}"],
+        "back_regexp": ["为中标人$", "为预中标供应商$", "为成交候选人$", "为该项目的预成交单位$", "为成交单位$", "为本次招标项目的中标单位$", "(公司$", "工期:0质量要求:null保证金金额:null$", ",报价:182391224.2900工期$", "786000元$", "预成交人地址",
+            "为该项目的预成交单位$", "为成交单位$", "预成交单位$", ",投标人被人民法院$",
+            ",?(投标报价|报价)\\d{1,}\\.\\d{1,}(万?)元$", ",?(投标报价|报价)\\d+.?\\d+(万?)元$", ",?(投标报价|报价)\\d+.?\\d+(万?)元/平方米$", ",\\[慧通主要产品报价点击下载\\]$", ",$", ",?中标金额\\d{1,}(万?)元$",",?中标金额\\d{1,}\\.\\d{1,}(万?)元$",
+            ",_$", "单位名称$", ",为本项目.*$", "[^a-zA-Z\\p{Han}]{1,}$"],
+        "back_rep_regexp": ["(有限公$)#有限公司", "(有限责任公$)#有限责任公司", "(公司公司&)#公司", "(公司等&)#公司"],
+        "blacklist": ["项目废标", "标项内容", "单位名称", "null", "不足3家", "中标单位", "公告时间", "(或印鉴)", "中标金额", "法定代表人", "员会名单", "含全部内", "工期", "报价", "名称得分", "元)备注", "的全部内", "地址", "\\.\\.", "法定家数", "的投标人",
+            "流水号", "到我公司", "投标报价", "工期", "一致确认", "多家中标商", "违法违规行", "中标人名称", "格式一致", "寄送中心", "\\.\\.\\.元", "电话地址", "全部内", "合同包合计", "楼青年公寓装修工程"]
+    },
+    "buyer": {
+        "pre_regexp": ["\\(不见面开标\\)", "\\(买方)", "\\(\\d+个月\\)", "\\d{4}年", "\\d+月", "<[^>]*>", "AB", "CC", "CB", "CA", "[A-Z]?\\d+", "\\(第(\\d+|一|二|三|四|五|六|七)(批|次)?\\)", "\\(\\d+局\\)", "\\d+标段", "\\(二次)", "(重招\\d)", "(\\(|(?)周(五|四|三|二|一|日|六)?(\\)|))",
+            "(\\(|()?[a-zA-Z](\\)|))?", "(\\(|()?\\d{0,8}.?\\d+(\\)|))?", "#(~|、)?", "#\\d{1,};?", "(\\+|\\.|、|-|#|%|:|:)+"],
+        "back_regexp": ["采购人$", "[^a-zA-Z\\p{Han}]{1,}$"],
+        "back_rep_regexp": ["(有限公$)#有限公司", "(有限责任公$)#er有限责任公司", "(公司公司&)#公司", "(公司等&)#公司"],
+        "blacklist": ["有问题", "个县公司", "T及分公司", "大厦\\d+室", "住宅楼", "购物中心", "盖法人章", "东侧路面拓宽", "、技术研发中心", "地下室", "车库", "二类居住", "钢芯铝绞线", "已仔细"]
+    },
+    "agency": {
+        "pre_regexp": ["发布人:", "<[^>]*>", "(\\(|()?[a-zA-Z](\\)|))?", "(\\(|()?\\d{0,8}.?\\d+(\\)|))?", "#(~|、)?", "#\\d{1,};?", "(\\+|\\.|、|-|#|%|:|:)+"],
+        "back_regexp": ["为其所$", "(简称$", ":我公司$"],
+        "back_rep_regexp": ["(有限公$)#有限公司", "(有限公同$)#有限公司", "(有限责任公$)#有限责任公司", "(公司公司&)#公司", "(公司等&)#公司", "([a-z]限公司$)#有限公司", "(有公司$)#有限公司", "(招标代理有$)#招标代理有限公司"],
+        "blacklist": ["承包项目", "(二标段", "举行了", "提交一份公司", "驱动更新", "邱海丽", "集中代理采购", "公司组织的", "利用项目", "地址:", "交易网", "年龄:", "性别:", "竞争性谈判", "null", "停车场", "招标人", "制造项目", "(中|招)标公告",
+            "“组组通”工程", "备案单位(章)", "小时", "梁工"]
+    }
+}

+ 9 - 0
data_project/go.mod

@@ -0,0 +1,9 @@
+module project
+
+go 1.16
+
+require (
+	app.yhyue.com/data_processing/common_utils v0.0.0-20230519053026-75421af90e41 // indirect
+	github.com/goinggo/mapstructure v0.0.0-20140717182941-194205d9b4a9 // indirect
+	github.com/robfig/cron v1.2.0 // indirect
+)

+ 175 - 0
data_project/go.sum

@@ -0,0 +1,175 @@
+app.yhyue.com/data_processing/common_utils v0.0.0-20230519053026-75421af90e41 h1:MRE2bNcC4rzlzkMozqeAqyMazU/t8EHwH0vp/6W49JE=
+app.yhyue.com/data_processing/common_utils v0.0.0-20230519053026-75421af90e41/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/goinggo/mapstructure v0.0.0-20140717182941-194205d9b4a9 h1:wqckanyE9qc/XnvnybC6SHOb8Nyd62QXAZOzA8twFig=
+github.com/goinggo/mapstructure v0.0.0-20140717182941-194205d9b4a9/go.mod h1:64ikIrMv84B+raz7akXOqbF7cK3/OQQ/6cClY10oy7A=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
+github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

+ 624 - 0
data_project/init.go

@@ -0,0 +1,624 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/redis"
+	"app.yhyue.com/data_processing/common_utils/udp"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"log"
+	"math"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+)
+
+var (
+	Sysconfig                                      map[string]interface{} //读取配置文件
+	MongoTool, MgoBidding, MgoSpider               *mongodb.MongodbSim    //mongodb连接
+	ExtractColl, ProjectColl, BackupColl, SiteColl string                 //抽取表、项目表、项目快照表、站点表
+	ExtractColl1                                   string
+	Thread                                         int //配置项线程数
+	BlackList                                      []interface{}
+	BlaskListMap                                   map[string]bool
+
+	RedisCode string
+)
+
+var (
+	//判断是日期
+	_datereg   = regexp.MustCompile("20[0-2][0-9][年-][0-9]{1,2}[月-][0-9]{1,2}[日-]([0-9]{1,2}时[0-9]{0,2})?")
+	_numreg1   = regexp.MustCompile("^[0-9-]{1,8}$")
+	_zimureg1  = regexp.MustCompile("^[a-zA-Z-]{1,7}$")
+	_nzreg     = regexp.MustCompile("^[0-9a-zA-Z-]+$")
+	_hanreg    = regexp.MustCompile(`^[\p{Han}::【】\\[\\]()()--、]+$`)
+	replaceStr = regexp.MustCompile("(工程|采购|项目|[?!、【】()—()--]|栏标价|中标候选人|招标代理)")
+	//判断带有分包、等特定词的
+	pStr = regexp.MustCompile("(勘察|监理|施工|设计|验收|标段|分包|子包|[0-9A-Z]包|[一二三四五六七八九十0-9]批)")
+	//判断包含数值
+	nreg1 = regexp.MustCompile("[0-9]{2,}")
+	//判断包含字母
+	zreg1 = regexp.MustCompile("[a-zA-Z]{1,}")
+	//判断包含汉字
+	hreg1 = regexp.MustCompile(`[\p{Han}]+`)
+	//判断项目编号是在10以内的纯数字结构
+	numCheckPc = regexp.MustCompile("^[0-9-]{1,10}$")
+	//仅初始化使用
+	compareNoPass = map[string]bool{}
+	compareAB     = map[string]bool{}
+	compareAB2D   = map[string]bool{}
+	compareABD    = map[string]bool{}
+	compareAB2CD  = map[string]bool{}
+	compareABCD   = map[string]bool{}
+)
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	MongoTool = &mongodb.MongodbSim{
+		MongodbAddr: Sysconfig["mongodbServers"].(string),
+		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
+		DbName:      Sysconfig["mongodbName"].(string),
+	}
+	MongoTool.InitPool()
+	bidding, _ := Sysconfig["bidding"].(map[string]interface{})
+	MgoBidding = &mongodb.MongodbSim{
+		MongodbAddr: bidding["addr"].(string),
+		Size:        util.IntAll(bidding["dbsize"]),
+		DbName:      bidding["dbname"].(string),
+		UserName:    bidding["uname"].(string),
+		Password:    bidding["upwd"].(string),
+	}
+	MgoBidding.InitPool()
+	spider, _ := Sysconfig["spider"].(map[string]interface{})
+	MgoSpider = &mongodb.MongodbSim{
+		MongodbAddr: spider["addr"].(string),
+		Size:        util.IntAll(spider["dbsize"]),
+		DbName:      spider["dbname"].(string),
+	}
+	MgoSpider.InitPool()
+
+	ExtractColl = Sysconfig["extractColl"].(string)
+	ExtractColl1 = Sysconfig["extractColl1"].(string)
+	ProjectColl = Sysconfig["projectColl"].(string)
+	BackupColl = Sysconfig["projectColl"].(string) + "_back"
+	SiteColl = Sysconfig["siteColl"].(string)
+	Thread = util.IntAll(Sysconfig["thread"])
+	//NextNode = Sysconfig["nextNode"].([]interface{})
+	udpport, _ := Sysconfig["udpport"].(string)
+	udpclient = udp.UdpClient{Local: udpport, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Println("Udp服务监听", udpport)
+
+	BlackList = Sysconfig["rp_blacklist"].([]interface{})
+	BlaskListMap = make(map[string]bool)
+	for _, v := range BlackList {
+		BlaskListMap[util.ObjToString(v)] = true
+	}
+
+	redisC := Sysconfig["redis-addr"].(string)
+	redisCon := Sysconfig["redis"].(map[string]interface{})
+	RedisCode = util.ObjToString(redisCon["dbname"])
+	redis_addr := util.ObjToString(redisCon["addr"])
+	redis_db := util.IntAll(redisCon["db"])
+	//"qyxy_buyer=172.17.4.189:8379"
+	redis.InitRedis1(fmt.Sprintf("%s=%s", RedisCode, redis_addr), redis_db) // 采购单位与中标单位初次合作项目
+	redis.InitRedis1(redisC, 0)
+
+	initWinnerRegexp()
+	initBuyerRegexp()
+	initAgencyRegexp()
+
+	//加载项目数据
+	//---不能通过
+	vm := []string{"C", "D"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				compareNoPass[key] = true
+				//fmt.Println(key)
+			}
+		}
+	}
+	//fmt.Println("-------------------")
+
+	//三个元素一致 [AB][AB][AB],分值最高
+	vm = []string{"A", "B"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				compareAB[key] = true
+				//fmt.Println(key)
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareAB))
+	//---至少两个一致,其他可能不存在
+	//[AB][AB][ABD]
+	//[AB][ABD][AB]
+	vm = []string{"A", "B"}
+	vm2 := []string{"A", "B", "D"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 3; k++ {
+				key := vm[i] + vm[j] + vm2[k]
+				if !compareAB[key] {
+					compareAB2D[key] = true
+					//fmt.Println(key)
+
+				}
+			}
+		}
+	}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 3; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm2[j] + vm[k]
+				if !compareAB[key] {
+					compareAB2D[key] = true
+					//fmt.Println(key)
+
+				}
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareAB2D))
+	//---至少一个一致,其他可能不存在
+	//[ABD][ABD][ABD] //已经删除DDD
+	vm = []string{"A", "B", "D"}
+	for i := 0; i < 3; i++ {
+		for j := 0; j < 3; j++ {
+			for k := 0; k < 3; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] {
+					compareABD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareABD))
+
+	//[AB][ABCD][AB]
+	//[AB][AB][ABCD]
+	vm = []string{"A", "B"}
+	vm2 = []string{"A", "B", "C", "D"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 4; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm2[j] + vm[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
+					compareAB2CD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 4; k++ {
+				key := vm[i] + vm[j] + vm2[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
+					compareAB2CD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareAB2CD))
+	//[ABECD][ABECD][ABECD]  //已经删除[CD][CD][CD]   //这个要重点讨论
+	vm = []string{"A", "B", "C", "D"}
+	for i := 0; i < 4; i++ {
+		for j := 0; j < 4; j++ {
+			for k := 0; k < 4; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareABD[key] && !compareNoPass[key] && !compareAB2CD[key] {
+					compareABCD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+}
+
+func CheckHanAndNum(str string) (b bool) {
+	return nreg1.MatchString(str) && hreg1.MatchString(str)
+}
+func CheckZimuAndNum(str string) (b bool) {
+	return zreg1.MatchString(str) && nreg1.MatchString(str)
+}
+
+type KeyMap struct {
+	Lock sync.Mutex
+	Map  map[string]*Key
+}
+
+type ID struct {
+	Id   string
+	Lock sync.Mutex
+	P    *ProjectCache
+}
+type Key struct {
+	Arr  []string
+	Lock sync.Mutex
+}
+type IdAndLock struct {
+	Id   string
+	Lock sync.Mutex
+}
+
+func NewKeyMap() *KeyMap {
+	return &KeyMap{
+		Map:  map[string]*Key{},
+		Lock: sync.Mutex{},
+	}
+}
+
+//招标信息实体类
+type Info struct {
+	Id                  string                   `json:"_id"`
+	Href                string                   `json:"href"` //源地址
+	Publishtime         int64                    `json:"publishtime"`
+	Comeintime          int64                    `json:"comeintime"`
+	Title               string                   `json:"title"`
+	TopType             string                   `json:"toptype"`
+	SubType             string                   `json:"subtype"`
+	ProjectName         string                   `json:"projectname"`
+	ProjectCode         string                   `json:"projectcode"`
+	ProjectScope        string                   `json:"projectscope"`
+	ContractCode        string                   `json:"contractcode"`
+	Buyer               string                   `json:"buyer"`
+	Buyerperson         string                   `json:"buyerperson"`
+	Buyertel            string                   `json:"buyertel"`
+	Agency              string                   `json:"agency"`
+	Area                string                   `json:"area"`
+	City                string                   `json:"city"`
+	District            string                   `json:"district"`
+	Infoformat          int                      `json:"infoformat"`
+	ReviewExperts       []string                 `json:"review_experts"`
+	Purchasing          string                   `json:"purchasing"`
+	WinnerOrder         []map[string]interface{} `json:"winnerorder"`
+	ProjectScale        string                   `json:"project_scale"`
+	ProjectDuration     int                      `json:"project_duration"`
+	ProjectTimeUnit     string                   `json:"project_timeunit"`
+	ProjectStartDate    int64                    `json:"project_startdate"`
+	ProjectCompleteDate int64                    `json:"project_completedate"`
+	Payway              string                   `json:"payway"`
+	ContractGuarantee   bool                     `json:"contract_guarantee"`
+	BidGuarantee        bool                     `json:"bid_guarantee"`
+	Qualifies           []map[string]interface{} `json:"qualifies"`
+	EntIdList           []string                 `json:"entidlist"`
+	HasPackage          bool                     // `json:"haspackage"`
+	Package             map[string]interface{}   `json:"package"`
+	Topscopeclass       []string                 `json:"topscopeclass"`
+	Subscopeclass       []string                 `json:"subscopeclass"`
+	Buyerclass          string                   `json:"buyerclass"`
+	BidOpenTime         int64                    `json:"bidopentime"`
+	BidEndTime          int64                    `json:"bidendtime"`
+	Budget              float64                  `json:"budget"`
+	Bidamount           float64                  `json:"bidamount"`
+	Winners             []string
+	dealtype            int
+	PTC                 string //从标题中抽的项目编号
+	pnbval              int    //项目名称、编号、采购单位存在的个数
+	LenPC               int    //项目编号长度
+	LenPN               int    //项目名称长度
+	LenPTC              int    //标题抽的项目编号长度
+	//以下三个元素做对比,计算包含时候使用
+	PNBH  int //0初始,+包含,-被包含
+	PCBH  int
+	PTCBH int
+}
+
+// 内存 项目信息
+type ProjectCache struct {
+	Id            primitive.ObjectID `json:"_id"`
+	Ids           []string           `json:"ids,omitempty"`
+	FirstTime     int64              `json:"firsttime,omitempty"`   //项目的最早时间
+	LastTime      int64              `json:"lasttime,omitempty"`    //项目的最后时间
+	ProjectName   string             `json:"projectname,omitempty"` //项目名称
+	ProjectCode   string             `json:"projectcode,omitempty"` //项目代码唯一(纯数字的权重低)
+	Buyer         string             `json:"buyer,omitempty"`       //采购单位唯一
+	Agency        string             `json:"agency"`                //代理机构
+	Area          string             `json:"area"`                  //地区
+	City          string             `json:"city"`                  //地市
+	District      string             `json:"district"`              //区县
+	Bidamount     float64            `json:"bidamount,omitempty"`   //中标金额
+	Budget        float64            `json:"budget,omitempty"`      //预算
+	Bidstatus     string             `json:"bidstatus"`             //
+	Bidtype       string             `json:"bidtype"`               //
+	score         int
+	comStr        string
+	MPN           []string `json:"mpn,omitempty"` //合并后多余的项目名称
+	MPC           []string `json:"mpc,omitempty"` //合并后多余的项目编号
+	resVal, pjVal int
+}
+
+// 项目信息
+type Project struct {
+	Id            primitive.ObjectID `json:"_id"`
+	Ids           []string           `json:"ids,omitempty"`
+	FirstTime     int64              `json:"firsttime,omitempty"`   //项目的最早时间
+	LastTime      int64              `json:"lasttime,omitempty"`    //项目的最后时间
+	ProjectName   string             `json:"projectname,omitempty"` //项目名称
+	ProjectCode   string             `json:"projectcode,omitempty"` //项目代码唯一(纯数字的权重低)
+	Buyer         string             `json:"buyer,omitempty"`       //采购单位唯一
+	Agency        string             `json:"agency"`                //代理机构
+	Area          string             `json:"area"`                  //地区
+	City          string             `json:"city"`                  //地市
+	District      string             `json:"district"`              //区县
+	Bidamount     float64            `json:"bidamount,omitempty"`   //中标金额
+	Budget        float64            `json:"budget,omitempty"`      //预算
+	score         int
+	comStr        string
+	MPN           []string `json:"mpn,omitempty"` //合并后多余的项目名称
+	MPC           []string `json:"mpc,omitempty"` //合并后多余的项目编号
+	resVal, pjVal int
+
+	Topscopeclass      []string               `json:"topscopeclass,omitempty"`
+	Subscopeclass      []string               `json:"subscopeclass,omitempty"` //子行业分类
+	Winners            string                 `json:"s_winner,omitempty"`      //中标人
+	ContractCode       string                 `json:"contractcode,omitempty"`  //合同编号
+	Buyerperson        string                 `json:"buyerperson"`             //采购联系人
+	Buyertel           string                 `json:"buyertel"`                //采购联系人电话
+	Bidstatus          string                 `json:"bidstatus"`               //
+	Bidtype            string                 `json:"bidtype"`                 //
+	ReviewExperts      []string               `json:"review_experts"`          // 项目评审专家
+	Purchasing         string                 `json:"purchasing"`              // 标的物
+	Package            map[string]interface{} `json:"package,omitempty"`       //分包的对比对象
+	Buyerclass         string                 `json:"buyerclass"`              //采购单位分类
+	BidOpenTime        int64                  `json:"bidopentime,omitempty"`   //开标时间
+	BidEndTime         int64                  `json:"bidendtime,omitempty"`    //开标时间
+	Jgtime             int64                  `json:"jgtime"`                  //结果中标时间
+	Zbtime             int64                  `json:"zbtime"`                  //招标时间
+	Winnerorder        []string               `json:"winnerorder"`             //中标候选人
+	ProjectScale       string                 `json:"project_scale"`           //项目规模
+	ProjectDuration    int                    `json:"project_duration"`        //工期时长
+	ProjectTimeunit    string                 `json:"project_timeunit"`        //工期时长单位
+	ProjectStartDate   int64                  `json:"project_startdate"`       //开工日期
+	ProjctCompleteDate int64                  `json:"projct_completedate"`     //竣工日期
+	Payway             string                 `json:"payway"`                  //付款方式
+	ContractGuarantee  bool                   `json:"contract_guarantee"`      //履约保证金 是否支持包含
+	BidGuarantee       bool                   `json:"bid_guarantee"`           //投标保证金 是否支持包含
+	Qualifies          string                 `json:"qualifies"`               //资质条件
+	EntIdList          []string               `json:"entidlist"`               //企业id
+	//FirstCooperation   []string               `json:"first_cooperation"`       //first_cooperation
+	InfoFiled    map[string]InfoField `json:"infofield"`    //逻辑处理需要的info字段
+	Budgettag    int                  `json:"budgettag"`    //预算是否有效标记
+	Bidamounttag int                  `json:"bidamounttag"` //中标金额是否有效标记
+}
+
+//存储部分招标信息字段,业务逻辑处理需要
+type InfoField struct {
+	Budget       float64 `json:"budget"`
+	Bidamount    float64 `json:"bidamount"`
+	ContractCode string  `json:"contractcode"`
+	ProjectName  string  `json:"projectname"`
+	ProjectCode  string  `json:"projectcode"`
+	Bidstatus    string  `json:"bidstatus"`
+}
+
+//站点信息
+type Site struct {
+	Id       string `json:"_id"`
+	Site     string `json:"site"`     //站点名字
+	Area     string `json:"area"`     //省
+	City     string `json:"city"`     //市
+	District string `json:"district"` //区、县
+	Domain   string `json:"domain"`   //地址
+	Status   int    `json:"status"`   //
+}
+
+//二分字符串查找
+func BinarySearch(s []string, k string) int {
+	sort.Strings(s)
+	lo, hi := 0, len(s)-1
+	for lo <= hi {
+		m := (lo + hi) >> 1
+		if s[m] < k {
+			lo = m + 1
+		} else if s[m] > k {
+			hi = m - 1
+		} else {
+			return m
+		}
+	}
+	return -1
+}
+
+func Duplicate(a interface{}) (ret []interface{}) {
+	va := reflect.ValueOf(a)
+	for i := 0; i < va.Len(); i++ {
+		if i > 0 && reflect.DeepEqual(va.Index(i-1).Interface(), va.Index(i).Interface()) {
+			continue
+		}
+		ret = append(ret, va.Index(i).Interface())
+	}
+	return ret
+}
+
+//计算文本相似度
+func CosineSimilar(srcWords1, dstWords1 string) float64 {
+	srcWords, dstWords := strings.Split(srcWords1, ""), strings.Split(dstWords1, "")
+	// get all words
+	allWordsMap := make(map[string]int, 0)
+	for _, word := range srcWords {
+		if _, found := allWordsMap[word]; !found {
+			allWordsMap[word] = 1
+		} else {
+			allWordsMap[word] += 1
+		}
+	}
+	for _, word := range dstWords {
+		if _, found := allWordsMap[word]; !found {
+			allWordsMap[word] = 1
+		} else {
+			allWordsMap[word] += 1
+		}
+	}
+
+	// stable the sort
+	allWordsSlice := make([]string, 0)
+	for word, _ := range allWordsMap {
+		allWordsSlice = append(allWordsSlice, word)
+	}
+
+	// assemble vector
+	srcVector := make([]int, len(allWordsSlice))
+	dstVector := make([]int, len(allWordsSlice))
+	for _, word := range srcWords {
+		if index := BinarySearch(allWordsSlice, word); index != -1 {
+			srcVector[index] += 1
+		}
+	}
+	for _, word := range dstWords {
+		if index := BinarySearch(allWordsSlice, word); index != -1 {
+			dstVector[index] += 1
+		}
+	}
+
+	// calc cos
+	numerator := float64(0)
+	srcSq := 0
+	dstSq := 0
+	for i, srcCount := range srcVector {
+		dstCount := dstVector[i]
+		numerator += float64(srcCount * dstCount)
+		srcSq += srcCount * srcCount
+		dstSq += dstCount * dstCount
+	}
+	denominator := math.Sqrt(float64(srcSq * dstSq))
+
+	v1 := numerator / denominator
+	//	if v1 > 0.6 {
+	//		log.Println(v1, srcWords1, dstWords1)
+	//	}
+	return v1
+}
+
+func initWinnerRegexp() {
+	winRegMap := Sysconfig["winner"].(map[string]interface{})
+	//preRegexps := winRegMap["pre_regexp"].([]interface{})
+	//backRegexps := winRegMap["back_regexp"].([]interface{})
+	//backRepRegexps := winRegMap["back_rep_regexp"].([]interface{})
+	backBlack := winRegMap["blacklist"].([]interface{})
+	//var winPreRegexps []*regexp.Regexp
+	//for _, v := range preRegexps {
+	//	reg := regexp.MustCompile("^" + v.(string))
+	//	winPreRegexps = append(winPreRegexps, reg)
+	//}
+	//PreRegexp["winner"] = winPreRegexps
+
+	//var winBackRegexps []*regexp.Regexp
+	//for _, v := range backRegexps {
+	//	reg := regexp.MustCompile(v.(string))
+	//	winBackRegexps = append(winBackRegexps, reg)
+	//}
+	//BackRegexp["winner"] = winBackRegexps
+
+	//var winBackRepRegexps []RegexpInfo
+	//for _, v := range backRepRegexps {
+	//	reps := strings.Split(v.(string), "#")
+	//	if len(reps) > 1 {
+	//		reg := RegexpInfo{
+	//			regs:   regexp.MustCompile(reps[0]),
+	//			repstr: reps[1],
+	//		}
+	//		winBackRepRegexps = append(winBackRepRegexps, reg)
+	//	}
+	//}
+	//BackRepRegexp["winner"] = winBackRepRegexps
+
+	var winBlackRegexps []*regexp.Regexp
+	for _, v := range backBlack {
+		reg := regexp.MustCompile(v.(string))
+		winBlackRegexps = append(winBlackRegexps, reg)
+	}
+	BlackRegexp["winner"] = winBlackRegexps
+}
+
+func initBuyerRegexp() {
+	buyRegMap := Sysconfig["buyer"].(map[string]interface{})
+	//preRegexps := buyRegMap["pre_regexp"].([]interface{})
+	//backRegexps := buyRegMap["back_regexp"].([]interface{})
+	//backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
+	backBlack := buyRegMap["blacklist"].([]interface{})
+	//var winPreRegexps []*regexp.Regexp
+	//for _, v := range preRegexps {
+	//	reg := regexp.MustCompile("^" + v.(string))
+	//	winPreRegexps = append(winPreRegexps, reg)
+	//}
+	//PreRegexp["buyer"] = winPreRegexps
+
+	//var winBackRegexps []*regexp.Regexp
+	//for _, v := range backRegexps {
+	//	reg := regexp.MustCompile(v.(string))
+	//	winBackRegexps = append(winBackRegexps, reg)
+	//}
+	//BackRegexp["buyer"] = winBackRegexps
+
+	//var winBackRepRegexps []RegexpInfo
+	//for _, v := range backRepRegexps {
+	//	reps := strings.Split(v.(string), "#")
+	//	if len(reps) > 1 {
+	//		reg := RegexpInfo{
+	//			regs:   regexp.MustCompile(reps[0]),
+	//			repstr: reps[1],
+	//		}
+	//		winBackRepRegexps = append(winBackRepRegexps, reg)
+	//	}
+	//}
+	//BackRepRegexp["buyer"] = winBackRepRegexps
+
+	var winBlackRegexps []*regexp.Regexp
+	for _, v := range backBlack {
+		reg := regexp.MustCompile(v.(string))
+		winBlackRegexps = append(winBlackRegexps, reg)
+	}
+	BlackRegexp["buyer"] = winBlackRegexps
+}
+
+func initAgencyRegexp() {
+	buyRegMap := Sysconfig["agency"].(map[string]interface{})
+	//preRegexps := buyRegMap["pre_regexp"].([]interface{})
+	//backRegexps := buyRegMap["back_regexp"].([]interface{})
+	//backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
+	backBlack := buyRegMap["blacklist"].([]interface{})
+	//var winPreRegexps []*regexp.Regexp
+	//for _, v := range preRegexps {
+	//	reg := regexp.MustCompile("^" + v.(string))
+	//	winPreRegexps = append(winPreRegexps, reg)
+	//}
+	//PreRegexp["agency"] = winPreRegexps
+
+	//var winBackRegexps []*regexp.Regexp
+	//for _, v := range backRegexps {
+	//	reg := regexp.MustCompile(v.(string))
+	//	winBackRegexps = append(winBackRegexps, reg)
+	//}
+	//BackRegexp["agency"] = winBackRegexps
+
+	//var winBackRepRegexps []RegexpInfo
+	//for _, v := range backRepRegexps {
+	//	reps := strings.Split(v.(string), "#")
+	//	if len(reps) > 1 {
+	//		reg := RegexpInfo{
+	//			regs:   regexp.MustCompile(reps[0]),
+	//			repstr: reps[1],
+	//		}
+	//		winBackRepRegexps = append(winBackRepRegexps, reg)
+	//	}
+	//}
+	//BackRepRegexp["agency"] = winBackRepRegexps
+
+	var winBlackRegexps []*regexp.Regexp
+	for _, v := range backBlack {
+		reg := regexp.MustCompile(v.(string))
+		winBlackRegexps = append(winBlackRegexps, reg)
+	}
+	BlackRegexp["agency"] = winBlackRegexps
+}

+ 236 - 0
data_project/load_data.go

@@ -0,0 +1,236 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/redis"
+	"encoding/json"
+	"log"
+	"strings"
+	"sync"
+	"time"
+)
+
+//  初始加载数据,默认加载最近6个月的数据
+func (p *ProjectTask) loadData(starttime int64) {
+	log.Println("load project start..", starttime)
+	p.findLock.Lock()
+	defer p.findLock.Unlock()
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+	loadOver := make(chan bool)
+	q := map[string]interface{}{
+		"lasttime": map[string]interface{}{"$gte": starttime},
+	}
+	field := map[string]interface{}{"list": 0}
+	it := sess.DB(MongoTool.DbName).C(p.coll).Find(&q).Select(field).Iter()
+	n, count := 0, 0
+	pool := make(chan *ProjectCache, 1000)
+	go func() {
+		for {
+			select {
+			case tmp := <-pool:
+				n++
+				if n%10000 == 0 {
+					log.Println("current", n, "\n", tmp.Id, len(p.mapPn), len(p.mapPc), len(p.mapPb), len(p.mapHref)) //, tmp.ProjectName, tmp.MPN, tmp.ProjectCode, tmp.MPC, tmp.Buyer)
+				}
+				if tmp != nil {
+					id := tmp.Id.Hex()
+					for _, v := range append([]string{tmp.ProjectName}, tmp.MPN...) {
+						if v != "" {
+							//v = pcReplace.ReplaceAllString(v, "")
+							if v != "" {
+								k := p.mapPn[v]
+								if k == nil {
+									k = &Key{Arr: []string{id}}
+									p.mapPn[v] = k
+								} else {
+									k.Arr = append(k.Arr, id)
+								}
+							}
+						}
+					}
+					for _, v := range append([]string{tmp.ProjectCode}, tmp.MPC...) {
+						if v != "" {
+							//v = pcReplace.ReplaceAllString(v, "")
+							if v != "" {
+								k := p.mapPc[v]
+								if k == nil {
+									k = &Key{Arr: []string{id}}
+									p.mapPc[v] = k
+								} else {
+									k.Arr = append(k.Arr, id)
+								}
+							}
+						}
+					}
+					if tmp.Buyer != "" && len([]rune(tmp.Buyer)) > 2 {
+						k := p.mapPb[tmp.Buyer]
+						if k == nil {
+							k = &Key{Arr: []string{id}}
+							p.mapPb[tmp.Buyer] = k
+						} else {
+							k.Arr = append(k.Arr, id)
+						}
+					}
+
+					p.AllIdsMapLock.Lock()
+					p.AllIdsMap[id] = &ID{Id: id, P: tmp}
+					p.AllIdsMapLock.Unlock()
+				}
+			case <-loadOver:
+				return
+			}
+		}
+	}()
+
+	ch := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+	for tmp := make(map[string]interface{}); it.Next(tmp); count++ {
+		//if count%20000 == 0 {
+		//	log.Println(fmt.Sprintf("iter --- %d", count))
+		//}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			bys, _ := json.Marshal(tmp)
+			var pc *ProjectCache
+			_ = json.Unmarshal(bys, &pc)
+			saveFiled(p, tmp, pc)
+			redis.PutCKV("project", pc.Id.Hex(), tmp)
+			pool <- pc
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	time.Sleep(2 * time.Second)
+	loadOver <- true
+	log.Println("load project over..", n)
+}
+
+func (p *ProjectTask) loadSite() {
+	log.Println("load site start..")
+	p.findLock.Lock()
+	defer p.findLock.Unlock()
+	p.mapSiteLock.Lock()
+	defer p.mapSiteLock.Unlock()
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	it := sess.DB(MongoTool.DbName).C(SiteColl).Find(&q).Iter()
+	n := 0
+	pool := make(chan *Site, 100)
+	over := make(chan bool)
+	go func() {
+		for {
+			select {
+			case tmp := <-pool:
+				n++
+				//站点有效标记state
+				if tmp != nil && tmp.Status == 5 {
+					p.mapSite[tmp.Site] = tmp
+				}
+			case <-over:
+				return
+			}
+		}
+	}()
+	for {
+		result := make(map[string]interface{})
+		if it.Next(&result) {
+			go func(res map[string]interface{}) {
+				bys, _ := json.Marshal(result)
+				var tmp *Site
+				_ = json.Unmarshal(bys, &tmp)
+				pool <- tmp
+			}(result)
+		} else {
+			break
+		}
+	}
+	time.Sleep(2 * time.Second)
+	over <- true
+	log.Println("load site over..", n)
+
+}
+
+func saveFiled(p *ProjectTask, res map[string]interface{}, tmp *ProjectCache) {
+	proHref := util.ObjToString(res["projecthref"])
+	if proHref != "" {
+		p.mapHrefLock.Lock()
+		p.mapHref[proHref] = tmp.Id.Hex()
+		p.mapHrefLock.Unlock()
+	}
+	if res["entidlist"] != nil {
+		elist := util.ObjArrToStringArr(res["entidlist"].([]interface{}))
+		wlist := strings.Split(util.ObjToString(res["s_winner"]), ",")
+		buyer := util.ObjToString(res["buyer"])
+		if len(elist) == len(wlist) && buyer != "" {
+			for i, eid := range elist {
+				if eid != "-" {
+					text := buyer + "," + wlist[i]
+					ex, _ := redis.Exists(RedisCode, text)
+					if !ex {
+						redis.PutCKV(RedisCode, text, tmp.Id.Hex())
+					}
+				}
+			}
+		}
+	}
+
+	//tmpMap := make(map[string]InfoField)
+	//infoMap := res["infofield"].(map[string]interface{})
+	//for _, v := range infoMap {
+	//	var field InfoField
+	//	b, _ := json.Marshal(v)
+	//	_ = json.Unmarshal(b, &field)
+	//	tmpMap[tmp.Id.Hex()] = field
+	//}
+	//tmp.InfoFiled = tmpMap
+}
+
+//  加载spidercode数据,isflow字段
+func (p *ProjectTask) loadSpiderCode() {
+	log.Println("load spider code start..")
+	p.findLock.Lock()
+	defer p.findLock.Unlock()
+	p.mapSpiderLock.Lock()
+	defer p.mapSpiderLock.Unlock()
+	sess := MgoSpider.GetMgoConn()
+	defer MgoSpider.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	field := map[string]interface{}{"code": 1, "isflow": 1}
+	it := sess.DB(MgoSpider.DbName).C("luaconfig").Find(&q).Select(field).Iter()
+	n := 0
+	pool := make(chan map[string]interface{}, 100)
+	over := make(chan bool)
+	go func() {
+		for {
+			select {
+			case tmp := <-pool:
+				n++
+				code := util.ObjToString(tmp["code"])
+				p.mapSpider[code] = util.IntAll(tmp["isflow"])
+			case <-over:
+				return
+			}
+		}
+	}()
+	for {
+		result := make(map[string]interface{})
+		if it.Next(&result) {
+			go func(res map[string]interface{}) {
+				pool <- result
+			}(result)
+		} else {
+			break
+		}
+	}
+	time.Sleep(2 * time.Second)
+	over <- true
+	log.Println("load spider over..", n)
+
+}

+ 266 - 0
data_project/main.go

@@ -0,0 +1,266 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/elastic"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/nsq"
+	"app.yhyue.com/data_processing/common_utils/udp"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+)
+
+var (
+	udpclient    udp.UdpClient //udp对象
+	SingleThread = make(chan bool, 1)
+	SingleClear  = 0
+	toaddr       = []*net.UDPAddr{} //下节点对象
+	ChSign       = make(chan os.Signal)
+	Es           *elastic.Elastic
+	Index        string
+	Itype        string
+	Mcmer        *gonsq.Consumer
+
+	sid, eid string //测试使用
+
+	UdpChan = make(chan map[string]interface{}, 500)
+)
+
+func init() {
+	signal.Notify(ChSign)
+	go DealSign()
+	nextNode := util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
+	for _, m := range nextNode {
+		toaddr = append(toaddr, &net.UDPAddr{
+			IP:   net.ParseIP(m["addr"].(string)),
+			Port: util.IntAll(m["port"]),
+		})
+	}
+	es := Sysconfig["es"].(map[string]interface{})
+	Es = &elastic.Elastic{
+		S_esurl: util.ObjToString(es["addr"]),
+		I_size:  util.IntAllDef(es["pool"], 10),
+	}
+	Index = util.ObjToString(es["index"])
+	Itype = util.ObjToString(es["itype"])
+	Es.InitElasticSize()
+
+	P_QL = NewPT()
+	go P_QL.updateAllQueue()
+	go P_QL.clearMem()
+
+}
+
+var queryClose = make(chan bool)
+var queryCloseOver = make(chan bool)
+
+func DealSign() {
+	for {
+		select {
+		case sign := <-ChSign:
+			//log.Println("receive:", sign)
+			if v, ok := sign.(syscall.Signal); ok && v == os.Interrupt {
+				log.Println("receice signal..,start close iter")
+				if P_QL.Brun {
+					queryClose <- true
+					select {
+					case <-queryCloseOver:
+					case <-time.After(30 * time.Second):
+					}
+				}
+				util.ReadConfig(&Sysconfig)
+				log.Println("signal deal over")
+			}
+		}
+	}
+}
+
+func main() {
+
+	P_QL.loadSpiderCode()
+	P_QL.loadSite()
+	if Sysconfig["loadStart"] != nil {
+		loadStart := util.Int64All(Sysconfig["loadStart"])
+		if loadStart > -1 {
+			P_QL.loadData(loadStart)
+		}
+	}
+	//go checkMapJob()
+	go P_QL.nsqMethod()
+
+	for {
+		mapinfo, ok := <-UdpChan
+		if !ok {
+			continue
+		}
+		SingleThread <- true
+		tasktype := util.ObjToString(mapinfo["stype"])
+		switch tasktype {
+		case "ql": //全量合并
+			go func() {
+				defer func() {
+					<-SingleThread
+				}()
+				P_QL.currentType = tasktype
+				P_QL.pici = time.Now().Unix()
+				P_QL.taskQl(mapinfo)
+			}()
+		case "project": //增量合并,
+			go func() {
+				defer func() {
+					<-SingleThread
+				}()
+				P_QL.currentType = tasktype
+				P_QL.pici = time.Now().Unix()
+				P_QL.taskZl(mapinfo)
+			}()
+		case "project_history": //增量合并, id段历史数据
+			go func() {
+				defer func() {
+					<-SingleThread
+				}()
+				P_QL.currentType = tasktype
+				P_QL.pici = time.Now().Unix()
+				P_QL.taskZl(mapinfo)
+			}()
+		case "updateInfo": //招标字段变更
+			go func() {
+				defer func() {
+					<-SingleThread
+				}()
+				P_QL.currentType = tasktype
+				P_QL.pici = time.Now().Unix()
+				P_QL.taskUpdateInfo(mapinfo)
+			}()
+		case "updatePro": //修改项目外围字段(只修改外围字段值)
+			go func() {
+				defer func() {
+					<-SingleThread
+				}()
+				P_QL.currentType = tasktype
+				P_QL.pici = time.Now().Unix()
+				P_QL.taskUpdatePro(mapinfo)
+			}()
+		case "deleteInfo": // 删除招标公告
+			go func() {
+				defer func() {
+					<-SingleThread
+				}()
+				P_QL.currentType = tasktype
+				P_QL.pici = time.Now().Unix()
+				P_QL.delInfoPro(mapinfo)
+			}()
+		case "spider": // 爬虫代码code、isflow
+			go func() {
+				defer func() {
+					<-SingleThread
+				}()
+				go P_QL.loadSpiderCode()
+			}()
+		case "history": //历史数据合并,暂时不写
+			go func() {
+				defer func() {
+					<-SingleThread
+				}()
+			}()
+		default:
+			<-SingleThread
+		}
+	}
+}
+
+func mainT() {
+	sid = "623d1ed4923488e17244bea7"
+	eid = "6327653467a6b0a286122df9"
+	//flag.StringVar(&sid, "sid", "", "开始id")
+	//flag.StringVar(&eid, "eid", "", "结束id")
+	//flag.Parse()
+
+	mapinfo := map[string]interface{}{}
+	if sid == "" || eid == "" {
+		log.Println("sid, eid参数不能为空")
+		os.Exit(0)
+	}
+	mapinfo["gtid"] = sid
+	mapinfo["lteid"] = eid
+	mapinfo["stype"] = "ql"
+	mapinfo["ip"] = "127.0.0.1"
+	mapinfo["port"] = Sysconfig["udpport"]
+
+	P_QL.loadSpiderCode()
+	P_QL.loadSite()
+	if Sysconfig["loadStart"] != nil {
+		loadStart := util.Int64All(Sysconfig["loadStart"])
+		if loadStart > -1 {
+			P_QL.loadData(loadStart)
+		}
+	}
+	P_QL.loadSite()
+	P_QL.currentType = mapinfo["stype"].(string)
+	P_QL.pici = time.Now().Unix()
+	P_QL.taskQl(mapinfo)
+	time.Sleep(99999 * time.Hour)
+}
+
+//udp调用信号
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case udp.OP_TYPE_DATA: //上个节点的数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Println("err:", err, "mapInfo:", mapInfo)
+		if err != nil {
+			_ = udpclient.WriteUdp([]byte("err:"+err.Error()), udp.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			key, _ := mapInfo["key"].(string)
+			if key == "" {
+				key = "udpok"
+			}
+			go udpclient.WriteUdp([]byte(key), udp.OP_NOOP, ra)
+			UdpChan <- mapInfo
+		}
+	case udp.OP_NOOP: //下个节点回应
+		ok := string(data)
+		if ok != "" {
+			udptaskmap.Delete(ok)
+			log.Println("ok:", ok)
+		}
+	}
+}
+
+// @Description nsq处理id不变,内容替换的竞品数据
+// @Author J 2022/8/10 11:40
+func (p *ProjectTask) nsqMethod() {
+	cof := Sysconfig["nsq_id"].(map[string]interface{})
+	var err error
+	Mcmer, err = gonsq.NewConsumer(&gonsq.Cconfig{
+		IsJsonEncode: true, //与生产者配置对应,设为true会取第1个字节进行类型判断
+		Addr:         util.ObjToString(cof["addr"]),
+		ConnectType:  0, //默认连接nsqd
+		Topic:        util.ObjToString(cof["topic"]),
+		Channel:      util.ObjToString(cof["channel"]),
+		Concurrent:   util.IntAllDef(cof["concurrent"], 1), //并发数
+	})
+	if err != nil {
+		util.Debug("nsqMethod err: ", err.Error())
+	}
+	for {
+		select {
+		case obj := <-Mcmer.Ch: //从通道读取即可
+			util.Debug("project nsq: " + fmt.Sprint(obj))
+			id := strings.Split(util.ObjToString(obj), "=")
+			if mongodb.IsObjectIdHex(id[1]) {
+				p.taskinfo(id[1])
+			} else {
+				util.Debug("jy nsq id err: ", id[1])
+			}
+		}
+	}
+}

+ 198 - 0
data_project/merge_comparepnc.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"strings"
+)
+
+//对比项目名称、项目编号
+
+func comparePNC(info *Info, compareProject *ProjectCache) (compareStr string, score int) {
+	if info.ProjectName != "" {
+		pns := []string{}
+		if compareProject.ProjectName != "" {
+			pns = append(pns, compareProject.ProjectName)
+		}
+		if len(compareProject.MPN) > 0 {
+			pns = append(pns, compareProject.MPN...)
+		}
+		ifind := 0
+		templen := 0
+		buyer := info.Buyer
+		if buyer == "" {
+			buyer = compareProject.Buyer
+		}
+		pn := info.ProjectName
+		if buyer != "" {
+			pn = strings.Replace(pn, buyer, "", -1)
+		}
+		for _, v := range pns {
+			if info.ProjectName == v {
+				ifind = 1
+				break
+			} else {
+				//if strings.Contains(info.ProjectName, v) || strings.Contains(v, info.ProjectName) ||
+				retv := CheckContain(info.ProjectName, v)
+				if retv == 1 {
+					ifind = 1
+					break
+				} else {
+					if buyer != "" {
+						v = strings.Replace(v, buyer, "", -1)
+					}
+					//v1 := CosineSimilar(pn, v)
+					if retv == 2 {
+						templen = len([]rune(v))
+						ifind = 2
+					} else if ifind == 0 {
+						ifind = 3
+					}
+				}
+			}
+		}
+		switch ifind {
+		case 0:
+			compareStr = "D"
+		case 1:
+			compareStr = "A"
+			score += 4
+			if len([]rune(info.ProjectName)) > 18 {
+				score += 2
+			}
+		case 2:
+			compareStr = "B"
+			score += 2
+			if templen > info.LenPN {
+				templen = info.LenPN
+			}
+			info.PNBH = templen
+			if templen > 12 {
+				score += 1
+			}
+		case 3:
+			compareStr = "C"
+		}
+	} else {
+		compareStr = "D"
+	}
+
+	/*
+				项目编号 - -()() 要注意
+				init_text = ["号","(重)","(第二次)","(重)"]
+		all_clean_mark = ["[","(","【","(","〖","]",")","】",")","〗","-","〔","〕","《","[","]","{","}","{","—"," ","-","﹝","﹞","–"]
+	*/
+	for index, pc := range []string{info.ProjectCode, info.PTC} {
+		if pc != "" {
+			pcs := []string{}
+			if compareProject.ProjectCode != "" {
+				pcs = append(pcs, compareProject.ProjectCode)
+			}
+			if len(compareProject.MPC) > 0 {
+				pcs = append(pcs, compareProject.MPC...)
+			}
+			ifind := 0
+			templen := 0
+			for _, v := range pcs {
+				if pc == v {
+					ifind = 1
+					break
+				} else {
+					// math.Abs(float64(len([]rune(pc))-len([]rune(v)))) < 6
+					//if !_numreg1.MatchString(pc) && !_zimureg1.MatchString(pc) && !_numreg1.MatchString(v) && !_zimureg1.MatchString(v)
+					if strings.Contains(pc, v) || strings.Contains(v, pc) {
+						t1 := pc
+						t2 := v
+						if len(v) > len(pc) {
+							t1 = v
+							t2 = pc
+						}
+						t3 := strings.Replace(t1, t2, "", -1)
+						t3 = _datereg.ReplaceAllString(t3, "")
+						if t3 == "" {
+							ifind = 1
+							break
+						} else {
+							ifind = 2
+							templen = len([]rune(v))
+						}
+					} else if ifind == 0 {
+						ifind = 3
+					}
+				}
+			}
+			switch ifind {
+			case 0:
+				compareStr += "D"
+			case 1:
+				compareStr += "A"
+				score += 4
+				if len([]rune(pc)) > 18 {
+					score += 2
+				}
+			case 2:
+				compareStr += "B"
+				score += 2
+				if index == 0 {
+					if templen > info.LenPC {
+						templen = info.LenPC
+					}
+					info.PCBH = templen
+					if templen > 12 {
+						score += 1
+					}
+
+				} else {
+					if templen > info.LenPTC {
+						templen = info.LenPTC
+					}
+					info.PTCBH = templen
+					if templen > 12 {
+						score += 1
+					}
+				}
+
+			case 3:
+				compareStr += "C"
+			}
+
+		} else {
+			compareStr += "D"
+		}
+	}
+	return
+}
+
+func CheckContain(b1, b2 string) (res int) {
+	b1 = replaceStr.ReplaceAllString(b1, "")
+	b2 = replaceStr.ReplaceAllString(b2, "")
+
+	if b1 == b2 {
+		res = 1 //相等
+		return
+	}
+	bs1 := []rune(b1)
+	bs2 := []rune(b2)
+	tmp := ""
+	for i := 0; i < len(bs1); i++ {
+		for j := 0; j < len(bs2); j++ {
+			if bs1[i] == bs2[j] {
+				tmp += string(bs1[i])
+			} else if tmp != "" {
+				b1 = strings.Replace(b1, tmp, "", -1)
+				b2 = strings.Replace(b2, tmp, "", -1)
+				tmp = ""
+			}
+		}
+	}
+	if tmp != "" {
+		b1 = strings.Replace(b1, tmp, "", -1)
+		b2 = strings.Replace(b2, tmp, "", -1)
+	}
+	if b1 == b2 {
+		res = 1 //相等
+	} else if b1 == "" || b2 == "" {
+		res = 2 //包含
+	} else {
+		res = 3 //不相等
+	}
+	return
+}

+ 510 - 0
data_project/merge_select.go

@@ -0,0 +1,510 @@
+package main
+
+//根据字符特征打分
+//3为最高分,pj为评级 A AD A  AA AA AB
+func Select(compareStr string, info *Info, compareInfo *ProjectCache) (res, pj int) {
+	//没有可对比的项目名称、或项目编号 //评级
+	if compareNoPass[compareStr] {
+
+	} else {
+		switch compareStr {
+		case "AAA":
+			res = 3
+			pj = 3
+		case "AAB":
+			res = 3
+			pj = 3
+		case "ABA":
+			res = 3
+			pj = 3
+		case "ABB":
+			if info.LenPN > 10 || info.PCBH > 8 || info.PTCBH > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BAA":
+			if info.PNBH > 10 || info.LenPC > 8 || info.LenPTC > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BAB":
+			if info.PNBH > 10 || info.LenPTC > 8 || info.PTCBH > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BBA":
+			if info.PNBH > 10 || info.PCBH > 8 || info.LenPC > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BBB":
+			v := 0
+			if info.PNBH > 10 {
+				v++
+			}
+			if info.PCBH > 8 {
+				v++
+			}
+			if info.PTCBH > 8 {
+				v++
+			}
+			if v > 1 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 2
+		case "AAD":
+			if info.LenPC > 8 || info.LenPN > 12 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "ABD":
+			if info.LenPN > 10 && info.PCBH > 8 {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BAD":
+			if info.LenPC > 13 || (info.PNBH > 10 && info.LenPC > 8) {
+				res = 3
+				pj = 3
+			} else if info.PNBH > 10 || info.LenPC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BBD":
+			if info.PNBH > 12 && info.PCBH > 10 {
+				res = 3
+				pj = 1
+			} else if info.PNBH > 10 && info.PCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "ADA":
+			if info.LenPN > 12 || (info.LenPTC > 8 && !StrOrNum2.MatchString(info.PTC)) {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 2
+		case "ADB":
+			if info.LenPN > 10 && info.PTCBH > 8 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PTCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BDA":
+			if info.PNBH > 10 && info.LenPTC > 8 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.PNBH > 10 || info.LenPTC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BDB":
+			if info.PNBH > 12 && info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+			} else if info.PNBH > 10 && info.PTCBH > 8 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 2
+		case "ADD":
+			if info.LenPN > 18 {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "BDD":
+			if info.PNBH > 10 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "DAA":
+			if info.LenPTC > 8 || info.LenPC > 8 {
+				res = 3
+				pj = 2
+			} else {
+				res = 2
+				pj = 3
+			}
+		case "DAB":
+			if info.LenPC > 8 && info.PTCBH > 8 {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 || info.PTCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "DAD":
+			if info.LenPC > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "DBA":
+			if info.PCBH > 8 && info.LenPC > 8 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.PCBH > 8 || info.LenPC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "DBB":
+			if info.PCBH > 10 && info.PTCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 1
+			} else if info.PCBH > 8 && info.PTCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "DBD":
+			if info.PCBH > 12 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+				pj = 1
+			} else {
+				res = 1
+				pj = 1
+			}
+		case "DDA":
+			if info.LenPTC > 14 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 1
+			} else if info.LenPTC > 8 {
+				res = 2
+				pj = 1
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "DDB":
+			if info.PTCBH > 12 && !StrOrNum2.MatchString(info.PTC) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "ACA":
+			if info.LenPN > 10 && info.LenPTC > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.LenPTC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "ACB":
+			if info.LenPN > 10 && info.PTCBH > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PTCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BCA":
+			if (info.PNBH > 10 && info.LenPTC > 8) || info.LenPTC > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.PNBH > 10 || info.LenPTC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BCB":
+			if info.PNBH > 12 && info.PTCBH > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 1
+			} else if info.PNBH > 10 || info.PTCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "AAC":
+			if (info.LenPN > 10 && info.LenPC > 8) || info.LenPN > 14 || (info.LenPC > 10 && !StrOrNum2.MatchString(info.ProjectCode)) {
+				res = 3
+				pj = 3
+			} else {
+				res = 2
+				pj = 3
+			}
+		case "ABC":
+			if info.LenPN > 14 && info.PCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BAC":
+			if info.PNBH > 14 && info.LenPC > 8 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.PNBH > 10 || info.LenPC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BBC":
+			if info.PNBH > 14 && info.PCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 1
+			} else if info.PNBH > 10 || info.PCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "ACC":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPN > 16 {
+					res = 2
+					pj = 1
+				} else {
+					res = 1
+					pj = 2
+				}
+			}
+		case "ACD":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPN > 16 {
+
+					res = 2
+				} else {
+					res = 1
+
+				}
+				pj = 1
+			}
+
+		case "ADC":
+			if info.LenPN > 16 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "BCC":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PNBH > 12 {
+					res = 1
+				}
+				pj = 1
+			}
+		case "BCD":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PNBH > 8 {
+					res = 1
+				}
+				pj = 1
+			}
+		case "BDC":
+			if info.PNBH > 7 {
+				res = 1
+			}
+			pj = 1
+		case "CAA":
+			if info.LenPC > 12 || info.LenPTC > 12 {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 || info.LenPTC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "CAB":
+			if info.LenPC > 12 && info.PTCBH > 8 {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 12 || info.PTCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "CAC":
+			if info.LenPC > 9 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 1
+			}
+		case "CAD":
+			if info.LenPC > 9 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "CBA":
+			if info.LenPTC > 14 && info.PCBH > 12 {
+				res = 3
+				pj = 2
+			} else if info.LenPTC > 12 || info.PCBH > 10 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "CBB":
+			if info.PCBH > 13 && info.PTCBH > 13 {
+				res = 3
+				pj = 1
+			} else if info.PCBH > 9 || info.PTCBH > 9 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "CBC":
+			if info.PCBH > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else if info.PCBH > 5 {
+				res = 1
+			}
+			pj = 1
+		case "CBD":
+			if info.PCBH > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else if info.PCBH > 5 {
+				res = 1
+			}
+			pj = 1
+		case "CCA":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPTC > 12 && !StrOrNum2.MatchString(info.PTC) {
+					res = 2
+				} else if info.LenPTC > 5 {
+					res = 1
+				}
+				pj = 1
+			}
+		case "CCB":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
+					res = 1
+				}
+				pj = 1
+			}
+		case "CDA":
+			if info.LenPTC > 12 && !StrOrNum2.MatchString(info.PTC) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "CDB":
+			if info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
+				res = 1
+				pj = 1
+			}
+		case "DAC":
+			if info.LenPC > 13 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+			} else if info.LenPC > 8 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "DBC":
+			if info.PCBH > 8 {
+				res = 1
+			}
+			pj = 1
+		case "DCA":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPTC > 10 {
+					res = 2
+				} else {
+					res = 1
+				}
+				pj = 1
+			}
+		case "DCB":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PTCBH > 8 && !StrOrNum2.MatchString(info.PTC) {
+					res = 1
+				}
+				pj = 1
+			}
+		}
+
+	}
+	return
+}

+ 1518 - 0
data_project/project.go

@@ -0,0 +1,1518 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/redis"
+	"encoding/json"
+	"math"
+	"sort"
+	"strings"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+/**
+项目合并,对比,计算,合并,生成项目
+**/
+
+//从对应map中获取对比的项目id
+func (p *ProjectTask) getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
+	//	p.ConCurrentLock(n1, n2, n3, n4)
+	//	defer p.ConCurrentUnLock(n1, n2, n3, n4)
+	p.wg.Add(1)
+	//查找到id数组
+	res = []*Key{}
+	//是否查找到,并标识位置。-1代表值为空或已经存在。
+	bpn, bpc, bptc, bpb = -1, -1, -1, -1
+	if pn != "" {
+		ids := p.mapPn[pn]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			p.mapPn[pn] = ids
+			bpn = 0
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+
+	if pc != "" {
+		ids := p.mapPc[pc]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			p.mapPc[pc] = ids
+			bpc = len(res)
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+
+	if ptc != "" {
+		ids := p.mapPc[ptc]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			p.mapPc[ptc] = ids
+			bptc = len(res)
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+
+	if pb != "" {
+		ids := p.mapPb[pb]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			p.mapPb[pb] = ids
+			bpb = len(res)
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+	repeatId := map[string]bool{}
+	idArr = []string{} //项目id
+	IDArr = []*ID{}    //项目信息
+	for _, m := range res {
+		for _, id := range m.Arr {
+			if !repeatId[id] {
+				repeatId[id] = true
+				//_, _ = strconv.ParseInt(id[0:8], 16, 64)
+				p.AllIdsMapLock.Lock()
+				Id := p.AllIdsMap[id]
+				p.AllIdsMapLock.Unlock()
+				if Id != nil {
+					Id.Lock.Lock()
+					idArr = append(idArr, id)
+					IDArr = append(IDArr, Id)
+				}
+			}
+		}
+	}
+	return
+}
+
+func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{}) {
+	p.findLock.Lock()
+	defer p.findLock.Unlock()
+	// 3.18 isfow=0数据不参与项目合并		(1表示正常数据招标流程)
+	code := strings.ReplaceAll(util.ObjToString(tmp["spidercode"]), " ", "")
+	p.mapSpiderLock.Lock()
+	defer p.mapSpiderLock.Unlock()
+	if v, ok := p.mapSpider[code]; ok && (v == 0) {
+		//qu.Debug("isflow=0 新建项目" + code)
+		p.NewProject(tmp, info)
+		return
+	}
+
+	//只有或没有采购单位的无法合并
+	//bpn, bpc, bptc, bpb 是否查找到,并标识位置。-1代表未查找到。
+	//pids 是项目id数组集合
+	//IDArr,是单个项目ID对象集合
+	bpn, bpc, bptc, bpb, pids, _, IDArr := p.getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer) //, info.LenPN, info.LenPC, info.LenPTC, len([]rune(info.Buyer)))
+	defer p.wg.Done()
+	//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock
+
+	for _, m := range pids {
+		defer m.Lock.Unlock()
+	}
+	for _, id := range IDArr {
+		defer id.Lock.Unlock()
+	}
+
+	bFindProject := false
+	findPid := ""
+	//获取完id,进行计算
+	//定义两组
+	comRes1 := []*ProjectCache{} //优先级最高的对比结果数组
+	comRes2 := []*ProjectCache{} //优化级其次
+	comRes3 := []*ProjectCache{}
+	for _, v := range IDArr {
+		comStr := ""
+		compareProject := v.P
+		compareProject.score = 0
+		//问题出地LastTime!!!!!
+		diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime)))
+		if diffTime <= p.validTime {
+			//代理机构完全不相同,直接新建项目
+			if CheckContain(compareProject.Agency, info.Agency) == 3 {
+				continue
+			}
+			//地区(省、市、区)不同,直接新建项目
+			if ComparePlace(compareProject, info) {
+				continue
+			}
+
+			info.PNBH = 0
+			info.PCBH = 0
+			info.PTCBH = 0
+			compareStr, score := comparePNC(info, compareProject)
+
+			resVal, pjVal := Select(compareStr, info, compareProject)
+			//---------------------------------------
+			if resVal > 0 {
+				compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount, score2 := p.compareBCTABB(info, compareProject, diffTime, score)
+
+				//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构
+				comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
+				compareProject.comStr = comStr
+				compareProject.pjVal = pjVal
+				compareProject.resVal = resVal
+				//log.Println(compareProject.comStr)
+				eqV := compareResult(resVal, pjVal, score2, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount)
+				if eqV == 1 {
+					comRes1 = append(comRes1, compareProject)
+				} else if eqV == 2 {
+					comRes2 = append(comRes2, compareProject)
+				} else if eqV == 3 {
+					comRes3 = append(comRes3, compareProject)
+				}
+			}
+		}
+	}
+	//--------------------------------对比完成-----------------------
+
+	//更新数组、更新项目
+	for kv, resN := range [][]*ProjectCache{comRes1, comRes2, comRes3} {
+		if len(resN) > 0 {
+			if len(resN) > 1 {
+				sort.Slice(resN, func(i, j int) bool {
+					return resN[i].score > resN[j].score
+				})
+			}
+
+			ex := 0
+			resArr := []*ProjectCache{}
+			for i, res := range resN {
+				choose, e := p.CompareStatus(resN[i], info)
+				if !choose {
+					ex = e
+					resArr = append(resArr, res)
+				}
+			}
+			if len(resArr) > 0 {
+				bFindProject = true
+				findPid = resArr[0].Id.Hex()
+				p.UpdateProject(tmp, info, resArr[0], kv+1, resArr[0].comStr, ex)
+				for k2, bv := range []int{bpn, bpc, bptc, bpb} {
+					if bv > -1 {
+						pids[bv].Arr = append(pids[bv].Arr, findPid)
+						if k2 == 0 {
+							if resArr[0].ProjectName == "" {
+								resArr[0].ProjectName = info.ProjectName
+							} else {
+								if resArr[0].MPN == nil {
+									resArr[0].MPN = []string{info.ProjectName}
+								} else {
+									resArr[0].MPN = append(resArr[0].MPN, info.ProjectName)
+								}
+							}
+						} else if k2 < 3 {
+							if resArr[0].ProjectCode == "" {
+								resArr[0].ProjectCode = util.If(k2 == 1, info.ProjectCode, info.PTC).(string)
+							} else {
+								if resArr[0].MPC == nil {
+									resArr[0].MPC = []string{util.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
+								} else {
+									resArr[0].MPC = append(resArr[0].MPC, util.If(k2 == 1, info.ProjectCode, info.PTC).(string))
+								}
+							}
+						} else {
+							if resArr[0].Buyer == "" {
+								resArr[0].Buyer = info.Buyer
+							}
+						}
+					}
+				}
+			} else {
+				bFindProject = false
+				findPid = ""
+			}
+			break
+		}
+	}
+
+	if !bFindProject {
+		//if !IsCreatePro(info) {
+		//	qu.Debug("舍弃数据---", info.Id)
+		//	return
+		//}
+		id, p1 := p.NewProject(tmp, info)
+		p.AllIdsMapLock.Lock()
+		p.AllIdsMap[id] = &ID{Id: id, P: p1}
+		p.AllIdsMapLock.Unlock()
+		for _, m := range pids {
+			m.Arr = append(m.Arr, id)
+		}
+	}
+}
+
+func (p *ProjectTask) compareBCTABB(info *Info, cp *ProjectCache, diffTime int64, score int) (compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string, score2 int) {
+	compareBuyer = "D"
+	if len([]rune(info.Buyer)) > 3 && len([]rune(cp.Buyer)) > 3 {
+		v := CheckContain(info.Buyer, cp.Buyer)
+		if v == 1 {
+			compareBuyer = "A"
+			score += 3
+		} else {
+			//v1 := CosineSimilar(info.Buyer, cp.Buyer)
+			if v == 2 {
+				compareBuyer = "B"
+				score += 1
+			} else {
+				compareBuyer = "C"
+			}
+		}
+	}
+	//---------------------------------------
+
+	compareCity = ""
+	if info.Area != "全国" && info.Area != "" && info.Area == cp.Area {
+		compareCity += "A"
+		score += 2
+	} else if info.Area == "全国" || cp.Area == "全国" {
+		compareCity += "B"
+		score += 1
+	} else {
+		compareCity += "C"
+	}
+	if compareCity != "C" {
+		if info.City != "" && info.City == cp.City {
+			compareCity += "A"
+			score += 2
+		} else {
+			if info.Area == "全国" || cp.Area == "全国" {
+				compareCity += "B"
+			} else if info.City == compareCity {
+				compareCity += "B"
+			} else {
+				compareCity += "C"
+			}
+		}
+	} else {
+		compareCity += "C"
+	}
+	score2 = 0
+	if compareCity == "AA" {
+		if info.District != "" && info.District == cp.District {
+			score2 = 1
+		}
+	}
+
+	compareTime = "D"
+	if diffTime < 45*86400 {
+		compareTime = "A"
+		score += 2
+	} else if diffTime < 90*86400 {
+		compareTime = "B"
+		score += 1
+	}
+
+	compareAgency = "D"
+	if info.Agency != "" {
+		if info.Agency == cp.Agency {
+			compareAgency = "A"
+			score += 2
+			score2 += 1
+		} else if cp.Agency != "" {
+			if strings.Contains(info.Agency, cp.Agency) || strings.Contains(cp.Agency, info.Agency) {
+				compareAgency = "B"
+				score += 1
+				score2 += 1
+			} else {
+				compareAgency = "C"
+			}
+		}
+	}
+	compareBudget = "C"
+	if info.Budget > 0 && (info.Budget == cp.Budget || (cp.Bidamount > 0 && info.Budget > cp.Bidamount && (info.Budget-cp.Bidamount) < (0.15*info.Budget))) {
+		compareBudget = "A"
+		score += 1
+		score2 += 1
+	}
+	//	else if info.Budget == 0 && cp.Budget == 0 {
+	//		compareBudget = "B"
+	//	}
+	compareBidmount = "C"
+	if info.Bidamount > 0 && (info.Bidamount == cp.Bidamount || (cp.Budget > 0 && cp.Budget > info.Bidamount && (cp.Budget-info.Bidamount) < 0.15*cp.Budget)) {
+		compareBidmount = "A"
+		score += 1
+		score2 += 1
+	}
+	//	else if info.Bidamount == 0 && cp.Bidamount == 0 {
+	//		compareBidmount = "B"
+	//	}
+
+	cp.score = score
+	return
+}
+
+func compareResult(resVal, pjVal, score2 int, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string) int {
+	eqV := 0
+	switch resVal {
+	case 3:
+		if pjVal == 3 && comStr[3:] != "CCCDCCC" {
+			eqV = 1
+		} else if compareBuyer < "C" {
+			if pjVal > 1 {
+				eqV = 1
+			} else { //if (compareCity[1:1] != "C" || compareTime != "D") && score2 > 0
+				eqV = 2
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && (compareCity[1:1] != "C" || score2 > 0) {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+		} else {
+			if pjVal == 3 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if pjVal == 2 && compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			} else if compareCity == "AA" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+		}
+	case 2:
+		if compareBuyer < "C" {
+			if pjVal > 1 {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" || score2 > 0 {
+				eqV = 3
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+
+		} else {
+			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		}
+	case 1:
+		if compareBuyer < "C" {
+			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		} else {
+			if pjVal > 1 && compareTime == "A" && score2 > 0 && (compareBudget == "A" || compareBidmount == "A") && compareCity[1:1] != "C" {
+				eqV = 3
+			}
+		}
+	}
+	return eqV
+}
+
+func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *ProjectCache) {
+	//pId := primitive.NewObjectID() //NewObjectId()
+	pId := mongodb.StringTOBsonId(thisinfo.Id)
+	set := map[string]interface{}{}
+	set["_id"] = pId
+	for _, f := range FIELDS {
+		if tmp[f] != nil && tmp[f] != "" {
+			set[f] = tmp[f]
+		}
+	}
+	bidopentime := util.Int64All(tmp["bidopentime"])
+	if bidopentime > 0 {
+		set["bidopentime"] = bidopentime
+	}
+	bidendtime := util.Int64All(tmp["bidendtime"])
+	if bidendtime > 0 {
+		set["bidendtime"] = bidendtime
+	}
+	//异常标记
+	if thisinfo.TopType != "招标" && thisinfo.TopType != "拟建" && thisinfo.TopType != "预告" {
+		set["exception"] = 1
+	}
+	//projecthref保存
+	if jsonData, ok := tmp["jsondata"].(map[string]interface{}); ok {
+		if jsonData != nil && util.ObjToString(jsonData["projecthref"]) != "" {
+			set["projecthref"] = jsonData["projecthref"]
+		}
+	}
+	//合同编号
+	if thisinfo.ContractCode != "" {
+		set["contractcode"] = thisinfo.ContractCode
+	}
+
+	bt := util.ObjToString(tmp["toptype"])
+	bs := util.ObjToString(tmp["subtype"])
+	p.mapBidLock.Lock()
+	if thisinfo.Infoformat == 2 || thisinfo.SubType == "拟建" {
+		set["bidstatus"] = "拟建"
+		bt = "拟建"
+	} else {
+		if bidtype[bs] != "" {
+			set["bidtype"] = bidtype[bs]
+		} else {
+			set["bidtype"] = "招标"
+		}
+		if bt == "招标" {
+			set["projectscope"] = util.ObjToString(tmp["projectscope"])
+			set["bidstatus"] = bt
+		} else {
+			if bidstatus[bs] != "" {
+				set["bidstatus"] = thisinfo.SubType
+				bt = thisinfo.SubType
+			} else if bs == "" {
+				set["bidstatus"] = ""
+				bt = ""
+			} else {
+				set["bidstatus"] = "其它"
+				bt = "其它"
+			}
+		}
+	}
+	p.mapBidLock.Unlock()
+
+	pkg := PackageFormat(thisinfo, nil)
+	p1 := p.NewCachePinfo(pId, thisinfo, bs, bt)
+
+	now := time.Now().Unix()
+	set["createtime"] = now
+	set["sourceinfoid"] = thisinfo.Id
+	set["sourceinfourl"] = tmp["href"]
+	set["firsttime"] = tmp["publishtime"]
+	set["lasttime"] = tmp["publishtime"]
+	//增量用系统时间,全量(历史)入库时间
+	if p.currentType == "pl" {
+		set["pici"] = tmp["comeintime"]
+	} else {
+		set["pici"] = p.pici
+	}
+	set["ids"] = []string{thisinfo.Id}
+	if thisinfo.TopType == "招标" {
+		//if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
+		set["zbtime"] = tmp["publishtime"]
+		//}
+	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
+		set["jgtime"] = tmp["publishtime"]
+	}
+	if len(thisinfo.Subscopeclass) > 0 {
+		set["s_subscopeclass"] = strings.Join(thisinfo.Subscopeclass, ",")
+	}
+	if len(thisinfo.Winners) > 0 {
+		set["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	}
+	if thisinfo.HasPackage {
+		set["multipackage"] = 1
+		set["package"] = pkg
+	} else {
+		set["multipackage"] = 0
+	}
+	//项目评审专家
+	if len(thisinfo.ReviewExperts) > 0 {
+		set["review_experts"] = thisinfo.ReviewExperts
+	}
+	//标的物
+	if thisinfo.Purchasing != "" {
+		list := Duplicate(strings.Split(thisinfo.Purchasing, ",")) //标的物 去重 03/03
+		p := strings.Join(util.ObjArrToStringArr(list), ",")
+		set["purchasing"] = p
+	}
+	//中标候选人
+	if len(thisinfo.WinnerOrder) > 0 {
+		var list = []string{}
+		for _, v := range thisinfo.WinnerOrder {
+			if BinarySearch(list, util.ObjToString(v["entname"])) == -1 {
+				list = append(list, util.ObjToString(v["entname"]))
+			}
+		}
+		set["winnerorder"] = list
+	}
+	//项目规模
+	if len(thisinfo.ProjectScale) > 0 {
+		set["project_scale"] = thisinfo.ProjectScale
+	}
+	//工期时长
+	if thisinfo.ProjectDuration > 0 {
+		set["project_duration"] = thisinfo.ProjectDuration
+	}
+	// 工期单位
+	if thisinfo.ProjectDuration > 0 && len(thisinfo.ProjectTimeUnit) > 0 {
+		set["project_timeunit"] = thisinfo.ProjectTimeUnit
+	}
+	//开工日期
+	if thisinfo.ProjectStartDate > 0 {
+		set["project_startdate"] = thisinfo.ProjectStartDate
+	}
+	//竣工日期
+	if thisinfo.ProjectCompleteDate > 0 {
+		set["project_completedate"] = thisinfo.ProjectCompleteDate
+	}
+	//付款方式
+	if len(thisinfo.Payway) > 0 {
+		set["payway"] = thisinfo.Payway
+	}
+	// 履约保证金
+	if tmp["contract_guarantee"] != nil {
+		set["contract_guarantee"] = thisinfo.ContractGuarantee
+	}
+	// 投标保证金
+	if tmp["bid_guarantee"] != nil {
+		set["bid_guarantee"] = thisinfo.BidGuarantee
+	}
+	// 资质条件
+	if len(thisinfo.Qualifies) > 0 {
+		var str []string
+		for _, v := range thisinfo.Qualifies {
+			if len(util.ObjToString(v["key"])) > 0 {
+				if BinarySearch(str, util.ObjToString(v["key"])) == -1 {
+					str = append(str, util.ObjToString(v["key"]))
+				}
+			}
+		}
+		if len(str) > 0 {
+			set["qualifies"] = strings.Join(str, ",")
+		}
+	}
+	if len(thisinfo.EntIdList) > 0 {
+		set["entidlist"] = thisinfo.EntIdList
+	}
+	// first_cooperation
+	if p1.Buyer != "" && len(thisinfo.Winners) > 0 {
+		FirstCooperation(set, p1.Buyer, thisinfo.Winners, thisinfo.EntIdList)
+	}
+
+	infofield := InfoField{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    bs,
+	}
+	res := StructToMap(infofield)
+	set["infofield"] = map[string]interface{}{
+		thisinfo.Id: res,
+	}
+	if tmp["budget"] != nil && tmp["budget"] != "" {
+		set["budget"] = thisinfo.Budget
+		set["budgettag"] = 0
+	} else {
+		set["budgettag"] = 1
+	}
+	if tmp["bidamount"] != nil && tmp["bidamount"] != "" {
+		set["bidamount"] = thisinfo.Bidamount
+		set["bidamounttag"] = 0
+	} else {
+		set["bidamounttag"] = 1
+	}
+	if p1.Bidamount > 0 {
+		set["sortprice"] = p1.Bidamount
+	} else {
+		if p1.Budget > 0 {
+			set["sortprice"] = p1.Budget
+		}
+	}
+	if tmp["bid_field"] != nil {
+		set["bid_field"] = tmp["bid_field"]
+	}
+	redis.PutCKV("project", thisinfo.Id, set)
+	push := p.PushListInfo(tmp, thisinfo.Id)
+	if len(thisinfo.Winners) > 0 {
+		push["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	}
+	set["list"] = []bson.M{
+		push,
+	}
+	p.updatePool <- []map[string]interface{}{
+		{
+			"_id": pId,
+		},
+		{
+			"$set": set,
+		},
+	}
+	return pId.Hex(), &p1
+}
+
+//项目中list的信息
+func (p *ProjectTask) PushListInfo(tmp map[string]interface{}, infoid string) bson.M {
+	res := bson.M{
+		"infoid": infoid,
+	}
+	for _, k := range INFOFIELDS {
+		if tmp[k] != nil {
+			res[k] = tmp[k]
+		}
+	}
+	return res
+}
+
+//生成存放在内存中的对象
+func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidtype, bidstatus string) ProjectCache {
+	p1 := ProjectCache{
+		Id:          id,
+		Ids:         []string{thisinfo.Id},
+		ProjectName: thisinfo.ProjectName,
+		ProjectCode: thisinfo.ProjectCode,
+		Buyer:       thisinfo.Buyer,
+		Agency:      thisinfo.Agency,
+		Area:        thisinfo.Area,
+		City:        thisinfo.City,
+		District:    thisinfo.District,
+		MPN:         []string{},
+		MPC:         []string{},
+		FirstTime:   thisinfo.Publishtime,
+		LastTime:    thisinfo.Publishtime,
+		Budget:      thisinfo.Budget,
+		Bidamount:   thisinfo.Bidamount,
+		Bidstatus:   bidstatus,
+		Bidtype:     bidtype,
+	}
+	if thisinfo.LenPTC > 5 {
+		p1.MPC = append(p1.MPC, thisinfo.PTC)
+	}
+	return p1
+}
+
+// @Description 更新项目	全量合并
+// @update 中间表查询需要更新的项目信息
+// @Author J 2022/7/6 14:02
+func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectCache, weight int, comStr string, ex int) {
+	if p.currentType != "updateInfo" {
+		if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
+			util.Debug("repeat", thisinfo.Id, ",pid", pInfo.Id)
+			return
+		}
+	}
+	pdata := redis.Get("project", pInfo.Id.Hex())
+	if pdata == nil {
+		util.Debug("redis err, not find project ", pInfo.Id.Hex())
+		return
+	}
+	projectMap := pdata.(map[string]interface{})
+	bys, _ := json.Marshal(projectMap)
+	var project *Project
+	err := json.Unmarshal(bys, &project)
+	if err != nil {
+		util.Debug("project Unmarshal err,", err)
+		return
+	}
+	set := map[string]interface{}{}
+	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
+	if len(pInfo.Ids) > 30 {
+		//异常标记
+		set["listtag"] = 1
+	}
+
+	//zbtime、lasttime、jgtime
+	pInfo.LastTime = thisinfo.Publishtime
+	set["lasttime"] = thisinfo.Publishtime
+
+	if thisinfo.TopType == "招标" {
+		if project.Zbtime <= 0 {
+			project.Zbtime = thisinfo.Publishtime
+			set["zbtime"] = tmp["publishtime"]
+		}
+		if project.Jgtime > 0 {
+			project.Jgtime = int64(0)
+			set["jgtime"] = int64(0)
+		}
+	} else if thisinfo.TopType == "结果" {
+		if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+			if project.Jgtime > 0 {
+				//jg1 := int64(math.Abs(float64(pInfo.Jgtime - thisinfo.Publishtime)))
+				//公告状态和项目状态同样都是中标或者成交,
+				if thisinfo.SubType == "成交" && project.Bidstatus == "中标" {
+					if p.jgTime < thisinfo.Publishtime {
+						project.Jgtime = thisinfo.Publishtime
+						set["jgtime"] = tmp["publishtime"]
+					}
+					//公告状态和项目状态同样是流标或者废标
+				} else if (thisinfo.SubType == "流标" || thisinfo.SubType == "废标") && (project.Bidstatus == "流标" || project.Bidstatus == "废标") {
+					if p.jgTime < thisinfo.Publishtime {
+						project.Jgtime = thisinfo.Publishtime
+						set["jgtime"] = tmp["publishtime"]
+					}
+				}
+			} else {
+				project.Jgtime = thisinfo.Publishtime
+				set["jgtime"] = tmp["publishtime"]
+			}
+		}
+	} else if thisinfo.SubType == "合同" {
+		if project.Bidstatus == "中标" || project.Bidstatus == "成交" || project.Bidstatus == "" {
+			//中标、成交不更新jgtime
+		} else {
+			project.Jgtime = thisinfo.Publishtime
+			set["jgtime"] = tmp["publishtime"]
+		}
+	}
+	if thisinfo.BidOpenTime > project.BidOpenTime {
+		project.BidOpenTime = thisinfo.BidOpenTime
+		set["bidopentime"] = project.BidOpenTime
+	}
+	if thisinfo.BidEndTime > 0 {
+		project.BidEndTime = thisinfo.BidEndTime
+		set["bidendtime"] = project.BidEndTime
+	}
+
+	bt := util.ObjToString(tmp["toptype"])
+	bs := util.ObjToString(tmp["subtype"])
+	p.mapBidLock.Lock()
+	if bt == "招标" {
+		//招标状态,更新projectscope
+		if tmp["projectscope"] != nil {
+			set["projectscope"] = util.ObjToString(tmp["projectscope"])
+		}
+		set["bidstatus"] = bt
+		project.Bidstatus = bt
+		if bidtype[bs] != "" {
+			set["bidtype"] = bidtype[bs]
+			project.Bidtype = bidtype[bs]
+		} else {
+			set["bidtype"] = "招标"
+			project.Bidtype = "招标"
+		}
+	} else {
+		if bidstatus[bs] != "" {
+			set["bidstatus"] = thisinfo.SubType
+			project.Bidstatus = thisinfo.SubType
+		} else if thisinfo.Infoformat == 2 {
+			set["bidstatus"] = "拟建"
+			project.Bidstatus = "拟建"
+		} else if bs == "" && bt == "结果" {
+			if project.Bidstatus == "招标" {
+				set["bidstatus"] = ""
+				project.Bidstatus = ""
+			}
+		} else {
+			set["bidstatus"] = "其它"
+			project.Bidstatus = "其它"
+		}
+	}
+	p.mapBidLock.Unlock()
+
+	//异常标记
+	if ex > 0 {
+		set["exception"] = ex
+	}
+	//3\4\5--省、市、县
+	//if thisinfo.Area != "全国" {
+	//	if pInfo.Area == "全国" {
+	//		pInfo.Area = thisinfo.Area
+	//		set["area"] = thisinfo.Area
+	//	} else if pInfo.Area != thisinfo.Area {
+	//		//xt = false
+	//	}
+	//	if pInfo.City == "" && thisinfo.City != "" {
+	//		pInfo.City = thisinfo.City
+	//		set["city"] = thisinfo.City
+	//	} else if pInfo.City != thisinfo.City {
+	//		//xt = false
+	//	}
+	//	if thisinfo.District != "" && pInfo.District == "" {
+	//		pInfo.District = thisinfo.District
+	//		set["district"] = thisinfo.District
+	//	}
+	//}
+	//相同城市的公告才会合并到一起(全国列外)
+	if thisinfo.Area != "全国" {
+		pInfo.Area = thisinfo.Area
+		project.Area = thisinfo.Area
+		set["area"] = thisinfo.Area
+		if thisinfo.City != "" {
+			pInfo.City = thisinfo.City
+			project.City = thisinfo.City
+			set["city"] = thisinfo.City
+		}
+		if thisinfo.District != "" {
+			pInfo.District = thisinfo.District
+			project.District = thisinfo.District
+			set["district"] = thisinfo.District
+		}
+	}
+	//项目名称
+	//if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
+	if pInfo.ProjectName == "" && thisinfo.ProjectName != "" {
+		pInfo.ProjectName = thisinfo.ProjectName
+		project.ProjectName = thisinfo.ProjectName
+		set["projectname"] = thisinfo.ProjectName
+	}
+	//7--项目编号
+	//if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+	if pInfo.ProjectCode == "" && thisinfo.ProjectCode != "" {
+		pInfo.ProjectCode = thisinfo.ProjectCode
+		project.ProjectCode = thisinfo.ProjectCode
+		set["projectcode"] = thisinfo.ProjectCode
+	}
+	//7--采购单位
+	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
+		pInfo.Buyer = thisinfo.Buyer
+		project.Buyer = thisinfo.Buyer
+		set["buyer"] = thisinfo.Buyer
+
+		project.Buyerclass = thisinfo.Buyerclass
+		set["buyerclass"] = thisinfo.Buyerclass
+	}
+	if pInfo.Buyer == "" {
+		project.Buyerclass = ""
+		set["buyerclass"] = ""
+	}
+	//采购单位联系人
+	if thisinfo.Buyerperson != "" {
+		project.Buyerperson = thisinfo.Buyerperson
+		set["buyerperson"] = thisinfo.Buyerperson
+	} else {
+		project.Buyerperson = ""
+		set["buyerperson"] = ""
+	}
+	//采购单位電話
+	if thisinfo.Buyertel != "" {
+		project.Buyertel = thisinfo.Buyertel
+		set["buyertel"] = project.Buyertel
+	} else {
+		project.Buyertel = ""
+		set["buyertel"] = ""
+	}
+	if thisinfo.ContractCode != "" {
+		if project.ContractCode == "" {
+			set["contractcode"] = thisinfo.ContractCode
+		} else {
+			list := strings.Split(project.ContractCode, ",")
+			if BinarySearch(list, thisinfo.ContractCode) == -1 {
+				list = append(list, thisinfo.ContractCode)
+				sort.Strings(list)
+			}
+			set["contractcode"] = strings.Join(list, ",")
+		}
+	}
+
+	//8--代理机构
+	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
+		pInfo.Agency = thisinfo.Agency
+		project.Agency = thisinfo.Agency
+		set["agency"] = thisinfo.Agency
+	}
+
+	if len(thisinfo.Topscopeclass) > 0 {
+		sort.Strings(project.Topscopeclass)
+		for _, k := range thisinfo.Topscopeclass {
+			if BinarySearch(project.Topscopeclass, k) == -1 {
+				project.Topscopeclass = append(project.Topscopeclass, k)
+				sort.Strings(project.Topscopeclass)
+			}
+		}
+		set["topscopeclass"] = project.Topscopeclass
+	}
+
+	//项目评审专家
+	if len(thisinfo.ReviewExperts) > 0 {
+		set["review_experts"] = thisinfo.ReviewExperts
+		project.ReviewExperts = thisinfo.ReviewExperts
+	}
+	if thisinfo.Purchasing != "" {
+		if project.Purchasing == "" {
+			list := Duplicate(strings.Split(thisinfo.Purchasing, ",")) //标的物 去重 03/03
+			p := strings.Join(util.ObjArrToStringArr(list), ",")
+			project.Purchasing = p
+			set["purchasing"] = p
+		} else {
+			list := strings.Split(project.Purchasing, ",")
+			list = util.ObjArrToStringArr(Duplicate(list))
+			for _, k := range list {
+				if BinarySearch(strings.Split(thisinfo.Purchasing, ","), k) == -1 {
+					list = append(list, k)
+					sort.Strings(list)
+				}
+			}
+			set["purchasing"] = strings.Join(list, ",")
+		}
+	}
+
+	//中标候选人
+	if len(thisinfo.WinnerOrder) > 0 {
+		var list = []string{}
+		for _, v := range thisinfo.WinnerOrder {
+			if BinarySearch(list, util.ObjToString(v["entname"])) == -1 {
+				list = append(list, util.ObjToString(v["entname"]))
+			}
+		}
+		set["winnerorder"] = list
+		project.Winnerorder = list
+	}
+
+	if len(thisinfo.Subscopeclass) > 0 {
+		sort.Strings(project.Subscopeclass)
+		for _, k := range thisinfo.Subscopeclass {
+			if BinarySearch(project.Subscopeclass, k) == -1 {
+				project.Subscopeclass = append(project.Subscopeclass, k)
+				sort.Strings(project.Subscopeclass)
+			}
+		}
+		set["subscopeclass"] = project.Subscopeclass
+		set["s_subscopeclass"] = strings.Join(project.Subscopeclass, ",")
+	}
+
+	if len(thisinfo.Winners) > 0 {
+		var winners []string
+		if project.Winners == "" {
+			set["winner"] = util.ObjToString(tmp["winner"])
+			for _, k := range thisinfo.Winners {
+				if k == "" {
+					continue
+				}
+				if BinarySearch(winners, k) == -1 {
+					winners = append(winners, k)
+				}
+			}
+		} else {
+			winners = strings.Split(project.Winners, ",")
+			for _, k := range thisinfo.Winners {
+				if k == "" {
+					continue
+				}
+				if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+					if BinarySearch(winners, k) != -1 {
+						deleteSlice(winners, k, "")
+					}
+				} else {
+					if BinarySearch(winners, k) == -1 {
+						winners = append(winners, k)
+					}
+				}
+			}
+		}
+		if len(winners) > 0 {
+			set["s_winner"] = strings.Join(util.ObjArrToStringArr(Duplicate(winners)), ",")
+		}
+
+	}
+
+	if len(thisinfo.EntIdList) > 0 {
+		for _, v := range thisinfo.EntIdList {
+			if BinarySearch(project.EntIdList, v) == -1 {
+				project.EntIdList = append(project.EntIdList, v)
+			}
+		}
+		set["entidlist"] = project.EntIdList
+	}
+	//first_cooperation
+	if pInfo.Buyer != "" && len(strings.Split(project.Winners, ",")) > 0 {
+		FirstCooperation(set, pInfo.Buyer, strings.Split(project.Winners, ","), project.EntIdList)
+	}
+
+	//项目规模
+	if len(thisinfo.ProjectScale) > 0 {
+		project.ProjectScale = thisinfo.ProjectScale
+		set["project_scale"] = thisinfo.ProjectScale
+	}
+	//工期时长
+	if thisinfo.ProjectDuration > 0 {
+		project.ProjectDuration = thisinfo.ProjectDuration
+		set["project_duration"] = thisinfo.ProjectDuration
+	}
+	// 工期单位
+	if thisinfo.ProjectDuration > 0 && len(thisinfo.ProjectTimeUnit) > 0 {
+		project.ProjectTimeunit = thisinfo.ProjectTimeUnit
+		set["project_timeunit"] = thisinfo.ProjectTimeUnit
+	}
+	//开工日期
+	if thisinfo.ProjectStartDate > 0 {
+		if project.ProjectStartDate > 0 {
+			if project.ProjectStartDate < thisinfo.ProjectStartDate {
+				project.ProjectStartDate = thisinfo.ProjectStartDate
+				set["project_startdate"] = thisinfo.ProjectStartDate
+			}
+		} else {
+			project.ProjectStartDate = thisinfo.ProjectStartDate
+			set["project_startdate"] = thisinfo.ProjectStartDate
+		}
+	}
+	//竣工日期
+	if thisinfo.ProjectCompleteDate > 0 {
+		if project.ProjctCompleteDate > 0 {
+			if project.ProjctCompleteDate < thisinfo.ProjectCompleteDate {
+				project.ProjctCompleteDate = thisinfo.ProjectCompleteDate
+				set["project_completedate"] = thisinfo.ProjectCompleteDate
+			}
+		} else {
+			project.ProjctCompleteDate = thisinfo.ProjectCompleteDate
+			set["project_completedate"] = thisinfo.ProjectCompleteDate
+		}
+	}
+	// 付款方式
+	if len(thisinfo.Payway) > 0 {
+		project.Payway = thisinfo.Payway
+		set["payway"] = thisinfo.Payway
+	}
+	// 履约保证金
+	if tmp["contract_guarantee"] != nil {
+		project.ContractGuarantee = thisinfo.ContractGuarantee
+		set["contract_guarantee"] = thisinfo.ContractGuarantee
+	}
+	// 投标保证金
+	if tmp["bid_guarantee"] != nil {
+		project.BidGuarantee = thisinfo.BidGuarantee
+		set["bid_guarantee"] = thisinfo.BidGuarantee
+	}
+	// 资质条件
+	if len(thisinfo.Qualifies) > 0 {
+		var str []string
+		if len(project.Qualifies) > 0 {
+			str = append(str, strings.Split(project.Qualifies, ",")...)
+		}
+		for _, v := range thisinfo.Qualifies {
+			if len(util.ObjToString(v["key"])) > 0 {
+				if BinarySearch(str, util.ObjToString(v["key"])) == -1 {
+					str = append(str, util.ObjToString(v["key"]))
+				}
+			}
+		}
+		if len(str) > 0 {
+			project.Qualifies = strings.Join(str, ",")
+			set["qualifies"] = strings.Join(str, ",")
+		}
+	}
+
+	if thisinfo.HasPackage { //多包处理
+		set["multipackage"] = 1
+		pkg := PackageFormat(thisinfo, project)
+		project.Package = pkg
+		set["package"] = project.Package
+	}
+	//处理多包后,计算预算金额、中标金额
+	CountAmount(project, thisinfo, tmp)
+	if project.Budget >= 0 && project.Budgettag != 1 {
+		pInfo.Budget = project.Budget
+		set["budget"] = project.Budget
+		set["budgettag"] = 0
+	}
+	if project.Bidamount >= 0 && project.Bidamounttag != 1 {
+		pInfo.Bidamount = project.Bidamount
+		set["bidamount"] = pInfo.Bidamount
+		set["bidamounttag"] = 0
+	}
+	if pInfo.Bidamount > 0 {
+		set["sortprice"] = pInfo.Bidamount
+	} else {
+		if pInfo.Budget > 0 {
+			set["sortprice"] = pInfo.Budget
+		}
+	}
+	if tmp["bid_field"] != nil {
+		set["bid_field"] = tmp["bid_field"]
+	}
+	infofield := InfoField{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    bs,
+	}
+	copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
+	copyMap[thisinfo.Id] = infofield
+	tmpMap := make(map[string]interface{})
+	for k, v := range copyMap {
+		tmpMap[k] = StructToMap(v)
+	}
+	tmpMap[thisinfo.Id] = StructToMap(infofield)
+	project.InfoFiled = copyMap
+	set["infofield"] = tmpMap
+
+	set["mpn"] = pInfo.MPN
+	set["mpc"] = pInfo.MPC
+	if p.currentType == "project" || p.currentType == "project_history" {
+		set["pici"] = p.pici
+	} else {
+		set["pici"] = tmp["comeintime"]
+	}
+	redis.PutCKV("project", project.Id.Hex(), set)
+	update := map[string]interface{}{}
+	if len(set) > 0 {
+		update["$set"] = set
+	}
+	push := p.PushListInfo(tmp, thisinfo.Id)
+	if len(thisinfo.Winners) > 0 {
+		push["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	}
+	push["compareStr"] = comStr
+	push["resVal"] = pInfo.resVal
+	push["pjVal"] = pInfo.pjVal
+	update["$push"] = map[string]interface{}{
+		"list": push,
+		"ids":  thisinfo.Id,
+	}
+	//clearMap := map[string]interface{}{}
+	//ClearData(clearMap, set)
+	//if len(clearMap) > 0 {
+	//	update["$unset"] = clearMap
+	//}
+	if len(update) > 0 {
+		updateInfo := []map[string]interface{}{
+			{
+				"_id": pInfo.Id,
+			},
+			update,
+		}
+		p.updatePool <- updateInfo
+	}
+}
+
+/**
+ *	更新项目时,项目状态的处理
+ *	返回是否新增项目,异常标记
+ *	1、新增项目时,招标信息的状态(toptype)不是招标、拟建、预告	异常:1
+ *	   异常1是在项目新建的时候才会产生
+ *	3、项目合并时,项目状态是”流标“/”废标“,招标信息状态不是”招标“		异常:2
+ *	4、项目合并时,项目状态是”合同“/”其它“,招标信息类型是”结果“		异常:3
+ */
+func (p *ProjectTask) CompareStatus(project *ProjectCache, info *Info) (bool, int) {
+	if info.TopType == "拟建" || info.TopType == "预告" || info.TopType == "招标" {
+		if project.Bidstatus == "拟建" || project.Bidstatus == "预告" || project.Bidstatus == "招标" {
+			return false, 0
+		} else if project.Bidstatus == "废标" || project.Bidstatus == "流标" {
+			return false, 0
+		} else {
+			return true, 0
+		}
+	} else if info.TopType == "结果" {
+		if project.Bidstatus == "拟建" || project.Bidstatus == "预告" || project.Bidstatus == "招标" {
+			return false, 0
+		} else if project.Bidstatus == info.SubType {
+			//状态一样,根据发布时间判断是否合并
+			if (info.Publishtime - project.LastTime) > p.statusTime {
+				return true, 0
+			} else {
+				return false, 0
+			}
+		} else if project.Bidstatus == "成交" && info.SubType == "中标" {
+			return true, 0
+		} else if project.Bidstatus == "流标" || project.Bidstatus == "废标" {
+			return false, 2
+		} else if project.Bidstatus == "合同" || project.Bidstatus == "其它" {
+			return false, 3
+		} else {
+			return false, 0
+		}
+	} else {
+		return false, 0
+	}
+}
+
+/*
+ *	对比地区(省、市),存在且不同,不能合并
+ *	返回是否新建项目
+ */
+func ComparePlace(project *ProjectCache, info *Info) bool {
+	if info.Area == "全国" || info.Area == "" {
+		return false
+	}
+	if project.Area == "全国" || project.Area == "" {
+		return false
+	}
+	if info.Area == project.Area {
+		if info.City == "" || project.City == "" {
+			return false
+		} else if info.City == project.City {
+			return false
+		}
+	} else {
+		return true
+	}
+	return true
+}
+
+func packageEle(map1 map[string]interface{}, id string) map[string]interface{} {
+	p2 := map[string]interface{}{}
+	for _, k := range PackageEle {
+		if map1[k] != nil {
+			p2[k] = map1[k]
+		}
+		infoid := p2["infoid"]
+		if infoid == nil {
+			p2["infoid"] = id
+		}
+	}
+	return p2
+}
+
+func PackageFormat(info *Info, project *Project) map[string]interface{} {
+	p1 := map[string]interface{}{}
+	if project != nil && project.Package != nil && len(project.Package) > 0 {
+		packageCopy := util.DeepCopy(project.Package).(map[string]interface{})
+		p1 = packageCopy
+		for k, v := range info.Package {
+			if v1, ok := v.(map[string]interface{}); ok {
+				v2 := map[string]interface{}{}
+				v2 = packageEle(v1, info.Id)
+				if v2["bidstatus"] == nil {
+					v2["bidstatus"] = info.SubType
+				}
+				addFlag := false
+				for k1, v3 := range p1 {
+					if v4, ok := v3.([]map[string]interface{}); ok {
+						//if qu.ObjToString(v4[0]["origin"]) == qu.ObjToString(v2["origin"]) && qu.ObjToString(v4[0]["name"]) == qu.ObjToString(v2["name"]) {
+						if k1 == k {
+							v4 = append(v4, v2)
+							p1[k1] = v4
+							addFlag = true
+							break
+						}
+					}
+				}
+				if !addFlag {
+					p1[k] = []map[string]interface{}{v2}
+				}
+			}
+		}
+	} else {
+		for k, v := range info.Package {
+			v1, _ := v.(map[string]interface{})
+			p2 := map[string]interface{}{}
+			p2 = packageEle(v1, info.Id)
+			if p2["bidstatus"] == nil {
+				p2["bidstatus"] = info.SubType
+			}
+			p1[k] = []map[string]interface{}{p2}
+		}
+	}
+	return p1
+}
+
+// 计算预算(budget)、中标金额(bidamount)
+func CountAmount(project *Project, info *Info, tmp map[string]interface{}) {
+	if info.HasPackage {
+		budget := 0.0
+		for _, v := range project.Package {
+			v1, _ := v.([]map[string]interface{})
+			for _, v2 := range v1 {
+				if v2["budget"] != nil {
+					b1 := util.Float64All(v2["budget"])
+					if b1 > 0 {
+						budget = budget + b1
+						break
+					}
+				} else {
+					project.Budgettag = 1
+				}
+			}
+		}
+		if budget > 0 {
+			project.Budget = budget
+			project.Budgettag = 0
+		} else if budget == 0 && info.Budget > 0 {
+			project.Budget = info.Budget
+			project.Budgettag = 0
+		}
+	} else {
+		//招标没有多包
+		k := KeyPackage.FindStringSubmatch(info.ProjectName)
+		if len(k) > 0 {
+			//招标是单包
+			if len(project.Package) > 0 {
+				//项目有多包
+				flag := false
+				for _, v := range project.Package {
+					v1, _ := v.([]map[string]interface{})
+					if len(v1) > 0 && v1[0]["name"] == info.ProjectName {
+						flag = true
+					}
+				}
+				if !flag {
+					project.Budget = project.Budget + info.Budget
+					project.Budgettag = 0
+				}
+			} else {
+				//项目没有多包
+				if info.Budget > 0 {
+					project.Budget = project.Budget + info.Budget
+					project.Budgettag = 0
+				} else if info.Budget == 0 && tmp["budget"] != nil {
+					project.Budgettag = 0
+				}
+			}
+		} else {
+			if info.Budget > 0 && project.Budget < info.Budget {
+				project.Budget = info.Budget
+				project.Budgettag = 0
+			}
+		}
+	}
+	if info.SubType == "中标" || info.SubType == "成交" || info.SubType == "合同" {
+		if info.HasPackage {
+			bidamount := 0.0
+			for _, v := range project.Package {
+				v1, _ := v.([]map[string]interface{})
+				for _, v2 := range v1 {
+					b1 := util.Float64All(v2["bidamount"])
+					if b1 > 0 {
+						bidamount = bidamount + b1
+						break
+					}
+				}
+			}
+			if bidamount > 0 {
+				project.Bidamount = bidamount
+				project.Bidamounttag = 0
+			} else if bidamount == 0 && info.Bidamount > 0 {
+				project.Bidamount = info.Bidamount
+				project.Bidamounttag = 0
+			}
+		} else {
+			//招标没有多包
+			k := KeyPackage.FindStringSubmatch(info.ProjectName)
+			if len(k) > 0 {
+				//招标是单包
+				if len(project.Package) > 0 {
+					//项目有多包
+					flag := false
+					for _, v := range project.Package {
+						v1, _ := v.([]map[string]interface{})
+						if len(v1) > 0 && v1[0]["name"] == info.ProjectName {
+							flag = true
+						}
+					}
+					if !flag {
+						project.Bidamount = project.Bidamount + info.Bidamount
+						project.Bidamounttag = 0
+					}
+				} else {
+					//项目没有多包
+					if info.Bidamount > 0 {
+						project.Bidamount = project.Bidamount + info.Bidamount
+						project.Bidamounttag = 0
+					} else if info.Bidamount == 0 && tmp["bidamount"] != nil {
+						project.Bidamounttag = 0
+					} else {
+						project.Bidamounttag = 1
+					}
+				}
+			} else {
+				if info.SubType == "中标" || info.SubType == "成交" {
+					if info.Bidamount > 0 {
+						project.Bidamount = info.Bidamount
+						project.Bidamounttag = 0
+					} else {
+						flag := false
+						if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
+							for _, res := range project.InfoFiled {
+								if res.ContractCode != "" && res.ContractCode == info.ContractCode {
+									flag = true
+									break
+								}
+								if res.Bidamount == project.Bidamount {
+									flag = true
+									break
+								}
+							}
+							if !flag {
+								project.Bidamount = project.Bidamount + info.Bidamount
+								project.Bidamounttag = 0
+							} else {
+								if info.Budget > 0 && project.Bidamount > info.Bidamount {
+									project.Bidamount = info.Bidamount
+									project.Bidamounttag = 0
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	} else {
+		project.Bidamounttag = 1
+	}
+
+}
+
+//结构体转map
+func StructToMap(filed InfoField) map[string]interface{} {
+	//先转json
+	result, err := json.Marshal(filed)
+	if err != nil {
+		return nil
+	}
+	//json转map
+	res := make(map[string]interface{})
+	err = json.Unmarshal(result, &res)
+	return res
+}
+
+func ClearData(clearMap, tmp map[string]interface{}) {
+	for k, v := range tmp {
+		if v == "" {
+			clearMap[k] = ""
+		}
+	}
+}
+
+func IsCreatePro(info *Info) (bol bool) {
+	bol = true
+	if info.SubType == "" || info.SubType == "变更" || info.SubType == "验收" || info.SubType == "违规" ||
+		info.SubType == "结果变更" || info.SubType == "其它" {
+		if info.ProjectName == "" && info.ProjectCode == "" {
+			bol = false
+		} else if info.ProjectName == "" && info.Buyer == "" {
+			bol = false
+		} else if info.ProjectCode == "" && info.Buyer == "" {
+			bol = false
+		}
+	}
+	return bol
+}
+
+func (p *ProjectTask) GetBidTypeAndBidStatus(info *Info) (string, string) {
+	p.mapBidLock.Lock()
+	defer p.mapBidLock.Unlock()
+	typeStr := bidtype[info.TopType]
+	statusStr := bidstatus[info.SubType]
+	if info.Infoformat == 2 || info.SubType == "拟建" {
+		statusStr = "拟建"
+		typeStr = ""
+	} else {
+		if bidtype[typeStr] == "" {
+			typeStr = "招标"
+		}
+		if typeStr == "招标" {
+			statusStr = typeStr
+		} else {
+			if statusStr == "" {
+				statusStr = "其它"
+			}
+		}
+	}
+	return typeStr, statusStr
+}
+
+func FirstCooperation(set map[string]interface{}, b string, winns, entidlist []string) {
+	defer func() {
+		// 处理数组越界异常
+		if r := recover(); r != nil {
+			//qu.Debug("recover...:", r)
+		}
+	}()
+	pid := mongodb.BsonIdToSId(set["_id"])
+	var eids []string
+	for i, eid := range entidlist {
+		if eid != "-" {
+			text := b + "," + winns[i]
+			ex, _ := redis.Exists(RedisCode, text)
+			if !ex {
+				redis.PutCKV(RedisCode, text, pid)
+				eids = append(eids, eid)
+			}
+		}
+	}
+	if len(eids) > 0 {
+		set["first_cooperation"] = entidlist
+	}
+}

+ 99 - 0
data_project/project_tool.go

@@ -0,0 +1,99 @@
+package main
+
+var DateTimeSelect = []string{"bidopentime", "bidendtime", "signaturedate", "comeintime"}
+
+//项目中的字段
+var FIELDS = []string{
+	"area",
+	"city",
+	"district",
+	"projectname",
+	"projectcode",
+	"buyer",
+	"buyerclass",
+	"buyerperson",
+	"buyertel",
+	"winner",
+	"agency",
+	"topscopeclass",
+	"subscopeclass",
+	"package",
+}
+
+//招标信息字段
+var INFOFIELDS = []string{
+	"projectname",
+	"projectcode",
+	"projectscope",
+	"contractcode",
+	"title",
+	"href",
+	"publishtime",
+	"comeintime",
+	"bidopentime",
+	"toptype",
+	"subtype",
+	"buyer",
+	"buyerclass",
+	"agency",
+	"winner",
+	"winnertel",
+	"winnerperson",
+	"budget",
+	"bidamount",
+	"topscopeclass",
+	"subscopeclass",
+	"infoformat",
+	"buyerperson",
+	"buyertel",
+	"area",
+	"city",
+	"district",
+	"spidercode",
+	"site",
+	"review_experts",
+	"purchasing",
+	"project_scale",
+	"project_duration",
+	"project_timeunit",
+	"project_startdate",
+	"project_completedate",
+	"payway",
+	"contract_guarantee",
+	"bid_guarantee",
+	"qualifies",
+	"entidlist",
+	"winnerorder",
+	"purchasinglist",
+}
+
+// 包信息
+var PackageEle = []string{
+	"origin",
+	"name",
+	//"text",
+	"budget",
+	"winner",
+	"bidamount",
+	"bidamounttype",
+	"currency",
+	"bidstatus",
+}
+
+var bidstatus = map[string]string{
+	"预告": "预告",
+	"中标": "中标",
+	"成交": "成交",
+	"废标": "废标",
+	"流标": "流标",
+	"合同": "合同",
+}
+
+var bidtype = map[string]string{
+	"招标": "招标",
+	"邀标": "邀标",
+	"询价": "询价",
+	"单一": "单一",
+	"竞价": "竞价",
+	"竞谈": "竞谈",
+}

+ 940 - 0
data_project/task.go

@@ -0,0 +1,940 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/redis"
+	"app.yhyue.com/data_processing/common_utils/udp"
+	"encoding/json"
+	"fmt"
+	"github.com/goinggo/mapstructure"
+	"github.com/robfig/cron"
+	"go.mongodb.org/mongo-driver/bson"
+	"log"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+	"unicode/utf8"
+)
+
+/**
+任务入口
+全量、增量合并
+更新、插入,内存清理
+转换成info对象
+**/
+
+//var PreRegexp = map[string][]*regexp.Regexp{}
+//var BackRegexp = map[string][]*regexp.Regexp{}
+//var BackRepRegexp = map[string][]RegexpInfo{}
+var BlackRegexp = map[string][]*regexp.Regexp{}
+
+var (
+	//从标题获取项目编号
+	titleGetPc  = regexp.MustCompile("^([-0-9a-zA-Z第号采招政询电审竞#]{8,}[-0-9a-zA-Z#]+)")
+	titleGetPc1 = regexp.MustCompile("[\\[【((](.{0,6}(编号|编码|项号|包号|代码|标段?号)[::为])?([-0-9a-zA-Z第号采招政询电审竞#]{5,}([\\[\\]()()][-0-9a-zA-Z第号采招审竞#]+[\\[\\]()()][-0-9a-zA-Z第号采招审竞#]+)?)[\\]】))]")
+	titleGetPc2 = regexp.MustCompile("([-0-9a-zA-Z第号采政招询电审竞#]{8,}[-0-9a-zA-Z#]+)(.{0,5}公告)?$")
+	//项目编号过滤
+	pcReplace = regexp.MustCompile("([\\[【((〖〔《{﹝{](重|第?[二三四再]次.{0,4})[\\]】))〗〕》}﹞}])$|[\\[\\]【】()()〖〗〔〕《》{}﹝﹞-;{}–  ]+|(号|重|第?[二三四五再]次(招标)?)$|[ __]+|((采购)?项目|采购(项目)?)$")
+	//项目编号只是数字或只是字母4个以下
+	StrOrNum = regexp.MustCompile("^[0-9_-]{1,4}$|^[a-zA-Z_-]{1,4}$")
+	//纯数字或纯字母
+	StrOrNum2 = regexp.MustCompile("^[0-9_-]+$|^[a-zA-Z_-]+$")
+	//含分包词,招标未识别分包  合并到一个项目
+	KeyPackage = regexp.MustCompile("[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}(包|段)|(包|段)[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}")
+)
+
+type RegexpInfo struct {
+	regs   *regexp.Regexp
+	repstr string
+}
+
+//项目合并对象
+type ProjectTask struct {
+	InitMinTime int64 //最小时间,小于0的处理一次
+	name        string
+	thread      int //线程数
+	//查找锁
+	findLock sync.Mutex
+	wg       sync.WaitGroup
+	//map锁
+	AllIdsMapLock sync.Mutex
+	//对应的id
+	AllIdsMap map[string]*ID
+	//采购单位、项目名称、项目编号
+	mapPb, mapPn, mapPc map[string]*Key
+	//流程数据  字段相同,直接合并
+	mapHref     map[string]string
+	mapHrefLock sync.Mutex
+	//站点
+	mapSite     map[string]*Site
+	mapSiteLock sync.Mutex
+	//spider isflow
+	mapSpider     map[string]int
+	mapSpiderLock sync.Mutex
+	//bidtype、bidstatus 锁
+	mapBidLock sync.Mutex
+	//更新或新增通道
+	updatePool chan []map[string]interface{}
+	//savePool   chan map[string]interface{}
+	//saveSign, updateSign chan bool
+	//表名
+	coll string
+	//当前状态是全量还是增量
+	currentType string //当前是跑全量还是跑增量
+	//
+	clearContimes int
+	//当前时间
+	currentTime int64
+	//保存长度
+	saveSize   int
+	pici       int64
+	validTime  int64
+	statusTime int64
+	//结果时间的更新		最近两天的公告不再更新jgtime
+	jgTime int64
+	Brun   bool
+}
+
+func NewPT() *ProjectTask {
+	p := &ProjectTask{
+		InitMinTime: int64(1325347200),
+		name:        "全/增量对象",
+		thread:      Thread,
+		updatePool:  make(chan []map[string]interface{}, 5000),
+		//savePool:    make(chan map[string]interface{}, 2000),
+		wg:        sync.WaitGroup{},
+		AllIdsMap: make(map[string]*ID, 5000000),
+		mapPb:     make(map[string]*Key, 1500000),
+		mapPn:     make(map[string]*Key, 5000000),
+		mapPc:     make(map[string]*Key, 5000000),
+		mapHref:   make(map[string]string, 1500000),
+		mapSite:   make(map[string]*Site, 1000000),
+		mapSpider: make(map[string]int, 1000000),
+		saveSize:  200,
+
+		//saveSign:   make(chan bool, 1),
+		//updateSign: make(chan bool, 1),
+		coll:       ProjectColl,
+		validTime:  int64(util.IntAllDef(Sysconfig["validdays"], 150) * 86400),
+		statusTime: int64(util.IntAllDef(Sysconfig["statusdays"], 15) * 86400),
+		jgTime:     int64(util.IntAllDef(7, 7) * 86400),
+	}
+	return p
+}
+
+var P_QL *ProjectTask
+var sp = make(chan bool, 1)
+
+func (p *ProjectTask) updateAllQueue() {
+	arru := make([][]map[string]interface{}, p.saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-p.updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == p.saveSize {
+				sp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-sp
+					}()
+					MongoTool.UpSertBulk(p.coll, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, p.saveSize)
+				indexu = 0
+			}
+		case <-time.After(1 * time.Second):
+			if indexu > 0 {
+				sp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-sp
+					}()
+					MongoTool.UpSertBulk(p.coll, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, p.saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+//项目合并内存更新
+func (p *ProjectTask) clearMem() {
+	c := cron.New()
+	// 创建项目的时间大于7天
+	//在内存中保留最近6个月的信息
+	//跑全量时每5分钟跑一次,跑增量时400分钟跑一次
+	_ = c.AddFunc("50 0/5 * * * *", func() {
+		if (p.currentType == "ql" && SingleClear == 0) || p.clearContimes >= 80 {
+			SingleClear = 1
+			//跳过的次数清零
+			p.clearContimes = 0
+			//信息进入查找对比全局锁
+			p.findLock.Lock()
+			//defer p.findLock.Unlock()
+			//合并进行的任务都完成
+			p.wg.Wait()
+			//遍历id
+			//所有内存中的项目信息
+			p.AllIdsMapLock.Lock()
+			p.mapHrefLock.Lock()
+			log.Println("清除开始")
+			//清除计数
+			clearNum := 0
+			for kHref, pid := range p.mapHref { //删除mapHref,p.AllIdsMap删除之前执行
+				v := p.AllIdsMap[pid]
+				if v != nil && p.currentTime-v.P.LastTime > p.validTime {
+					delete(p.mapHref, kHref)
+				}
+			}
+			for k, v := range p.AllIdsMap {
+				if p.currentTime-v.P.LastTime > p.validTime {
+					clearNum++
+					redis.Del("project", k)
+					//删除id的map
+					delete(p.AllIdsMap, k)
+					//删除pb
+					if v.P.Buyer != "" {
+						ids := p.mapPb[v.P.Buyer]
+						if ids != nil {
+							ids.Lock.Lock()
+							ids.Arr = deleteSlice(ids.Arr, k, "pb")
+							if len(ids.Arr) == 0 {
+								delete(p.mapPb, v.P.Buyer)
+							}
+							ids.Lock.Unlock()
+						}
+					}
+					//删除mapPn
+					for _, vn := range append([]string{v.P.ProjectName}, v.P.MPN...) {
+						if vn != "" {
+							ids := p.mapPn[vn]
+							if ids != nil {
+								ids.Lock.Lock()
+								ids.Arr = deleteSlice(ids.Arr, k, "pn")
+								if len(ids.Arr) == 0 {
+									delete(p.mapPn, vn)
+								}
+								ids.Lock.Unlock()
+							}
+						}
+					}
+					//删除mapPc
+					for _, vn := range append([]string{v.P.ProjectCode}, v.P.MPC...) {
+						if vn != "" {
+							ids := p.mapPc[vn]
+							if ids != nil {
+								ids.Lock.Lock()
+								ids.Arr = deleteSlice(ids.Arr, k, "pc")
+								if len(ids.Arr) == 0 {
+									delete(p.mapPc, vn)
+								}
+								ids.Lock.Unlock()
+							}
+						}
+					}
+					v = nil
+				}
+			}
+			p.mapHrefLock.Unlock()
+			p.AllIdsMapLock.Unlock()
+			p.findLock.Unlock()
+			SingleClear = 0
+			log.Println("清除完成:", clearNum, len(p.AllIdsMap), len(p.mapPn), len(p.mapPc), len(p.mapPb), len(p.mapHref))
+		} else {
+			p.clearContimes++
+		}
+	})
+	c.Start()
+}
+
+//全量合并
+func (p *ProjectTask) taskQl(udpInfo map[string]interface{}) {
+	defer util.Catch()
+	p.thread = util.IntAllDef(Thread, 4)
+	q, _ := udpInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{}
+		lteid, _ := udpInfo["lteid"].(string)
+		var idmap map[string]interface{}
+		if len(lteid) > 15 {
+			idmap = map[string]interface{}{
+				"$lte": mongodb.StringTOBsonId(lteid),
+			}
+		}
+		gtid, _ := udpInfo["gtid"].(string)
+		if len(gtid) > 15 {
+			if idmap == nil {
+				idmap = map[string]interface{}{}
+			}
+			idmap["$gte"] = mongodb.StringTOBsonId(gtid)
+		}
+		if idmap != nil {
+			q["_id"] = idmap
+		}
+	}
+	//生成查询语句执行
+	log.Println("查询语句:", q)
+	p.enter(MongoTool.DbName, ExtractColl, q)
+
+}
+
+//增量合并
+func (p *ProjectTask) taskZl(udpInfo map[string]interface{}) {
+	defer util.Catch()
+	//1、检查pubilshtime索引
+	db, _ := udpInfo["db"].(string)
+	if db == "" {
+		db = MongoTool.DbName
+	}
+	coll, _ := udpInfo["coll"].(string)
+	if coll == "" {
+		coll = ExtractColl
+	}
+	thread := util.IntAllDef(Thread, 1)
+	if thread > 0 {
+		p.thread = thread
+	}
+	//开始id和结束id
+	q, _ := udpInfo["query"].(map[string]interface{})
+	gtid := udpInfo["gtid"].(string)
+	lteid := udpInfo["lteid"].(string)
+	q = map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  mongodb.StringTOBsonId(gtid),
+			"$lte": mongodb.StringTOBsonId(lteid),
+		},
+	}
+	p.enter(db, coll, q)
+	if udpInfo["stop"] == nil {
+		for i := 0; i < 1; i++ {
+			sp <- true
+		}
+		for i := 0; i < 1; i++ {
+			<-sp
+		}
+	}
+	log.Println("保存完成,生索引", p.pici)
+	time.Sleep(5 * time.Second)
+	nextNode(udpInfo, p.pici)
+}
+
+//招标字段更新
+func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
+	defer util.Catch()
+	infoid := udpInfo["infoid"].(string)
+	infoMap, _ := MgoBidding.FindById(util.ObjToString(udpInfo["coll"]), infoid, nil)
+	if (*infoMap)["modifyinfo"] == nil {
+		util.Debug("does not exist modifyinfo ---,", infoid)
+		return
+	}
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+	esquery := `{"query": {"bool": {"must": [{"match": {"ids": "` + infoid + `"}}]}}}`
+	data := Es.Get(Index, esquery)
+	if len(*data) > 0 {
+		pid := util.ObjToString(((*data)[0])["_id"])
+		p.updateJudge(*infoMap, pid)
+	} else {
+		util.Debug("not find project---,", infoid)
+	}
+}
+
+func (p *ProjectTask) taskUpdatePro(udpInfo map[string]interface{}) {
+	defer util.Catch()
+	util.Debug(udpInfo)
+	pid := util.ObjToString(udpInfo["pid"])
+	updateMap := util.ObjToMap(udpInfo["updateField"])
+	if pid == "" || len(*updateMap) == 0 {
+		util.Debug("参数有误")
+		return
+	}
+	proMap, _ := MongoTool.FindById(ProjectColl, pid, nil)
+	if len(*proMap) > 1 {
+		(*proMap)["reason"] = "直接修改项目字段信息"
+		backupPro(*proMap)
+		delete(*proMap, "reason")
+		updataMap := make(map[string]interface{})
+		modifyInfo := make(map[string]interface{})
+		for k, v := range *updateMap {
+			if strings.Contains(k, "time") {
+				updataMap[k] = util.Int64All(v)
+			} else {
+				updataMap[k] = v
+			}
+			modifyInfo[k] = true
+		}
+		updataMap["modifyinfo"] = modifyInfo
+		util.Debug(updataMap)
+		bol := MongoTool.UpdateById(ProjectColl, pid, map[string]interface{}{"$set": updataMap})
+		if bol {
+			//es索引
+			by, _ := json.Marshal(map[string]interface{}{
+				"query": map[string]interface{}{
+					"_id": bson.M{
+						"$gte": pid,
+						"$lte": pid,
+					}},
+				"stype": "project",
+			})
+			util.Debug(string(by))
+			_ = udpclient.WriteUdp(by, udp.OP_TYPE_DATA, toaddr[1])
+		}
+		// 内存
+		var pro ProjectCache
+		err := mapstructure.Decode(proMap, &pro)
+		if err != nil {
+			util.Debug(err)
+		}
+		p.AllIdsMapLock.Lock()
+		if v, ok := p.AllIdsMap[pid]; ok {
+			v.P = &pro
+		}
+		p.AllIdsMapLock.Unlock()
+	} else {
+		util.Debug("Not find project---", pid)
+	}
+}
+
+func (p *ProjectTask) delInfoPro(udpInfo map[string]interface{}) {
+	defer util.Catch()
+	util.Debug(udpInfo)
+	infoid := util.ObjToString(udpInfo["infoid"])
+	if infoid == "" {
+		util.Debug("参数有误")
+		return
+	}
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+	esquery := `{"query": {"bool": {"must": [{"term": {"ids": "` + infoid + `"}}]}}}`
+	data := Es.Get(Index, esquery)
+	if len(*data) > 0 {
+		pid := util.ObjToString(((*data)[0])["_id"])
+		p.delJudge(infoid, pid)
+	} else {
+		util.Debug("not find project---,", infoid)
+	}
+}
+
+//通知下个节点nextNode
+func nextNode(mapInfo map[string]interface{}, pici int64) {
+	mapInfo["stype"] = "project"
+	mapInfo["query"] = map[string]interface{}{
+		"pici": pici,
+	}
+	key := fmt.Sprintf("%d-%s-%d", pici, "project", 0)
+	mapInfo["key"] = key
+	datas, _ := json.Marshal(mapInfo)
+	node := &udpNode{datas, toaddr[0], time.Now().Unix(), 0}
+	udptaskmap.Store(key, node)
+	_ = udpclient.WriteUdp(datas, udp.OP_TYPE_DATA, toaddr[0])
+}
+
+func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
+	defer util.Catch()
+	defer func() {
+		p.Brun = false
+	}()
+	p.Brun = true
+	count := 0
+	countRepeat := 0
+
+	pool := make(chan bool, p.thread)
+	util.Debug("start project", q, p.pici)
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+	infoPool := make(chan map[string]interface{}, 2000)
+	over := make(chan bool)
+	go func() {
+	L:
+		for {
+			select {
+			case tmp := <-infoPool:
+				pool <- true
+				go func(tmp map[string]interface{}) {
+					defer func() {
+						<-pool
+					}()
+					p.fillInPlace(tmp)
+					info := ParseInfo(tmp)
+					p.currentTime = info.Publishtime
+					//普通合并
+					p.CommonMerge(tmp, info)
+				}(tmp)
+			default:
+				select {
+				case tmp := <-infoPool:
+					pool <- true
+					go func(tmp map[string]interface{}) {
+						defer func() {
+							<-pool
+						}()
+						p.fillInPlace(tmp)
+						info := ParseInfo(tmp)
+						p.currentTime = info.Publishtime
+						//普通合并
+						p.CommonMerge(tmp, info)
+					}(tmp)
+				case <-over:
+					break L
+				}
+			}
+		}
+	}()
+	//fields := map[string]interface{}{"kvtext": 0, "repeat_reason": 0, "field_source": 0}
+	fields := map[string]interface{}{"detail": 0, "contenthtml": 0, "jsondata": 0, "regions_log": 0, "field_source": 0}
+	if p.currentType == "project" || p.currentType == "project_history" {
+		c, _ := sess.DB(db).C(coll).Find(q).Count()
+		util.Debug("共查询:", c, "条")
+	}
+	ms := sess.DB(db).C(coll).Find(q).Select(fields).Sort("publishtime")
+	query := ms.Iter()
+	var lastid interface{}
+L:
+	for {
+		select {
+		case <-queryClose:
+			log.Println("receive interrupt sign")
+			log.Println("close iter..", lastid, query.Cursor.Close(nil))
+			queryCloseOver <- true
+			break L
+		default:
+			tmp := make(map[string]interface{})
+			if query.Next(&tmp) {
+				lastid = tmp["_id"]
+				if P_QL.currentType == "ql" {
+					if count%50000 == 0 {
+						util.Debug("current", count, lastid)
+					}
+				} else {
+					if count%2000 == 0 {
+						util.Debug("current", count, lastid)
+					}
+				}
+				//extracttype -1: 重复,1: 不重复
+				if util.IntAll(tmp["extracttype"]) == 1 {
+					if util.ObjToString(tmp["toptype"]) != "采购意向" && util.ObjToString(tmp["toptype"]) != "产权" {
+						if P_QL.currentType == "ql" {
+							infoPool <- tmp
+						} else if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {
+							// id段增量数据
+							infoPool <- tmp
+						} else if P_QL.currentType == "project_history" && tmp["history_updatetime"] != nil {
+							// id段 历史数据
+							infoPool <- tmp
+						}
+					}
+				} else {
+					countRepeat++
+					//if P_QL.currentType == "project" {
+					//	util.Debug("repeat err---", tmp["_id"])
+					//}
+				}
+				count++
+			} else {
+				break L
+			}
+		}
+	}
+	time.Sleep(5 * time.Second)
+	over <- true
+	//阻塞
+	for n := 0; n < p.thread; n++ {
+		pool <- true
+	}
+	log.Println("所有线程执行完成...", count, countRepeat)
+
+}
+
+func (p *ProjectTask) CommonMerge(tmp map[string]interface{}, info *Info) {
+	if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
+		if jsonData, ok := tmp["jsondata"].(map[string]interface{}); ok {
+			proHref := util.ObjToString(jsonData["projecthref"]) // 网站本身发布的公告具有招投标流程,直接参与合并
+			if jsonData != nil && proHref != "" {
+				//projectHref字段合并
+				tmp["projecthref"] = proHref
+				p.mapHrefLock.Lock()
+				pid := p.mapHref[proHref]
+				p.mapHrefLock.Unlock()
+				if pid != "" {
+					p.AllIdsMapLock.Lock()
+					res := p.AllIdsMap[pid]
+					p.AllIdsMapLock.Unlock()
+					if res != nil {
+						comparePro := res.P
+						_, ex := p.CompareStatus(comparePro, info)
+						p.UpdateProject(tmp, info, comparePro, -1, "AAAAAAAAAA", ex)
+					} else {
+						p.startProjectMerge(info, tmp)
+					}
+				} else {
+					id, p1 := p.NewProject(tmp, info)
+					p.mapHrefLock.Lock()
+					p.mapHref[proHref] = id
+					p.mapHrefLock.Unlock()
+					p.AllIdsMapLock.Lock()
+					p.AllIdsMap[id] = &ID{Id: id, P: p1}
+					p.AllIdsMapLock.Unlock()
+				}
+			} else {
+				//项目合并
+				p.startProjectMerge(info, tmp)
+			}
+		} else {
+			//项目合并
+			p.startProjectMerge(info, tmp)
+		}
+	} else {
+	}
+}
+
+func ParseInfo(tmp map[string]interface{}) (info *Info) {
+	bys, _ := json.Marshal(tmp)
+	var thisinfo *Info
+	_ = json.Unmarshal(bys, &thisinfo)
+	if thisinfo == nil {
+		return nil
+	}
+
+	// 处理publishtime为空
+	if thisinfo.Publishtime <= 0 {
+		for _, d := range DateTimeSelect {
+			if tmp[d] != nil {
+				thisinfo.Publishtime = util.Int64All(tmp[d])
+				tmp["publishtime"] = tmp[d]
+				break
+			}
+		}
+	}
+
+	if len(thisinfo.Topscopeclass) == 0 {
+		thisinfo.Topscopeclass = []string{}
+	}
+	if len(thisinfo.Subscopeclass) == 0 {
+		thisinfo.Subscopeclass = []string{}
+	}
+	if thisinfo.SubType == "" {
+		thisinfo.SubType = util.ObjToString(tmp["bidstatus"])
+	}
+
+	//从标题中查找项目编号
+	res := titleGetPc.FindStringSubmatch(thisinfo.Title)
+	if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
+		thisinfo.PTC = res[1]
+	} else {
+		res = titleGetPc1.FindStringSubmatch(thisinfo.Title)
+		if len(res) > 3 && len(res[3]) > 6 && thisinfo.ProjectCode != res[3] && !numCheckPc.MatchString(res[3]) && !_zimureg1.MatchString(res[3]) {
+			thisinfo.PTC = res[3]
+		} else {
+			res = titleGetPc2.FindStringSubmatch(thisinfo.Title)
+			if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
+				thisinfo.PTC = res[1]
+			}
+		}
+	}
+
+	if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
+		//thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
+		//if thisinfo.ProjectName != "" {
+		thisinfo.pnbval++
+		//}
+	}
+
+	if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
+		if thisinfo.ProjectCode != "" {
+			//thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
+			if thisinfo.pnbval == 0 && len([]rune(thisinfo.ProjectCode)) < 5 {
+				thisinfo.ProjectCode = StrOrNum.ReplaceAllString(thisinfo.ProjectCode, "")
+			}
+		} else {
+			//thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
+			if thisinfo.pnbval == 0 && len([]rune(thisinfo.PTC)) < 5 {
+				thisinfo.PTC = StrOrNum.ReplaceAllString(thisinfo.PTC, "")
+			}
+		}
+		thisinfo.pnbval++
+	}
+	if thisinfo.ProjectCode == thisinfo.PTC || strings.Index(thisinfo.ProjectCode, thisinfo.PTC) > -1 {
+		thisinfo.PTC = ""
+	}
+
+	if thisinfo.Buyer != "" && len([]rune(thisinfo.Buyer)) > 2 {
+		thisinfo.pnbval++
+	} else {
+		thisinfo.Buyer = ""
+	}
+	//清理评审专家名单
+	if len(thisinfo.ReviewExperts) > 0 {
+		thisinfo.ReviewExperts = ClearRp(thisinfo.ReviewExperts)
+	}
+	//winners整理、清理
+	winner := QyFilter(util.ObjToString(tmp["winner"]), "winner")
+	tmp["winner"] = winner
+	m1 := map[string]bool{}
+	winners := []string{}
+	if winner != "" {
+		m1[winner] = true
+		winners = append(winners, winner)
+	}
+	packageM, _ := tmp["package"].(map[string]interface{})
+	if packageM != nil {
+		thisinfo.HasPackage = true
+		for _, p := range packageM {
+			pm, _ := p.(map[string]interface{})
+			pw := QyFilter(util.ObjToString(pm["winner"]), "winner")
+			if pw != "" && !m1[pw] {
+				m1[pw] = true
+				winners = append(winners, pw)
+			}
+		}
+	}
+	thisinfo.Winners = winners
+	//清理winnerorder
+	var wins []map[string]interface{}
+	for _, v := range thisinfo.WinnerOrder {
+		w := QyFilter(util.ObjToString(v["entname"]), "winner")
+		if w != "" {
+			v["entname"] = w
+			wins = append(wins, v)
+		}
+	}
+	thisinfo.WinnerOrder = wins
+	//清理buyer
+	buyer := QyFilter(util.ObjToString(tmp["buyer"]), "buyer")
+	tmp["buyer"] = buyer
+	thisinfo.Buyer = buyer
+
+	thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
+	thisinfo.LenPTC = len([]rune(thisinfo.PTC))
+	thisinfo.LenPN = len([]rune(thisinfo.ProjectName))
+
+	//处理分包中数据异常问题
+	for k, tmp := range thisinfo.Package {
+		if ps, ok := tmp.([]map[string]interface{}); ok {
+			for i, p := range ps {
+				name, _ := p["name"].(string)
+				if len([]rune(name)) > 100 {
+					p["name"] = fmt.Sprint([]rune(name[:100]))
+				}
+				ps[i] = p
+			}
+			thisinfo.Package[k] = ps
+		}
+	}
+	return thisinfo
+}
+
+func (p *ProjectTask) updateJudge(infoMap map[string]interface{}, pid string) {
+	tmpPro, _ := MongoTool.FindById(ProjectColl, pid, nil)
+	modifyProMap := make(map[string]interface{}) // 修改项目的字段
+	if modifyMap, ok := infoMap["modifyinfo"].(map[string]interface{}); ok {
+		for k := range modifyMap {
+			if modifyMap[k] != nil && modifyMap[k] != "" && (*tmpPro)[k] != nil {
+				modifyProMap[k] = infoMap[k]
+			}
+		}
+	}
+	if len(modifyProMap) == 0 {
+		util.Debug("修改招标公告信息不需要修改项目信息字段", infoMap["_id"])
+		return
+	}
+	p.AllIdsMapLock.Lock()
+	_, ok := p.AllIdsMap[pid]
+	p.AllIdsMapLock.Unlock()
+	ids := (*tmpPro)["ids"].([]interface{})
+	index, position := -1, 0 // index 0:第一个,1:中间,2:最后一个   position list中位置
+
+	for i, v := range ids {
+		if util.ObjToString(v) == mongodb.BsonIdToSId(infoMap["_id"]) {
+			position = i
+			if i == 0 {
+				index = 0
+			} else if i == len(ids)-1 {
+				index = 2
+			} else {
+				index = 1
+			}
+		}
+	}
+	if ok {
+		// 周期内
+		//projecthref字段
+		if infoMap["jsondata"] != nil {
+			jsonData := infoMap["jsondata"].(map[string]interface{})
+			if proHref, ok := jsonData["projecthref"].(string); ok {
+				p.mapHrefLock.Lock()
+				tempId := p.mapHref[proHref]
+				p.mapHrefLock.Unlock()
+				if pid == tempId {
+					p.modifyUpdate(pid, index, position, *tmpPro, modifyProMap)
+				} else {
+					util.Debug("projecthref data id err---pid="+pid, "---"+tempId)
+				}
+			} else {
+				f := modifyEle(modifyProMap)
+				if f {
+					//合并、修改
+					util.Debug("合并修改更新", "----------------------------")
+					p.mergeAndModify(pid, index, position, infoMap, *tmpPro, modifyProMap)
+				} else {
+					//修改
+					util.Debug("修改更新", "----------------------------")
+					p.modifyUpdate(pid, index, position, *tmpPro, modifyProMap)
+				}
+			}
+		} else {
+			f := modifyEle(modifyProMap)
+			if f {
+				//合并、修改
+				util.Debug("合并修改更新", "----------------------------")
+				p.mergeAndModify(pid, index, position, infoMap, *tmpPro, modifyProMap)
+			} else {
+				//修改
+				util.Debug("修改更新", "----------------------------")
+				p.modifyUpdate(pid, index, position, *tmpPro, modifyProMap)
+			}
+		}
+	} else {
+		// 周期外
+		util.Debug("周期外数据直接修改", "----------------------------")
+		p.modifyUpdate(pid, index, position, *tmpPro, modifyProMap)
+	}
+}
+
+var Elements = []string{
+	"projectname",
+	"projectcode",
+	"buyer",
+	"agency",
+	"area",
+	"city",
+	"publishtime",
+	"toptype",
+	"subtype",
+}
+
+/**
+修改的字段
+修改的字段是否是影响合并流程的要素字段
+*/
+func modifyEle(tmp map[string]interface{}) bool {
+	merge := false
+	for _, str := range Elements {
+		if tmp[str] != nil {
+			merge = true
+			break
+		}
+	}
+	return merge
+}
+
+//补全位置信息
+func (p *ProjectTask) fillInPlace(tmp map[string]interface{}) {
+	area := util.ObjToString(tmp["area"])
+	city := util.ObjToString(tmp["city"])
+	if area != "" && city != "" {
+		return
+	}
+
+	tmpSite := util.ObjToString(tmp["site"])
+	if tmpSite == "" {
+		return
+	}
+	p.mapSiteLock.Lock()
+	defer p.mapSiteLock.Unlock()
+	site := p.mapSite[tmpSite]
+	if site != nil {
+		if area != "" {
+			if area == "全国" {
+				tmp["area"] = site.Area
+				tmp["city"] = site.City
+				tmp["district"] = site.District
+				return
+			}
+			if area != site.Area {
+				return
+			} else {
+				if site.City != "" {
+					tmp["area"] = site.Area
+					tmp["city"] = site.City
+					tmp["district"] = site.District
+				}
+			}
+		} else {
+			tmp["area"] = site.Area
+			tmp["city"] = site.City
+			tmp["district"] = site.District
+			return
+		}
+	}
+}
+
+//从数组中删除元素
+func deleteSlice(arr []string, v, stype string) []string {
+	for k, v1 := range arr {
+		if v1 == v {
+			ts := time.Now().Unix()
+			arr = append(arr[:k], arr[k+1:]...)
+			rt := time.Now().Unix() - ts
+			if rt > 0 {
+				log.Println("deleteSlice", stype, rt, v, len(arr))
+			}
+			return arr
+		}
+	}
+	return arr
+}
+
+//校验评审专家
+func ClearRp(tmp []string) []string {
+	arrTmp := []string{}
+	for _, v := range tmp {
+		// 汉字过滤(全汉字,2-4个字)
+		if ok, _ := regexp.MatchString("^[\\p{Han}]{2,4}$", v); !ok {
+			continue
+		}
+		//黑名单过滤
+		if BlaskListMap[v] {
+			continue
+		}
+		arrTmp = append(arrTmp, v)
+	}
+	return arrTmp
+}
+
+func QyFilter(name, stype string) string {
+	name = strings.ReplaceAll(name, " ", "")
+
+	//preReg := PreRegexp[stype]
+	//for _, v := range preReg {
+	//	name = v.ReplaceAllString(name, "")
+	//}
+	//backReg := BackRegexp[stype]
+	//for _, v := range backReg {
+	//	name = v.ReplaceAllString(name, "")
+	//}
+	//backRepReg := BackRepRegexp[stype]
+	//for _, v := range backRepReg {
+	//	name = v.regs.ReplaceAllString(name, v.repstr)
+	//}
+
+	blackReg := BlackRegexp[stype]
+	for _, v := range blackReg {
+		if v.MatchString(name) {
+			name = ""
+			break
+		}
+	}
+
+	if !regexp.MustCompile("[\\p{Han}]{4,}").MatchString(name) {
+		name = ""
+	}
+	if utf8.RuneCountInString(name) > 60 {
+		name = ""
+	}
+	return name
+}

+ 59 - 0
data_project/udptaskmap.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"app.yhyue.com/data_processing/common_utils/udp"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+)
+
+var udptaskmap = &sync.Map{}
+var tomail string
+var api string
+
+type udpNode struct {
+	data      []byte
+	addr      *net.UDPAddr
+	timestamp int64
+	retry     int
+}
+
+func checkMapJob() {
+	//阿里云内网无法发送邮件
+	jkmail, _ := Sysconfig["jkmail"].(map[string]interface{})
+	if jkmail != nil {
+		tomail, _ = jkmail["to"].(string)
+		api, _ = jkmail["api"].(string)
+		log.Println("start checkMapJob", tomail, Sysconfig["jkmail"])
+		for {
+			udptaskmap.Range(func(k, v interface{}) bool {
+				now := time.Now().Unix()
+				node, _ := v.(*udpNode)
+				if now-node.timestamp > 120 {
+					node.retry++
+					if node.retry > 5 {
+						log.Println("udp重试失败", k)
+						udptaskmap.Delete(k)
+						res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, tomail, "project-send-fail", k.(string)))
+						if err == nil {
+							defer res.Body.Close()
+							read, err := ioutil.ReadAll(res.Body)
+							log.Println("邮件发发送:", string(read), err)
+						}
+					} else {
+						log.Println("udp重发", k)
+						udpclient.WriteUdp(node.data, udp.OP_TYPE_DATA, node.addr)
+					}
+				} else if now-node.timestamp > 10 {
+					log.Println("udp任务超时中..", k)
+				}
+				return true
+			})
+			time.Sleep(60 * time.Second)
+		}
+	}
+}

+ 1908 - 0
data_project/update.go

@@ -0,0 +1,1908 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/redis"
+	"app.yhyue.com/data_processing/common_utils/udp"
+	"encoding/json"
+	"fmt"
+	"github.com/goinggo/mapstructure"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"log"
+	"math"
+	"reflect"
+	"sort"
+	"strings"
+	"time"
+)
+
+/**
+直接修改保存项目字段信息
+*/
+func (p *ProjectTask) modifyUpdate(pid string, index, position int, tmpPro, modifyProMap map[string]interface{}) {
+	// 1-0 备份
+	tmpPro["reason"] = "直接在原有项目上修改字段信息"
+	backupPro(tmpPro)
+	delete(tmpPro, "reason")
+	// 1-1 修改字段信息
+	updateSet := UpdateValue(tmpPro, index, position, modifyProMap)
+	updateSet["list"] = tmpPro["list"]
+	// 1-2 修改mgo
+	bol := MongoTool.UpdateById(ProjectColl, pid, map[string]interface{}{"$set": updateSet})
+	if bol {
+		// 1-3 修改es
+		by, _ := json.Marshal(map[string]interface{}{
+			"query": map[string]interface{}{
+				"_id": bson.M{
+					"$gte": pid,
+					"$lte": pid,
+				}},
+			"stype": "project",
+		})
+		util.Debug(string(by))
+		_ = udpclient.WriteUdp(by, udp.OP_TYPE_DATA, toaddr[1])
+	}
+	// 修改内存
+	p.modifyMem(tmpPro)
+}
+
+func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp map[string]interface{}, tmpPro map[string]interface{}, modifyMap map[string]interface{}) {
+	proList := tmpPro["list"].([]interface{})
+	info := ParseInfo(tmp)
+	//项目中list大小等于1
+	if len(proList) == 1 {
+		log.Println("list大小等于1    合并", "------1------")
+		// 2.1 重新合并
+		merge := p.ReMerge(info, tmp, tmpPro)
+		if merge {
+			//合并到新项目中
+			//删除老项目
+			p.AllIdsMapLock.Lock()
+			delete(p.AllIdsMap, pInfoId)
+			p.AllIdsMapLock.Unlock()
+			delOldPro(pInfoId)
+			// 2.2 备份
+			tmpPro["reason"] = "删除原有项目,修改合并到新的项目中"
+			backupPro(tmpPro)
+		} else {
+			//没有合并到新项目中
+			p.modifyUpdate(pInfoId, index, position, tmpPro, modifyMap)
+			// 修改内存
+			p.modifyMem(tmpPro)
+		}
+	} else if index == 0 {
+		// 招标公告信息在原有项目中位于第1个
+		merge := p.ReMerge(info, tmp, tmpPro)
+		if merge {
+			// 合并到新项目中
+			// 该条招标公告信息重新参与合并,该项目的其它招标公告信息逐条参与合并
+			tmpPro["reason"] = "修改list中第一个条招标公告信息"
+			backupPro(tmpPro)
+			delete(tmpPro, "reason")
+			proList = deleteSlice1(proList, proList[0])
+			p.AllIdsMapLock.Lock()
+			Id := p.AllIdsMap[pInfoId]
+			p.AllIdsMapLock.Unlock()
+			proMem := Id.P
+			flag := true // 标记是否需要删除原有项目信息
+			for _, v := range proList {
+				v1 := v.(map[string]interface{})
+				temp, _ := MongoTool.FindById(ExtractColl, util.ObjToString(v1["infoid"]), nil)
+				if len(*temp) == 0 {
+					temp, _ = MongoTool.FindById(ExtractColl1, util.ObjToString(v1["infoid"]), nil)
+					if len(*temp) == 0 {
+						util.Debug("extract not find id...", v1["infoid"])
+						continue
+					}
+				}
+				tempInfo := ParseInfo(*temp)
+				if flag {
+					merge := p.ReMerge(tempInfo, *temp, tmpPro)
+					if !merge {
+						flag = false
+						break
+					}
+				}
+			}
+			if flag {
+				delOldPro(pInfoId)
+			} else {
+				//合并到原有项目中, 不用继续重新合并, 原有项目内部合并即可
+				p.innerMerge(proList, tmpPro)
+				bol := MongoTool.UpdateById(ProjectColl, pInfoId, map[string]interface{}{"$set": tmpPro})
+				if bol {
+					by, _ := json.Marshal(map[string]interface{}{
+						"query": map[string]interface{}{
+							"_id": bson.M{
+								"$gte": pInfoId,
+								"$lte": pInfoId,
+							}},
+						"stype": "project",
+					})
+					util.Debug(string(by))
+					_ = udpclient.WriteUdp(by, udp.OP_TYPE_DATA, toaddr[1])
+				}
+				// 修改内存
+				p.AllIdsMapLock.Lock()
+				if v, ok := p.AllIdsMap[pInfoId]; ok {
+					v.P = proMem
+				}
+				p.AllIdsMapLock.Unlock()
+			}
+		} else {
+			//没有合并到新项目中,更新list字段,有条件更新项目外围字段
+			p.modifyUpdate(pInfoId, index, position, tmpPro, modifyMap)
+			// 修改内存
+			p.modifyMem(tmpPro)
+		}
+	} else if index == 1 {
+		// 招标公告信息在原有项目中处于中间某一条
+		merge := p.ReMerge(info, tmp, tmpPro)
+		if merge {
+			tmpPro["reason"] = "修改list中位于中间某一条招标公告信息"
+			backupPro(tmpPro)
+			delete(tmpPro, "reason")
+			p.innerMerge1(proList, util.ObjToString(tmp["infoid"]), tmpPro)
+			bol := MongoTool.UpdateById(ProjectColl, pInfoId, map[string]interface{}{"$set": tmpPro})
+			if bol {
+				by, _ := json.Marshal(map[string]interface{}{
+					"query": map[string]interface{}{
+						"_id": bson.M{
+							"$gte": pInfoId,
+							"$lte": pInfoId,
+						}},
+					"stype": "project",
+				})
+				util.Debug(string(by))
+				_ = udpclient.WriteUdp(by, udp.OP_TYPE_DATA, toaddr[1])
+			}
+		} else {
+			// 未合并到新项目中
+			p.modifyUpdate(pInfoId, index, position, tmpPro, modifyMap)
+			// 修改内存
+			p.modifyMem(tmpPro)
+		}
+	} else if index == 2 {
+		// 招标公告信息在原有项目中位于最后一条
+		merge := p.ReMerge(info, tmp, tmpPro)
+		if merge {
+			// 合并到新项目中
+			tmpPro["reason"] = "修改list中最后一条招标公告信息"
+			backupPro(tmpPro)
+			w := len(proList) - 1
+			proList = deleteSlice1(proList, proList[w])
+			p.innerMerge(proList, tmpPro)
+			bol := MongoTool.UpdateById(ProjectColl, pInfoId, map[string]interface{}{"$set": tmpPro})
+			if bol {
+				by, _ := json.Marshal(map[string]interface{}{
+					"query": map[string]interface{}{
+						"_id": bson.M{
+							"$gte": pInfoId,
+							"$lte": pInfoId,
+						}},
+					"stype": "project",
+				})
+				util.Debug(string(by))
+				_ = udpclient.WriteUdp(by, udp.OP_TYPE_DATA, toaddr[1])
+			}
+		} else {
+			// 未合并到新项目中
+			p.modifyUpdate(pInfoId, index, position, tmpPro, modifyMap)
+			// 修改内存
+			p.modifyMem(tmpPro)
+		}
+	}
+}
+
+// 删除
+func (p *ProjectTask) delJudge(infoid, pid string) {
+	tmpPro, _ := MongoTool.FindById(ProjectColl, pid, nil)
+	ids := (*tmpPro)["ids"].([]interface{})
+	proList := (*tmpPro)["list"].([]interface{})
+	if len(ids) == 1 {
+		(*tmpPro)["reason"] = "删除项目信息"
+		backupPro(*tmpPro)
+		c := MongoTool.Delete(ProjectColl, map[string]interface{}{"_id": mongodb.StringTOBsonId(pid)})
+		if c > 0 {
+			client := Es.GetEsConn()
+			defer Es.DestoryEsConn(client)
+			Es.DelById(Itype, pid)
+		}
+		return
+	}
+	var index = -1 //0:第一个,1:中间,2:最后一个
+	for i, v := range ids {
+		if util.ObjToString(v) == infoid {
+			if i == 0 {
+				index = 0
+			} else if i == len(ids)-1 {
+				index = 2
+			} else {
+				index = 1
+			}
+		}
+	}
+	if index == 0 {
+		(*tmpPro)["reason"] = "删除list中第一个条招标公告信息"
+		backupPro(*tmpPro)
+		delete(*tmpPro, "reason")
+		proList = deleteSlice1(proList, proList[0])
+		var pro *ProjectCache
+		flag := true // 标记是否需要删除原有项目信息
+		for _, v := range proList {
+			v1 := v.(map[string]interface{})
+			temp, _ := MongoTool.FindById(ExtractColl, util.ObjToString(v1["infoid"]), nil)
+			if len(*temp) == 0 {
+				temp, _ = MongoTool.FindById(ExtractColl1, util.ObjToString(v1["infoid"]), nil)
+				if len(*temp) == 0 {
+					util.Debug("extract not find id...", v1["infoid"])
+					continue
+				}
+			}
+			tempInfo := ParseInfo(*temp)
+			if flag {
+				merge := p.ReMerge(tempInfo, *temp, *tmpPro)
+				if !merge {
+					flag = false
+					break
+				}
+			}
+		}
+		if flag {
+			delOldPro(pid)
+		} else {
+			*tmpPro, pro = p.innerMerge(proList, *tmpPro)
+			bol := MongoTool.UpdateById(ProjectColl, pid, map[string]interface{}{"$set": tmpPro})
+			if bol {
+				by, _ := json.Marshal(map[string]interface{}{
+					"query": map[string]interface{}{
+						"_id": bson.M{
+							"$gte": pid,
+							"$lte": pid,
+						}},
+					"stype": "project",
+				})
+				util.Debug(string(by))
+				_ = udpclient.WriteUdp(by, udp.OP_TYPE_DATA, toaddr[1])
+			}
+			// 修改内存
+			p.AllIdsMapLock.Lock()
+			if v, ok := p.AllIdsMap[pid]; ok {
+				v.P = pro
+			}
+			p.AllIdsMapLock.Unlock()
+		}
+	} else if index == 1 {
+		(*tmpPro)["reason"] = "删除list中间某一条招标公告信息"
+		backupPro(*tmpPro)
+		delete(*tmpPro, "reason")
+		var pro *ProjectCache
+		*tmpPro, pro = p.innerMerge1(proList, infoid, *tmpPro)
+		bol := MongoTool.UpdateById(ProjectColl, pid, map[string]interface{}{"$set": tmpPro})
+		if bol {
+			by, _ := json.Marshal(map[string]interface{}{
+				"query": map[string]interface{}{
+					"_id": bson.M{
+						"$gte": pid,
+						"$lte": pid,
+					}},
+				"stype": "project",
+			})
+			util.Debug(string(by))
+			_ = udpclient.WriteUdp(by, udp.OP_TYPE_DATA, toaddr[1])
+		}
+		// 内存
+		p.AllIdsMapLock.Lock()
+		if v, ok := p.AllIdsMap[pid]; ok {
+			v.P = pro
+		}
+		p.AllIdsMapLock.Unlock()
+	} else if index == 2 {
+		(*tmpPro)["reason"] = "删除list中最后一条招标公告信息"
+		backupPro(*tmpPro)
+		delete(*tmpPro, "reason")
+		w := len(proList) - 1
+		proList = deleteSlice1(proList, proList[w])
+		var pro *ProjectCache
+		*tmpPro, pro = p.innerMerge(proList, *tmpPro)
+		bol := MongoTool.UpdateById(ProjectColl, pid, map[string]interface{}{"$set": tmpPro})
+		if bol {
+			by, _ := json.Marshal(map[string]interface{}{
+				"query": map[string]interface{}{
+					"_id": bson.M{
+						"$gte": pid,
+						"$lte": pid,
+					}},
+				"stype": "project",
+			})
+			util.Debug(string(by))
+			_ = udpclient.WriteUdp(by, udp.OP_TYPE_DATA, toaddr[1])
+		}
+		// 内存
+		p.AllIdsMapLock.Lock()
+		if v, ok := p.AllIdsMap[pid]; ok {
+			v.P = pro
+		}
+		p.AllIdsMapLock.Unlock()
+	}
+}
+
+// 合并
+func (p *ProjectTask) ReMerge(info *Info, tmp map[string]interface{}, tmpPro map[string]interface{}) bool {
+	bpn, bpc, bptc, bpb, pids, _, IDArr := p.getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer)
+	defer p.wg.Done()
+	for _, m := range pids {
+		defer m.Lock.Unlock()
+	}
+	for _, id := range IDArr {
+		defer id.Lock.Unlock()
+	}
+
+	bFindProject := false
+	findPid := ""
+	comRes1 := []*ProjectCache{}
+	comRes2 := []*ProjectCache{}
+	comRes3 := []*ProjectCache{}
+	for _, v := range IDArr {
+		comStr := ""
+		compareProject := v.P
+		compareProject.score = 0
+		diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime)))
+		if diffTime <= p.validTime {
+			if CheckContain(compareProject.Agency, info.Agency) == 3 {
+				continue
+			}
+			if ComparePlace(compareProject, info) {
+				continue
+			}
+			info.PNBH = 0
+			info.PCBH = 0
+			info.PTCBH = 0
+			compareStr, score := comparePNC(info, compareProject)
+			resVal, pjVal := Select(compareStr, info, compareProject)
+			if resVal > 0 {
+				compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount, score2 := p.compareBCTABB(info, compareProject, diffTime, score)
+				comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
+				compareProject.comStr = comStr
+				compareProject.pjVal = pjVal
+				compareProject.resVal = resVal
+				eqV := compareResult(resVal, pjVal, score2, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount)
+				if eqV == 1 {
+					comRes1 = append(comRes1, compareProject)
+				} else if eqV == 2 {
+					comRes2 = append(comRes2, compareProject)
+				} else if eqV == 3 {
+					comRes3 = append(comRes3, compareProject)
+				}
+			}
+		}
+	}
+	//--------------------------------对比完成-----------------------
+	for kv, resN := range [][]*ProjectCache{comRes1, comRes2, comRes3} {
+		if len(resN) > 0 {
+			if len(resN) > 1 {
+				sort.Slice(resN, func(i, j int) bool {
+					return resN[i].score > resN[j].score
+				})
+			}
+
+			ex := 0
+			resArr := []*ProjectCache{}
+			for i, res := range resN {
+				choose, e := p.CompareStatus(resN[i], info)
+				if !choose {
+					ex = e
+					resArr = append(resArr, res)
+				}
+			}
+			if len(resArr) > 0 {
+				bFindProject = true
+				findPid = resArr[0].Id.Hex()
+				if findPid == mongodb.BsonIdToSId(tmpPro["_id"]) {
+					return false
+				}
+				p.updateProFiled(tmp, info, resArr[0], kv+1, resArr[0].comStr, ex)
+				for k2, bv := range []int{bpn, bpc, bptc, bpb} {
+					if bv > -1 {
+						pids[bv].Arr = append(pids[bv].Arr, findPid)
+						if k2 == 0 {
+							if resArr[0].ProjectName == "" {
+								resArr[0].ProjectName = info.ProjectName
+							} else {
+								if resArr[0].MPN == nil {
+									resArr[0].MPN = []string{info.ProjectName}
+								} else {
+									resArr[0].MPN = append(resArr[0].MPN, info.ProjectName)
+								}
+							}
+
+						} else if k2 < 3 {
+							if resArr[0].ProjectCode == "" {
+								resArr[0].ProjectCode = util.If(k2 == 1, info.ProjectCode, info.PTC).(string)
+							} else {
+								if resArr[0].MPC == nil {
+									resArr[0].MPC = []string{util.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
+								} else {
+									resArr[0].MPC = append(resArr[0].MPC, util.If(k2 == 1, info.ProjectCode, info.PTC).(string))
+								}
+							}
+						} else {
+							if resArr[0].Buyer == "" {
+								resArr[0].Buyer = info.Buyer
+							}
+						}
+					}
+				}
+			} else {
+				bFindProject = false
+				findPid = ""
+			}
+			break
+		}
+	}
+	if !bFindProject {
+		id, p1 := p.NewProject(tmp, info)
+		p.AllIdsMapLock.Lock()
+		p.AllIdsMap[id] = &ID{Id: id, P: p1}
+		p.AllIdsMapLock.Unlock()
+		for _, m := range pids {
+			m.Arr = append(m.Arr, id)
+		}
+		return false
+	}
+	return true
+}
+
+//内部合并
+func (p *ProjectTask) innerMerge(infoList []interface{}, tmpPro map[string]interface{}) (map[string]interface{}, *ProjectCache) {
+	newP := make(map[string]interface{})
+	newP["_id"] = tmpPro["_id"]
+	var p1 *ProjectCache
+	for k, m := range infoList {
+		m1 := m.(map[string]interface{})
+		temp, _ := MongoTool.FindById(ExtractColl, util.ObjToString(m1["infoid"]), nil)
+		if len(*temp) == 0 {
+			temp, _ = MongoTool.FindById(ExtractColl1, util.ObjToString(m1["infoid"]), nil)
+			if len(*temp) == 0 {
+				util.Debug("extract not find id...", m1["infoid"])
+				continue
+			}
+		}
+		tempInfo := ParseInfo(*temp)
+		if k == 0 {
+			p1 = p.newPro(*temp, newP, tempInfo)
+		} else {
+			p.updateOldProField(p1, tempInfo, newP, *temp, m1)
+		}
+	}
+	return newP, p1
+}
+
+func (p *ProjectTask) innerMerge1(infoList []interface{}, infoid string, tmpPro map[string]interface{}) (map[string]interface{}, *ProjectCache) {
+	newP := make(map[string]interface{})
+	newP["_id"] = tmpPro["_id"]
+	var p1 *ProjectCache
+	for k, m := range infoList {
+		m1 := m.(map[string]interface{})
+		if m1["infoid"] == infoid {
+			continue
+		}
+		temp, _ := MongoTool.FindById(ExtractColl, util.ObjToString(m1["infoid"]), nil)
+		if len(*temp) == 0 {
+			temp, _ = MongoTool.FindById(ExtractColl1, util.ObjToString(m1["infoid"]), nil)
+			if len(*temp) == 0 {
+				util.Debug("extract not find id...", m1["infoid"])
+				continue
+			}
+		}
+		tempInfo := ParseInfo(*temp)
+		if k == 0 {
+			p1 = p.newPro(*temp, newP, tempInfo)
+		} else {
+			p.updateOldProField(p1, tempInfo, newP, *temp, m1)
+		}
+	}
+	return newP, p1
+}
+
+// 更新招标公告到新的项目中
+func (p *ProjectTask) updateProFiled(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectCache, weight int, comStr string, ex int) {
+	pdata := redis.Get("project", pInfo.Id.Hex())
+	projectMap := pdata.(map[string]interface{})
+	bys, _ := json.Marshal(projectMap)
+	var project *Project
+	err := json.Unmarshal(bys, &project)
+	if err != nil {
+		util.Debug("project Unmarshal err,", err)
+		return
+	}
+	set := map[string]interface{}{}
+	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
+	project.Ids = append(project.Ids, thisinfo.Id)
+	if len(pInfo.Ids) > 30 {
+		//异常标记
+		set["listtag"] = 1
+	}
+	if thisinfo.Publishtime > pInfo.LastTime {
+		pInfo.LastTime = thisinfo.Publishtime
+		project.LastTime = thisinfo.Publishtime
+		set["lasttime"] = thisinfo.Publishtime
+	}
+	if thisinfo.TopType == "招标" {
+		if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" && project.Zbtime <= 0 {
+			set["zbtime"] = tmp["publishtime"]
+		}
+	} else if thisinfo.TopType == "结果" {
+		if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+			if project.Jgtime > 0 {
+				jg1 := int64(math.Abs(float64(project.Jgtime - thisinfo.Publishtime)))
+				//公告状态和项目状态同样都是中标或者成交,
+				if (thisinfo.SubType == "中标" || thisinfo.SubType == "成交") && (pInfo.Bidstatus == "中标" || pInfo.Bidstatus == "成交") {
+					if jg1 > p.jgTime {
+						set["jgtime"] = tmp["publishtime"]
+						project.Jgtime = thisinfo.Publishtime
+					}
+				} else if (thisinfo.SubType == "流标" || thisinfo.SubType == "废标") && (pInfo.Bidstatus == "流标" || pInfo.Bidstatus == "废标") {
+					//公告状态和项目状态同样是流标或者废标
+					if jg1 > p.jgTime {
+						set["jgtime"] = tmp["publishtime"]
+						project.Jgtime = thisinfo.Publishtime
+					}
+				}
+			} else {
+				set["jgtime"] = tmp["publishtime"]
+				project.Jgtime = thisinfo.Publishtime
+			}
+		}
+	} else if thisinfo.SubType == "合同" {
+		if pInfo.Bidstatus == "中标" || pInfo.Bidstatus == "成交" || pInfo.Bidstatus == "" {
+			//中标、成交不更新jgtime
+		} else {
+			set["jgtime"] = tmp["publishtime"]
+			project.Jgtime = thisinfo.Publishtime
+		}
+	}
+	if thisinfo.Bidopentime > project.Bidopentime {
+		project.Bidopentime = thisinfo.Bidopentime
+		set["bidopentime"] = thisinfo.Bidopentime
+	}
+	// bidtype、bidstatus
+	pInfo.Bidtype, pInfo.Bidstatus = p.GetBidTypeAndBidStatus(thisinfo)
+	project.Bidtype, project.Bidstatus = p.GetBidTypeAndBidStatus(thisinfo)
+	set["bidtype"] = pInfo.Bidtype
+	set["bidstatus"] = pInfo.Bidstatus
+
+	//异常标记
+	if ex > 0 {
+		set["exception"] = ex
+	}
+	//相同城市的公告才会合并到一起(全国列外)
+	if thisinfo.Area != "全国" {
+		pInfo.Area = thisinfo.Area
+		project.Area = thisinfo.Area
+		set["area"] = thisinfo.Area
+		pInfo.City = thisinfo.City
+		project.City = thisinfo.City
+		set["city"] = thisinfo.City
+		if thisinfo.District != "" {
+			pInfo.District = thisinfo.District
+			project.District = thisinfo.District
+			set["district"] = thisinfo.District
+		}
+	}
+	// 项目名称
+	if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
+		pInfo.ProjectName = thisinfo.ProjectName
+		project.ProjectName = thisinfo.ProjectName
+		set["projectname"] = thisinfo.ProjectName
+	}
+	// 项目编号
+	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+		pInfo.ProjectCode = thisinfo.ProjectCode
+		project.ProjectCode = thisinfo.ProjectCode
+		set["projectcode"] = thisinfo.ProjectCode
+	}
+	// 采购单位
+	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
+		pInfo.Buyer = thisinfo.Buyer
+		set["buyer"] = thisinfo.Buyer
+		project.Buyerclass = thisinfo.Buyerclass
+		set["buyerclass"] = thisinfo.Buyerclass
+	}
+	if pInfo.Buyer == "" {
+		project.Buyerclass = ""
+		set["buyerclass"] = ""
+	}
+	// 采购单位联系人
+	if thisinfo.Buyerperson != "" {
+		project.Buyerperson = thisinfo.Buyerperson
+		set["buyerperson"] = thisinfo.Buyerperson
+	} else {
+		project.Buyerperson = ""
+		set["buyerperson"] = ""
+	}
+	// 采购单位電話
+	if thisinfo.Buyertel != "" {
+		project.Buyertel = thisinfo.Buyertel
+		set["buyertel"] = thisinfo.Buyertel
+	} else {
+		project.Buyertel = ""
+		set["buyertel"] = ""
+	}
+	if thisinfo.ContractCode != "" {
+		set["contractcode"] = project.ContractCode + "," + thisinfo.ContractCode
+	}
+	// 代理机构	相同的代理机构才会合并到一个项目 2020.11.9
+	//if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
+	//	pInfo.Agency = thisinfo.Agency
+	//	set["agency"] = thisinfo.Agency
+	//}
+	if len(thisinfo.Topscopeclass) > 0 && thisinfo.Publishtime > pInfo.LastTime {
+		sort.Strings(project.Topscopeclass)
+		for _, k := range thisinfo.Topscopeclass {
+			if BinarySearch(project.Topscopeclass, k) == -1 {
+				project.Topscopeclass = append(project.Topscopeclass, k)
+				sort.Strings(project.Topscopeclass)
+			}
+		}
+		set["topscopeclass"] = project.Topscopeclass
+	}
+	// 项目评审专家
+	if len(thisinfo.ReviewExperts) > 0 && thisinfo.Publishtime > pInfo.LastTime {
+		set["review_experts"] = thisinfo.ReviewExperts
+		project.ReviewExperts = thisinfo.ReviewExperts
+	}
+	if thisinfo.Purchasing != "" && thisinfo.Publishtime > pInfo.LastTime {
+		if project.Purchasing == "" {
+			project.Purchasing = thisinfo.Purchasing
+			set["purchasing"] = thisinfo.Purchasing
+		} else {
+			list := strings.Split(project.Purchasing, ",")
+			for _, k := range list {
+				if BinarySearch(strings.Split(thisinfo.Purchasing, ","), k) == -1 {
+					list = append(list, k)
+					sort.Strings(list)
+				}
+			}
+			set["purchasing"] = strings.Join(list, ",")
+		}
+	}
+	//中标候选人
+	if len(thisinfo.WinnerOrder) > 0 && thisinfo.Publishtime > pInfo.LastTime {
+		var list = []string{}
+		for _, v := range thisinfo.WinnerOrder {
+			if BinarySearch(list, util.ObjToString(v["entname"])) == -1 {
+				list = append(list, util.ObjToString(v["entname"]))
+			}
+		}
+		set["winnerorder"] = list
+		project.Winnerorder = list
+	}
+
+	if len(thisinfo.Subscopeclass) > 0 && thisinfo.Publishtime > pInfo.LastTime {
+		sort.Strings(project.Subscopeclass)
+		for _, k := range thisinfo.Subscopeclass {
+			if BinarySearch(project.Subscopeclass, k) == -1 {
+				project.Subscopeclass = append(project.Subscopeclass, k)
+				sort.Strings(project.Subscopeclass)
+			}
+		}
+		set["subscopeclass"] = project.Subscopeclass
+		set["s_subscopeclass"] = strings.Join(project.Subscopeclass, ",")
+	}
+
+	if len(thisinfo.Winners) > 0 && thisinfo.Publishtime > pInfo.LastTime {
+		winners := strings.Split(project.Winners, ",")
+		if len(winners) <= 0 {
+			set["winner"] = util.ObjToString(tmp["winner"])
+		}
+		for _, k := range thisinfo.Winners {
+			if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+				if BinarySearch(winners, k) != -1 {
+					deleteSlice(winners, k, "")
+					sort.Strings(winners)
+				}
+			} else {
+				if BinarySearch(winners, k) == -1 {
+					winners = append(winners, k)
+					sort.Strings(winners)
+				}
+			}
+		}
+		set["s_winner"] = strings.Join(winners, ",")
+	}
+
+	if thisinfo.HasPackage { //多包处理
+		set["multipackage"] = 1
+		pkg := PackageFormat(thisinfo, project)
+		project.Package = pkg
+		set["package"] = project.Package
+	}
+	//处理多包后,计算预算金额、中标金额
+	CountAmount(project, thisinfo, tmp)
+	if project.Budget >= 0 && project.Budgettag != 1 {
+		pInfo.Budget = project.Budget
+		set["budget"] = pInfo.Budget
+		set["budgettag"] = 0
+	}
+	if pInfo.Bidamount >= 0 && project.Bidamounttag != 1 {
+		pInfo.Bidamount = project.Bidamount
+		set["bidamount"] = pInfo.Bidamount
+		set["bidamounttag"] = 0
+	}
+	if pInfo.Bidamount > 0 {
+		set["sortprice"] = pInfo.Bidamount
+	} else {
+		if pInfo.Budget > 0 {
+			set["sortprice"] = pInfo.Budget
+		}
+	}
+
+	infofield := InfoField{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    pInfo.Bidstatus,
+	}
+	copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
+	copyMap[thisinfo.Id] = infofield
+	tmpMap := make(map[string]interface{})
+	for k, v := range copyMap {
+		tmpMap[k] = StructToMap(v)
+	}
+	tmpMap[thisinfo.Id] = StructToMap(infofield)
+	project.InfoFiled = copyMap
+	set["infofield"] = tmpMap
+
+	set["mpn"] = pInfo.MPN
+	set["mpc"] = pInfo.MPC
+	set["updatetime"] = p.pici
+	redis.PutCKV("project", project.Id.Hex(), set)
+	update := map[string]interface{}{}
+	if len(set) > 0 {
+		update["$set"] = set
+	}
+	push := p.PushListInfo(tmp, thisinfo.Id)
+	push["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	push["compareStr"] = comStr
+	push["resVal"] = pInfo.resVal
+	push["pjVal"] = pInfo.pjVal
+	update["$push"] = map[string]interface{}{
+		"list": push,
+		"ids":  thisinfo.Id,
+	}
+	bol := MongoTool.UpdateById(ProjectColl, pInfo.Id.Hex(), update)
+	if bol {
+		by, _ := json.Marshal(map[string]interface{}{
+			"query": map[string]interface{}{
+				"_id": bson.M{
+					"$gte": pInfo.Id.Hex(),
+					"$lte": pInfo.Id.Hex(),
+				}},
+			"stype": "project",
+		})
+		util.Debug(string(by))
+		_ = udpclient.WriteUdp(by, udp.OP_TYPE_DATA, toaddr[1])
+	}
+}
+
+// 内部合并时,原有项目id基础上新建项目
+func (p *ProjectTask) newPro(tmp, tmpPro map[string]interface{}, thisinfo *Info) *ProjectCache {
+	for _, f := range FIELDS {
+		if tmp[f] != nil && tmp[f] != "" {
+			tmpPro[f] = tmp[f]
+		}
+	}
+	bidopentime := util.Int64All(tmp["bidopentime"])
+	if bidopentime > 0 {
+		tmpPro["bidopentime"] = bidopentime
+	}
+	//异常标记
+	if thisinfo.TopType != "招标" && thisinfo.TopType != "拟建" && thisinfo.TopType != "预告" {
+		tmpPro["exception"] = 1
+	}
+	//projecthref保存
+	if jsonData, ok := tmp["jsondata"].(map[string]interface{}); ok {
+		if jsonData != nil && util.ObjToString(jsonData["projecthref"]) != "" {
+			tmpPro["projecthref"] = jsonData["projecthref"]
+		}
+	}
+	//合同编号
+	if thisinfo.ContractCode != "" {
+		tmpPro["contractcode"] = thisinfo.ContractCode
+	}
+	bs, bt := p.GetBidTypeAndBidStatus(thisinfo)
+	tmpPro["bidtype"] = bs
+	tmpPro["bidstatus"] = bt
+	pkg := PackageFormat(thisinfo, nil)
+	p1 := p.NewCachePinfo(tmpPro["_id"].(primitive.ObjectID), thisinfo, bs, bt)
+
+	now := time.Now().Unix()
+	tmpPro["createtime"] = now
+	tmpPro["sourceinfoid"] = thisinfo.Id
+	tmpPro["sourceinfourl"] = tmp["href"]
+	tmpPro["firsttime"] = tmp["publishtime"]
+	tmpPro["lasttime"] = tmp["publishtime"]
+	tmpPro["pici"] = tmp["comeintime"]
+	tmpPro["ids"] = []string{thisinfo.Id}
+	if thisinfo.TopType == "招标" {
+		if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
+			tmpPro["zbtime"] = tmp["publishtime"]
+		}
+	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
+		tmpPro["jgtime"] = tmp["publishtime"]
+	}
+	if len(thisinfo.Subscopeclass) > 0 {
+		tmpPro["s_subscopeclass"] = strings.Join(thisinfo.Subscopeclass, ",")
+	}
+	if len(thisinfo.Winners) > 0 {
+		tmpPro["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	}
+	if thisinfo.HasPackage {
+		tmpPro["multipackage"] = 1
+		tmpPro["package"] = pkg
+	} else {
+		tmpPro["multipackage"] = 0
+	}
+	//项目评审专家
+	if len(thisinfo.ReviewExperts) > 0 {
+		tmpPro["review_experts"] = thisinfo.ReviewExperts
+	}
+	//标的物
+	if thisinfo.Purchasing != "" {
+		tmpPro["purchasing"] = thisinfo.Purchasing
+	}
+	//中标候选人
+	if len(thisinfo.WinnerOrder) > 0 {
+		var list = []string{}
+		for _, v := range thisinfo.WinnerOrder {
+			if BinarySearch(list, util.ObjToString(v["entname"])) == -1 {
+				list = append(list, util.ObjToString(v["entname"]))
+			}
+		}
+		tmpPro["winnerorder"] = list
+	}
+	infofield := InfoField{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    bs,
+	}
+	res := StructToMap(infofield)
+	tmpPro["infofield"] = map[string]interface{}{
+		thisinfo.Id: res,
+	}
+	if tmp["budget"] != nil && tmp["budget"] != "" {
+		tmpPro["budget"] = thisinfo.Budget
+		tmpPro["budgettag"] = 0
+	} else {
+		tmpPro["budgettag"] = 1
+	}
+	if tmp["bidamount"] != nil && tmp["bidamount"] != "" {
+		tmpPro["bidamount"] = thisinfo.Bidamount
+		tmpPro["bidamounttag"] = 0
+	} else {
+		tmpPro["bidamounttag"] = 1
+	}
+	if p1.Bidamount > 0 {
+		tmpPro["sortprice"] = p1.Bidamount
+	} else if p1.Budget > 0 {
+		tmpPro["sortprice"] = p1.Budget
+	}
+	push := p.PushListInfo(tmp, thisinfo.Id)
+	push["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	tmpPro["list"] = []map[string]interface{}{
+		push,
+	}
+	return &p1
+}
+
+// 合并字段到老项目中
+func (p *ProjectTask) updateOldProField(pInfo *ProjectCache, thisinfo *Info, tmpPro, tmp, m1 map[string]interface{}) {
+	pdata := redis.Get("project", pInfo.Id.Hex())
+	projectMap := pdata.(map[string]interface{})
+	bys, _ := json.Marshal(projectMap)
+	var project *Project
+	err := json.Unmarshal(bys, &project)
+	if err != nil {
+		util.Debug("project Unmarshal err,", err)
+		return
+	}
+	if len(pInfo.Ids) > 30 {
+		//异常标记
+		tmpPro["listtag"] = 1
+	}
+	if thisinfo.Publishtime > pInfo.LastTime {
+		pInfo.LastTime = thisinfo.Publishtime
+		project.LastTime = thisinfo.Publishtime
+		tmpPro["lasttime"] = thisinfo.Publishtime
+	}
+	if thisinfo.TopType == "招标" {
+		if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" && project.Zbtime <= 0 {
+			tmpPro["zbtime"] = tmp["publishtime"]
+		}
+	} else if thisinfo.TopType == "结果" {
+		if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+			if project.Jgtime > 0 {
+				jg1 := int64(math.Abs(float64(project.Jgtime - thisinfo.Publishtime)))
+				//公告状态和项目状态同样都是中标或者成交,
+				if (thisinfo.SubType == "中标" || thisinfo.SubType == "成交") && (pInfo.Bidstatus == "中标" || pInfo.Bidstatus == "成交") {
+					if jg1 > p.jgTime {
+						tmpPro["jgtime"] = tmp["publishtime"]
+						project.Jgtime = thisinfo.Publishtime
+					}
+				} else if (thisinfo.SubType == "流标" || thisinfo.SubType == "废标") && (pInfo.Bidstatus == "流标" || pInfo.Bidstatus == "废标") {
+					//公告状态和项目状态同样是流标或者废标
+					if jg1 > p.jgTime {
+						tmpPro["jgtime"] = tmp["publishtime"]
+						project.Jgtime = thisinfo.Publishtime
+					}
+				}
+			} else {
+				tmpPro["jgtime"] = tmp["publishtime"]
+				project.Jgtime = thisinfo.Publishtime
+			}
+		}
+	} else if thisinfo.SubType == "合同" {
+		if pInfo.Bidstatus == "中标" || pInfo.Bidstatus == "成交" || pInfo.Bidstatus == "" {
+			//中标、成交不更新jgtime
+		} else {
+			tmpPro["jgtime"] = tmp["publishtime"]
+			project.Jgtime = thisinfo.Publishtime
+		}
+	}
+	if thisinfo.Bidopentime > project.Bidopentime {
+		project.Bidopentime = thisinfo.Bidopentime
+		tmpPro["bidopentime"] = thisinfo.Bidopentime
+	}
+	// bidtype、bidstatus
+	pInfo.Bidtype, pInfo.Bidstatus = p.GetBidTypeAndBidStatus(thisinfo)
+	project.Bidtype, project.Bidstatus = p.GetBidTypeAndBidStatus(thisinfo)
+	tmpPro["bidtype"] = pInfo.Bidtype
+	tmpPro["bidstatus"] = pInfo.Bidstatus
+
+	//相同城市的公告才会合并到一起(全国列外)
+	if thisinfo.Area != "全国" {
+		pInfo.Area = thisinfo.Area
+		project.Area = thisinfo.Area
+		tmpPro["area"] = thisinfo.Area
+		pInfo.City = thisinfo.City
+		project.City = thisinfo.City
+		tmpPro["city"] = thisinfo.City
+		if thisinfo.District != "" {
+			pInfo.District = thisinfo.District
+			project.District = thisinfo.District
+			tmpPro["district"] = thisinfo.District
+		}
+	}
+	// 项目名称
+	if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
+		pInfo.ProjectName = thisinfo.ProjectName
+		project.ProjectName = thisinfo.ProjectName
+		tmpPro["projectname"] = thisinfo.ProjectName
+	}
+	// 项目编号
+	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+		pInfo.ProjectCode = thisinfo.ProjectCode
+		project.ProjectCode = thisinfo.ProjectCode
+		tmpPro["projectcode"] = thisinfo.ProjectCode
+	}
+	// 采购单位
+	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
+		pInfo.Buyer = thisinfo.Buyer
+		tmpPro["buyer"] = thisinfo.Buyer
+		project.Buyerclass = thisinfo.Buyerclass
+		tmpPro["buyerclass"] = thisinfo.Buyerclass
+	}
+	if pInfo.Buyer == "" {
+		project.Buyerclass = ""
+		tmpPro["buyerclass"] = ""
+	}
+	// 采购单位联系人
+	if thisinfo.Buyerperson != "" {
+		project.Buyerperson = thisinfo.Buyerperson
+		tmpPro["buyerperson"] = thisinfo.Buyerperson
+	} else {
+		project.Buyerperson = ""
+		tmpPro["buyerperson"] = ""
+	}
+	// 采购单位電話
+	if thisinfo.Buyertel != "" {
+		project.Buyertel = thisinfo.Buyertel
+		tmpPro["buyertel"] = thisinfo.Buyertel
+	} else {
+		project.Buyertel = ""
+		tmpPro["buyertel"] = ""
+	}
+	if thisinfo.ContractCode != "" {
+		tmpPro["contractcode"] = project.ContractCode + "," + thisinfo.ContractCode
+	}
+	// 代理机构	相同的代理机构才会合并到一个项目 2020.11.9
+	//if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
+	//	pInfo.Agency = thisinfo.Agency
+	//	set["agency"] = thisinfo.Agency
+	//}
+	if len(thisinfo.Topscopeclass) > 0 {
+		sort.Strings(project.Topscopeclass)
+		for _, k := range thisinfo.Topscopeclass {
+			if BinarySearch(project.Topscopeclass, k) == -1 {
+				project.Topscopeclass = append(project.Topscopeclass, k)
+				sort.Strings(project.Topscopeclass)
+			}
+		}
+		tmpPro["topscopeclass"] = project.Topscopeclass
+	}
+	// 项目评审专家
+	if len(thisinfo.ReviewExperts) > 0 {
+		tmpPro["review_experts"] = thisinfo.ReviewExperts
+		project.ReviewExperts = thisinfo.ReviewExperts
+	}
+	if thisinfo.Purchasing != "" {
+		if project.Purchasing == "" {
+			project.Purchasing = thisinfo.Purchasing
+			tmpPro["purchasing"] = thisinfo.Purchasing
+		} else {
+			list := strings.Split(project.Purchasing, ",")
+			for _, k := range list {
+				if BinarySearch(strings.Split(thisinfo.Purchasing, ","), k) == -1 {
+					list = append(list, k)
+					sort.Strings(list)
+				}
+			}
+			tmpPro["purchasing"] = strings.Join(list, ",")
+		}
+	}
+	//中标候选人
+	if len(thisinfo.WinnerOrder) > 0 {
+		var list = []string{}
+		for _, v := range thisinfo.WinnerOrder {
+			if BinarySearch(list, util.ObjToString(v["entname"])) == -1 {
+				list = append(list, util.ObjToString(v["entname"]))
+			}
+		}
+		tmpPro["winnerorder"] = list
+		project.Winnerorder = list
+	}
+
+	if len(thisinfo.Subscopeclass) > 0 {
+		sort.Strings(project.Subscopeclass)
+		for _, k := range thisinfo.Subscopeclass {
+			if BinarySearch(project.Subscopeclass, k) == -1 {
+				project.Subscopeclass = append(project.Subscopeclass, k)
+				sort.Strings(project.Subscopeclass)
+			}
+		}
+		tmpPro["subscopeclass"] = project.Subscopeclass
+		tmpPro["s_subscopeclass"] = strings.Join(project.Subscopeclass, ",")
+	}
+
+	if len(thisinfo.Winners) > 0 {
+		if len(project.Winners) <= 0 {
+			tmpPro["winner"] = util.ObjToString(tmp["winner"])
+		}
+		winners := strings.Split(project.Winners, ",")
+		for _, k := range thisinfo.Winners {
+			if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+				if BinarySearch(winners, k) != -1 {
+					deleteSlice(winners, k, "")
+				}
+			} else {
+				if BinarySearch(winners, k) == -1 {
+					winners = append(winners, k)
+				}
+			}
+		}
+		tmpPro["s_winner"] = strings.Join(winners, ",")
+	}
+
+	if thisinfo.HasPackage { //多包处理
+		tmpPro["multipackage"] = 1
+		pkg := PackageFormat(thisinfo, project)
+		project.Package = pkg
+		tmpPro["package"] = project.Package
+	}
+	//处理多包后,计算预算金额、中标金额
+	CountAmount(project, thisinfo, tmp)
+	if project.Budget >= 0 && project.Budgettag != 1 {
+		pInfo.Budget = project.Budget
+		tmpPro["budget"] = pInfo.Budget
+		tmpPro["budgettag"] = 0
+	}
+	if project.Bidamount >= 0 && project.Bidamounttag != 1 {
+		pInfo.Bidamount = project.Bidamount
+		tmpPro["bidamount"] = pInfo.Bidamount
+		tmpPro["bidamounttag"] = 0
+	}
+	if pInfo.Bidamount >= pInfo.Budget {
+		tmpPro["sortprice"] = pInfo.Bidamount
+	} else if pInfo.Budget >= pInfo.Bidamount {
+		tmpPro["sortprice"] = pInfo.Budget
+	}
+
+	infofield := InfoField{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    pInfo.Bidstatus,
+	}
+	copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
+	copyMap[thisinfo.Id] = infofield
+	tmpMap := make(map[string]interface{})
+	for k, v := range copyMap {
+		tmpMap[k] = StructToMap(v)
+	}
+	tmpMap[thisinfo.Id] = StructToMap(infofield)
+	project.InfoFiled = copyMap
+	tmpPro["infofield"] = tmpMap
+	tmpPro["mpn"] = pInfo.MPN
+	tmpPro["mpc"] = pInfo.MPC
+	tmpPro["updatetime"] = p.pici
+
+	push := p.PushListInfo(tmp, thisinfo.Id)
+	push["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	push["compareStr"] = m1["compareStr"]
+	push["resVal"] = m1["resVal"]
+	push["pjVal"] = m1["pjVal"]
+	list := tmpPro["list"].([]map[string]interface{})
+	tmpPro["list"] = append(list, push)
+	tmpPro["ids"] = append(tmpPro["ids"].([]string), thisinfo.Id)
+}
+
+func deleteSlice1(arr []interface{}, v interface{}) []interface{} {
+	for k, v1 := range arr {
+		if reflect.DeepEqual(v1, v) {
+			return append(arr[:k], arr[k+1:]...)
+		}
+	}
+	return arr
+}
+
+//	修改字段值
+func UpdateValue(proMap map[string]interface{}, index, position int, modifyMap map[string]interface{}) map[string]interface{} {
+	updataSet := make(map[string]interface{})
+	infoList := proMap["list"].([]interface{})
+	tempMap := infoList[position].(map[string]interface{})
+
+	for k := range modifyMap {
+		//项目中lsit大小等于1 或者 修改的招标公告处于项目的list中最后一个
+		if len(infoList) == 1 || index == 2 {
+			if k == "budget" || k == "bidamount" {
+				if proMap["sortprice"] == proMap[k] {
+					updataSet["sortprice"] = modifyMap[k]
+				}
+				updataSet[k] = modifyMap[k]
+				proMap[k] = modifyMap[k]
+				tempMap[k] = modifyMap[k]
+			} else {
+				updataSet[k] = modifyMap[k]
+				proMap[k] = modifyMap[k]
+				tempMap[k] = modifyMap[k]
+			}
+
+		} else {
+			tempMap[k] = modifyMap[k]
+			if k == "budget" || k == "bidamount" {
+				if proMap["sortprice"] == proMap[k] {
+					proMap["sortprice"] = modifyMap[k]
+				}
+				updataSet[k] = modifyMap[k]
+				proMap[k] = modifyMap[k]
+			}
+			// 有条件更新项目外围字段值(非空)
+			if proMap[k] == nil || util.ObjToString(k) == "" {
+				updataSet[k] = modifyMap[k]
+				proMap[k] = modifyMap[k]
+			}
+		}
+	}
+	return updataSet
+}
+
+//备份(快照)
+func backupPro(tmp map[string]interface{}) {
+	tmp1 := make(map[string]interface{})
+	for k, v := range tmp {
+		tmp1[k] = v
+	}
+	if Sysconfig["backupFlag"].(bool) {
+		tmp1["sourceprojectid"] = mongodb.BsonIdToSId(tmp1["_id"])
+		delete(tmp1, "_id")
+		MongoTool.Save(BackupColl, tmp1)
+	}
+}
+
+// 删除原有项目数据
+func delOldPro(pid string) {
+	t := MongoTool.Delete(ProjectColl, map[string]interface{}{"_id": mongodb.StringTOBsonId(pid)})
+	if t >= 0 {
+		client := Es.GetEsConn()
+		defer Es.DestoryEsConn(client)
+		Es.DelById(Index, pid)
+	}
+}
+
+// 修改内存中的数据
+func (p *ProjectTask) modifyMem(tmpPro map[string]interface{}) {
+	pid := util.ObjToString(tmpPro["_id"])
+	var pro ProjectCache
+	err := mapstructure.Decode(tmpPro, &pro)
+	if err != nil {
+		util.Debug(err)
+	}
+	pro.Id = mongodb.StringTOBsonId(pid)
+	tmpMap := make(map[string]InfoField)
+	infoMap := tmpPro["infofield"].(map[string]interface{})
+	for _, v := range infoMap {
+		var field InfoField
+		b, _ := json.Marshal(v)
+		_ = json.Unmarshal(b, &field)
+		tmpMap[pid] = field
+	}
+	p.AllIdsMapLock.Lock()
+	if v, ok := p.AllIdsMap[pid]; ok {
+		v.P = &pro
+	}
+	p.AllIdsMapLock.Unlock()
+	p.printMemPro(pid)
+}
+
+// 打印内存中的项目信息
+func (p *ProjectTask) printMemPro(pid string) {
+	p.AllIdsMapLock.Lock()
+	if v, ok := p.AllIdsMap[pid]; ok {
+		util.Debug("mem pro ----------", *v.P)
+	}
+	p.AllIdsMapLock.Unlock()
+}
+
+// @Description id不变,内容变化 重新进行项目合并
+// @Author J 2022/8/10 14:51
+func (p *ProjectTask) taskinfo(id string) {
+	tmpPro, _ := MongoTool.FindOneByField(ProjectColl, map[string]interface{}{"ids": id}, nil)
+	if tmpPro == nil || len(*tmpPro) == 0 {
+		util.Debug(fmt.Sprintf("taskinfo bidding id=%s 未查询到项目数据", id))
+		return
+	}
+	pid := mongodb.BsonIdToSId((*tmpPro)["_id"])
+	ids := util.ObjArrToStringArr((*tmpPro)["ids"].([]interface{}))
+	list := (*tmpPro)["list"].([]interface{})
+
+	var newP map[string]interface{}
+	var p1 *ProjectCache
+	for i, s := range ids {
+		temp, _ := MongoTool.FindById(ExtractColl, s, nil)
+		if temp == nil || len(*temp) == 0 {
+			temp, _ = MongoTool.FindById(ExtractColl1, s, nil)
+			if temp == nil || len(*temp) == 0 {
+				util.Debug("extract not find id...", s)
+				return
+			}
+		}
+		push := p.PushListInfo(*temp, id)
+		if s == id {
+			list[i] = push
+		}
+
+		info := ParseInfo(*temp)
+		if i == 0 {
+			newP, p1 = p.createPro(*temp, info, pid)
+		} else {
+			p.updatePro(newP, *temp, p1, info)
+		}
+	}
+
+	newP["ids"] = ids
+	newP["pici"] = p.pici
+	p.AllIdsMapLock.Lock()
+	_, ok := p.AllIdsMap[pid]
+	if ok {
+		redis.PutCKV("project", pid, newP)
+		p.AllIdsMap[id] = &ID{Id: id, P: p1}
+	}
+	p.AllIdsMapLock.Unlock()
+	newP["list"] = list
+	if len(newP) > 0 {
+		updateInfo := []map[string]interface{}{
+			{
+				"_id": mongodb.StringTOBsonId(pid),
+			},
+			{"$set": newP},
+		}
+		p.updatePool <- updateInfo
+	}
+}
+
+// @Description 创建项目
+// @Author J 2022/8/11 09:17
+func (p *ProjectTask) createPro(tmp map[string]interface{}, thisinfo *Info, pid string) (map[string]interface{}, *ProjectCache) {
+	set := map[string]interface{}{}
+	set["_id"] = mongodb.StringTOBsonId(pid)
+	for _, f := range FIELDS {
+		if tmp[f] != nil && tmp[f] != "" {
+			set[f] = tmp[f]
+		}
+	}
+	bidopentime := util.Int64All(tmp["bidopentime"])
+	if bidopentime > 0 {
+		set["bidopentime"] = bidopentime
+	}
+	//异常标记
+	if thisinfo.TopType != "招标" && thisinfo.TopType != "拟建" && thisinfo.TopType != "预告" {
+		set["exception"] = 1
+	}
+	//projecthref保存
+	if jsonData, ok := tmp["jsondata"].(map[string]interface{}); ok {
+		if jsonData != nil && util.ObjToString(jsonData["projecthref"]) != "" {
+			set["projecthref"] = jsonData["projecthref"]
+		}
+	}
+	//合同编号
+	if thisinfo.ContractCode != "" {
+		set["contractcode"] = thisinfo.ContractCode
+	}
+
+	bt := util.ObjToString(tmp["toptype"])
+	bs := util.ObjToString(tmp["subtype"])
+	p.mapBidLock.Lock()
+	if thisinfo.Infoformat == 2 || thisinfo.SubType == "拟建" {
+		set["bidstatus"] = "拟建"
+		bt = "拟建"
+	} else {
+		if bidtype[bs] != "" {
+			set["bidtype"] = bidtype[bs]
+		} else {
+			set["bidtype"] = "招标"
+		}
+		if bt == "招标" {
+			set["projectscope"] = util.ObjToString(tmp["projectscope"])
+			set["bidstatus"] = bt
+		} else {
+			if bidstatus[bs] != "" {
+				set["bidstatus"] = thisinfo.SubType
+				bt = thisinfo.SubType
+			} else if bs == "" {
+				set["bidstatus"] = ""
+				bt = ""
+			} else {
+				set["bidstatus"] = "其它"
+				bt = "其它"
+			}
+		}
+	}
+	p.mapBidLock.Unlock()
+
+	pkg := PackageFormat(thisinfo, nil)
+	p1 := p.NewCachePinfo(mongodb.StringTOBsonId(pid), thisinfo, bs, bt)
+
+	now := time.Now().Unix()
+	set["createtime"] = now
+	set["sourceinfoid"] = thisinfo.Id
+	set["sourceinfourl"] = tmp["href"]
+	set["firsttime"] = tmp["publishtime"]
+	set["lasttime"] = tmp["publishtime"]
+	//增量用系统时间,全量(历史)入库时间
+	if p.currentType == "pl" {
+		set["pici"] = tmp["comeintime"]
+	} else {
+		set["pici"] = p.pici
+	}
+	set["ids"] = []string{thisinfo.Id}
+	if thisinfo.TopType == "招标" {
+		//if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
+		set["zbtime"] = tmp["publishtime"]
+		//}
+	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
+		set["jgtime"] = tmp["publishtime"]
+	}
+	if len(thisinfo.Subscopeclass) > 0 {
+		set["s_subscopeclass"] = strings.Join(thisinfo.Subscopeclass, ",")
+	}
+	if len(thisinfo.Winners) > 0 {
+		set["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	}
+	if thisinfo.HasPackage {
+		set["multipackage"] = 1
+		set["package"] = pkg
+	} else {
+		set["multipackage"] = 0
+	}
+	//项目评审专家
+	if len(thisinfo.ReviewExperts) > 0 {
+		set["review_experts"] = thisinfo.ReviewExperts
+	}
+	//标的物
+	if thisinfo.Purchasing != "" {
+		list := Duplicate(strings.Split(thisinfo.Purchasing, ",")) //标的物 去重 03/03
+		p := strings.Join(util.ObjArrToStringArr(list), ",")
+		set["purchasing"] = p
+	}
+	//中标候选人
+	if len(thisinfo.WinnerOrder) > 0 {
+		var list = []string{}
+		for _, v := range thisinfo.WinnerOrder {
+			if BinarySearch(list, util.ObjToString(v["entname"])) == -1 {
+				list = append(list, util.ObjToString(v["entname"]))
+			}
+		}
+		set["winnerorder"] = list
+	}
+	//项目规模
+	if len(thisinfo.ProjectScale) > 0 {
+		set["project_scale"] = thisinfo.ProjectScale
+	}
+	//工期时长
+	if thisinfo.ProjectDuration > 0 {
+		set["project_duration"] = thisinfo.ProjectDuration
+	}
+	// 工期单位
+	if thisinfo.ProjectDuration > 0 && len(thisinfo.ProjectTimeUnit) > 0 {
+		set["project_timeunit"] = thisinfo.ProjectTimeUnit
+	}
+	//开工日期
+	if thisinfo.ProjectStartDate > 0 {
+		set["project_startdate"] = thisinfo.ProjectStartDate
+	}
+	//竣工日期
+	if thisinfo.ProjectCompleteDate > 0 {
+		set["project_completedate"] = thisinfo.ProjectCompleteDate
+	}
+	//付款方式
+	if len(thisinfo.Payway) > 0 {
+		set["payway"] = thisinfo.Payway
+	}
+	// 履约保证金
+	if tmp["contract_guarantee"] != nil {
+		set["contract_guarantee"] = thisinfo.ContractGuarantee
+	}
+	// 投标保证金
+	if tmp["bid_guarantee"] != nil {
+		set["bid_guarantee"] = thisinfo.BidGuarantee
+	}
+	// 资质条件
+	if len(thisinfo.Qualifies) > 0 {
+		var str []string
+		for _, v := range thisinfo.Qualifies {
+			if len(util.ObjToString(v["key"])) > 0 {
+				if BinarySearch(str, util.ObjToString(v["key"])) == -1 {
+					str = append(str, util.ObjToString(v["key"]))
+				}
+			}
+		}
+		if len(str) > 0 {
+			set["qualifies"] = strings.Join(str, ",")
+		}
+	}
+	if len(thisinfo.EntIdList) > 0 {
+		set["entidlist"] = thisinfo.EntIdList
+	}
+	// first_cooperation
+	if p1.Buyer != "" && len(thisinfo.Winners) > 0 {
+		FirstCooperation(set, p1.Buyer, thisinfo.Winners, thisinfo.EntIdList)
+	}
+
+	infofield := InfoField{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    bs,
+	}
+	res := StructToMap(infofield)
+	set["infofield"] = map[string]interface{}{
+		thisinfo.Id: res,
+	}
+	if tmp["budget"] != nil && tmp["budget"] != "" {
+		set["budget"] = thisinfo.Budget
+		set["budgettag"] = 0
+	} else {
+		set["budgettag"] = 1
+	}
+	if tmp["bidamount"] != nil && tmp["bidamount"] != "" {
+		set["bidamount"] = thisinfo.Bidamount
+		set["bidamounttag"] = 0
+	} else {
+		set["bidamounttag"] = 1
+	}
+	if p1.Bidamount > 0 {
+		set["sortprice"] = p1.Bidamount
+	} else {
+		if p1.Budget > 0 {
+			set["sortprice"] = p1.Budget
+		}
+	}
+	return set, &p1
+}
+
+// @Description id不变,内容覆盖 修改项目
+// @Author J 2022/8/10 17:00
+func (p *ProjectTask) updatePro(set, tmp map[string]interface{}, pInfo *ProjectCache, thisinfo *Info) {
+	bys, _ := json.Marshal(set)
+	var project *Project
+	err := json.Unmarshal(bys, &project)
+	if err != nil {
+		util.Debug("project Unmarshal err,", err)
+		return
+	}
+
+	//zbtime、lasttime、jgtime
+	pInfo.LastTime = thisinfo.Publishtime
+	set["lasttime"] = thisinfo.Publishtime
+
+	if thisinfo.TopType == "招标" {
+		if project.Zbtime <= 0 {
+			project.Zbtime = thisinfo.Publishtime
+			set["zbtime"] = tmp["publishtime"]
+		}
+		if project.Jgtime > 0 {
+			project.Jgtime = int64(0)
+			set["jgtime"] = int64(0)
+		}
+	} else if thisinfo.TopType == "结果" {
+		if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+			if project.Jgtime > 0 {
+				//jg1 := int64(math.Abs(float64(pInfo.Jgtime - thisinfo.Publishtime)))
+				//公告状态和项目状态同样都是中标或者成交,
+				if thisinfo.SubType == "成交" && project.Bidstatus == "中标" {
+					if p.jgTime < thisinfo.Publishtime {
+						project.Jgtime = thisinfo.Publishtime
+						set["jgtime"] = tmp["publishtime"]
+					}
+					//公告状态和项目状态同样是流标或者废标
+				} else if (thisinfo.SubType == "流标" || thisinfo.SubType == "废标") && (project.Bidstatus == "流标" || project.Bidstatus == "废标") {
+					if p.jgTime < thisinfo.Publishtime {
+						project.Jgtime = thisinfo.Publishtime
+						set["jgtime"] = tmp["publishtime"]
+					}
+				}
+			} else {
+				project.Jgtime = thisinfo.Publishtime
+				set["jgtime"] = tmp["publishtime"]
+			}
+		}
+	} else if thisinfo.SubType == "合同" {
+		if project.Bidstatus == "中标" || project.Bidstatus == "成交" || project.Bidstatus == "" {
+			//中标、成交不更新jgtime
+		} else {
+			project.Jgtime = thisinfo.Publishtime
+			set["jgtime"] = tmp["publishtime"]
+		}
+	}
+	if thisinfo.Bidopentime > project.Bidopentime {
+		project.Bidopentime = thisinfo.Bidopentime
+		set["bidopentime"] = project.Bidopentime
+	}
+
+	bt := util.ObjToString(tmp["toptype"])
+	bs := util.ObjToString(tmp["subtype"])
+	p.mapBidLock.Lock()
+	if bt == "招标" {
+		//招标状态,更新projectscope
+		if tmp["projectscope"] != nil {
+			set["projectscope"] = util.ObjToString(tmp["projectscope"])
+		}
+		set["bidstatus"] = bt
+		project.Bidstatus = bt
+		if bidtype[bs] != "" {
+			set["bidtype"] = bidtype[bs]
+			project.Bidtype = bidtype[bs]
+		} else {
+			set["bidtype"] = "招标"
+			project.Bidtype = "招标"
+		}
+	} else {
+		if bidstatus[bs] != "" {
+			set["bidstatus"] = thisinfo.SubType
+			project.Bidstatus = thisinfo.SubType
+		} else if thisinfo.Infoformat == 2 {
+			set["bidstatus"] = "拟建"
+			project.Bidstatus = "拟建"
+		} else if bs == "" && bt == "结果" {
+			if project.Bidstatus == "招标" {
+				set["bidstatus"] = ""
+				project.Bidstatus = ""
+			}
+		} else {
+			set["bidstatus"] = "其它"
+			project.Bidstatus = "其它"
+		}
+	}
+	p.mapBidLock.Unlock()
+
+	//相同城市的公告才会合并到一起(全国列外)
+	if thisinfo.Area != "全国" {
+		pInfo.Area = thisinfo.Area
+		project.Area = thisinfo.Area
+		set["area"] = thisinfo.Area
+		if thisinfo.City != "" {
+			pInfo.City = thisinfo.City
+			project.City = thisinfo.City
+			set["city"] = thisinfo.City
+		}
+		if thisinfo.District != "" {
+			pInfo.District = thisinfo.District
+			project.District = thisinfo.District
+			set["district"] = thisinfo.District
+		}
+	}
+	//项目名称
+	//if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
+	if pInfo.ProjectName == "" && thisinfo.ProjectName != "" {
+		pInfo.ProjectName = thisinfo.ProjectName
+		project.ProjectName = thisinfo.ProjectName
+		set["projectname"] = thisinfo.ProjectName
+	}
+	//7--项目编号
+	//if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+	if pInfo.ProjectCode == "" && thisinfo.ProjectCode != "" {
+		pInfo.ProjectCode = thisinfo.ProjectCode
+		project.ProjectCode = thisinfo.ProjectCode
+		set["projectcode"] = thisinfo.ProjectCode
+	}
+	//7--采购单位
+	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
+		pInfo.Buyer = thisinfo.Buyer
+		project.Buyer = thisinfo.Buyer
+		set["buyer"] = thisinfo.Buyer
+
+		project.Buyerclass = thisinfo.Buyerclass
+		set["buyerclass"] = thisinfo.Buyerclass
+	}
+	if pInfo.Buyer == "" {
+		project.Buyerclass = ""
+		set["buyerclass"] = ""
+	}
+	//采购单位联系人
+	if thisinfo.Buyerperson != "" {
+		project.Buyerperson = thisinfo.Buyerperson
+		set["buyerperson"] = thisinfo.Buyerperson
+	} else {
+		project.Buyerperson = ""
+		set["buyerperson"] = ""
+	}
+	//采购单位電話
+	if thisinfo.Buyertel != "" {
+		project.Buyertel = thisinfo.Buyertel
+		set["buyertel"] = project.Buyertel
+	} else {
+		project.Buyertel = ""
+		set["buyertel"] = ""
+	}
+	if thisinfo.ContractCode != "" {
+		if project.ContractCode == "" {
+			set["contractcode"] = thisinfo.ContractCode
+		} else {
+			list := strings.Split(project.ContractCode, ",")
+			if BinarySearch(list, thisinfo.ContractCode) == -1 {
+				list = append(list, thisinfo.ContractCode)
+				sort.Strings(list)
+			}
+			set["contractcode"] = strings.Join(list, ",")
+		}
+	}
+
+	//8--代理机构
+	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
+		pInfo.Agency = thisinfo.Agency
+		project.Agency = thisinfo.Agency
+		set["agency"] = thisinfo.Agency
+	}
+
+	if len(thisinfo.Topscopeclass) > 0 {
+		sort.Strings(project.Topscopeclass)
+		for _, k := range thisinfo.Topscopeclass {
+			if BinarySearch(project.Topscopeclass, k) == -1 {
+				project.Topscopeclass = append(project.Topscopeclass, k)
+				sort.Strings(project.Topscopeclass)
+			}
+		}
+		set["topscopeclass"] = project.Topscopeclass
+	}
+
+	//项目评审专家
+	if len(thisinfo.ReviewExperts) > 0 {
+		set["review_experts"] = thisinfo.ReviewExperts
+		project.ReviewExperts = thisinfo.ReviewExperts
+	}
+	if thisinfo.Purchasing != "" {
+		if project.Purchasing == "" {
+			list := Duplicate(strings.Split(thisinfo.Purchasing, ",")) //标的物 去重 03/03
+			p := strings.Join(util.ObjArrToStringArr(list), ",")
+			project.Purchasing = p
+			set["purchasing"] = p
+		} else {
+			list := strings.Split(project.Purchasing, ",")
+			list = util.ObjArrToStringArr(Duplicate(list))
+			for _, k := range list {
+				if BinarySearch(strings.Split(thisinfo.Purchasing, ","), k) == -1 {
+					list = append(list, k)
+					sort.Strings(list)
+				}
+			}
+			set["purchasing"] = strings.Join(list, ",")
+		}
+	}
+
+	//中标候选人
+	if len(thisinfo.WinnerOrder) > 0 {
+		var list = []string{}
+		for _, v := range thisinfo.WinnerOrder {
+			if BinarySearch(list, util.ObjToString(v["entname"])) == -1 {
+				list = append(list, util.ObjToString(v["entname"]))
+			}
+		}
+		set["winnerorder"] = list
+		project.Winnerorder = list
+	}
+
+	if len(thisinfo.Subscopeclass) > 0 {
+		sort.Strings(project.Subscopeclass)
+		for _, k := range thisinfo.Subscopeclass {
+			if BinarySearch(project.Subscopeclass, k) == -1 {
+				project.Subscopeclass = append(project.Subscopeclass, k)
+				sort.Strings(project.Subscopeclass)
+			}
+		}
+		set["subscopeclass"] = project.Subscopeclass
+		set["s_subscopeclass"] = strings.Join(project.Subscopeclass, ",")
+	}
+
+	if len(thisinfo.Winners) > 0 {
+		if len(project.Winners) <= 0 {
+			set["winner"] = util.ObjToString(tmp["winner"])
+		}
+		winners := strings.Split(project.Winners, ",")
+		for _, k := range thisinfo.Winners {
+			if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+				if BinarySearch(winners, k) != -1 {
+					deleteSlice(winners, k, "")
+				}
+			} else {
+				if BinarySearch(winners, k) == -1 {
+					winners = append(winners, k)
+				}
+			}
+		}
+
+		set["s_winner"] = strings.Join(util.ObjArrToStringArr(Duplicate(winners)), ",")
+	}
+
+	if len(thisinfo.EntIdList) > 0 {
+		for _, v := range thisinfo.EntIdList {
+			if BinarySearch(project.EntIdList, v) == -1 {
+				project.EntIdList = append(project.EntIdList, v)
+			}
+		}
+		set["entidlist"] = project.EntIdList
+	}
+	// first_cooperation
+	if pInfo.Buyer != "" && len(strings.Split(project.Winners, ",")) > 0 {
+		FirstCooperation(set, pInfo.Buyer, strings.Split(project.Winners, ","), project.EntIdList)
+	}
+
+	//项目规模
+	if len(thisinfo.ProjectScale) > 0 {
+		project.ProjectScale = thisinfo.ProjectScale
+		set["project_scale"] = thisinfo.ProjectScale
+	}
+	//工期时长
+	if thisinfo.ProjectDuration > 0 {
+		project.ProjectDuration = thisinfo.ProjectDuration
+		set["project_duration"] = thisinfo.ProjectDuration
+	}
+	// 工期单位
+	if thisinfo.ProjectDuration > 0 && len(thisinfo.ProjectTimeUnit) > 0 {
+		project.ProjectTimeunit = thisinfo.ProjectTimeUnit
+		set["project_timeunit"] = thisinfo.ProjectTimeUnit
+	}
+	//开工日期
+	if thisinfo.ProjectStartDate > 0 {
+		if project.ProjectStartDate > 0 {
+			if project.ProjectStartDate < thisinfo.ProjectStartDate {
+				project.ProjectStartDate = thisinfo.ProjectStartDate
+				set["project_startdate"] = thisinfo.ProjectStartDate
+			}
+		} else {
+			project.ProjectStartDate = thisinfo.ProjectStartDate
+			set["project_startdate"] = thisinfo.ProjectStartDate
+		}
+	}
+	//竣工日期
+	if thisinfo.ProjectCompleteDate > 0 {
+		if project.ProjctCompleteDate > 0 {
+			if project.ProjctCompleteDate < thisinfo.ProjectCompleteDate {
+				project.ProjctCompleteDate = thisinfo.ProjectCompleteDate
+				set["project_completedate"] = thisinfo.ProjectCompleteDate
+			}
+		} else {
+			project.ProjctCompleteDate = thisinfo.ProjectCompleteDate
+			set["project_completedate"] = thisinfo.ProjectCompleteDate
+		}
+	}
+	// 付款方式
+	if len(thisinfo.Payway) > 0 {
+		project.Payway = thisinfo.Payway
+		set["payway"] = thisinfo.Payway
+	}
+	// 履约保证金
+	if tmp["contract_guarantee"] != nil {
+		project.ContractGuarantee = thisinfo.ContractGuarantee
+		set["contract_guarantee"] = thisinfo.ContractGuarantee
+	}
+	// 投标保证金
+	if tmp["bid_guarantee"] != nil {
+		project.BidGuarantee = thisinfo.BidGuarantee
+		set["bid_guarantee"] = thisinfo.BidGuarantee
+	}
+	// 资质条件
+	if len(thisinfo.Qualifies) > 0 {
+		var str []string
+		if len(project.Qualifies) > 0 {
+			str = append(str, strings.Split(project.Qualifies, ",")...)
+		}
+		for _, v := range thisinfo.Qualifies {
+			if len(util.ObjToString(v["key"])) > 0 {
+				if BinarySearch(str, util.ObjToString(v["key"])) == -1 {
+					str = append(str, util.ObjToString(v["key"]))
+				}
+			}
+		}
+		if len(str) > 0 {
+			project.Qualifies = strings.Join(str, ",")
+			set["qualifies"] = strings.Join(str, ",")
+		}
+	}
+
+	if thisinfo.HasPackage { //多包处理
+		set["multipackage"] = 1
+		pkg := PackageFormat(thisinfo, project)
+		project.Package = pkg
+		set["package"] = project.Package
+	}
+	//处理多包后,计算预算金额、中标金额
+	CountAmount(project, thisinfo, tmp)
+	if project.Budget >= 0 && project.Budgettag != 1 {
+		pInfo.Budget = project.Budget
+		set["budget"] = project.Budget
+		set["budgettag"] = 0
+	}
+	if project.Bidamount >= 0 && project.Bidamounttag != 1 {
+		pInfo.Bidamount = project.Bidamount
+		set["bidamount"] = pInfo.Bidamount
+		set["bidamounttag"] = 0
+	}
+	if pInfo.Bidamount > 0 {
+		set["sortprice"] = pInfo.Bidamount
+	} else {
+		if pInfo.Budget > 0 {
+			set["sortprice"] = pInfo.Budget
+		}
+	}
+
+	infofield := InfoField{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    bs,
+	}
+
+	tmpMap := set["infofield"].(map[string]interface{})
+	tmpMap[thisinfo.Id] = StructToMap(infofield)
+	set["infofield"] = tmpMap
+	copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
+	copyMap[thisinfo.Id] = infofield
+	project.InfoFiled = copyMap
+
+	set["mpn"] = pInfo.MPN
+	set["mpc"] = pInfo.MPC
+}

+ 0 - 1
data_sync/data_sync

@@ -1 +0,0 @@
-ELF