数据清洗

dataclean/src/config.json

@@ -0,0 +1,25 @@
+{
+    "udpport": "1501",
+    "mongodbs": {
+        "qfw": {
+            "servers": "192.168.3.207:27081",
+            "size": 5,
+            "name": "qfw",
+            "coll": "bidding"
+        },
+        "extract": {
+            "servers": "192.168.3.207:27082",
+            "size": 5,
+            "name": "extract_kf",
+            "coll": "result"
+        },
+        "project": {
+            "servers": "192.168.3.207:27081",
+            "size": 5,
+            "name": "qfw",
+            "coll": "project"
+        }
+    },
+    "es": {},
+    "nextnode": []
+}

dataclean/src/main.go

@@ -0,0 +1,46 @@
+// main
+package main
+
+import (
+	"log"
+	mu "mfw/util"
+	qu "qfw/util"
+	"qfw/util/mongodb"
+	"service"
+)
+
+func init() {
+	qu.ReadConfig(&service.Sysconfig)
+	mongodbs, ok := service.Sysconfig["mongodbs"].(map[string]interface{})
+	if ok {
+		service.Mogdbs = make(map[string]service.MongoTool)
+		for k, v := range mongodbs {
+			if db, ok := v.(map[string]interface{}); ok {
+				mongoTool := mongodb.MongodbSim{
+					MongodbAddr: db["servers"].(string),
+					Size:        qu.IntAll(db["size"]),
+					DbName:      db["name"].(string),
+				}
+				mongoTool.InitPool()
+				mdb := service.MongoTool{
+					Mongodb: mongoTool,
+					Name:    db["name"].(string),
+					Coll:    db["coll"].(string),
+				}
+				service.Mogdbs[k] = mdb
+			}
+		}
+		log.Println("mongodbs 初始化完成")
+	} else {
+		log.Fatal("mongodbs 初始化失败")
+	}
+}
+
+func main() {
+	updport := service.Sysconfig["udpport"].(string)
+	service.Udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
+	service.Udpclient.Listen(service.ProcessUdpMsg)
+	log.Println("Udp服务监听", updport)
+	lock := make(chan bool)
+	<-lock
+}

dataclean/src/service/bidamount.go

@@ -0,0 +1,18 @@
+// bidamount,中标金额清洗
+package service
+
+import (
+	"log"
+)
+
+//返回
+func (d *ResultInfo) BidAmount(data map[string]interface{}) {
+	log.Println("执行：BidAmout")
+	state := false //值未发生改变
+	//todo 清洗操作
+
+	if state {
+		d.Flag++
+		d.Extract.UpParam = append(d.Extract.UpParam, "bidamount") //更新属性
+	}
+}

dataclean/src/service/budget.go

@@ -0,0 +1,2 @@
+// budget
+package service

dataclean/src/service/fieldmain.go

@@ -0,0 +1,73 @@
+// fieldmain
+package service
+
+import (
+	"log"
+	qu "qfw/util"
+	"sync"
+	"time"
+)
+
+func init() {
+	ResultInfos = []*ResultInfo{}
+}
+
+func DataClean(udpInfo map[string]interface{}) {
+	defer qu.Catch()
+	gtid := udpInfo["gtid"].(string)
+	lteid := udpInfo["lteid"].(string)
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  qu.StringTOBsonId(gtid),
+			"$lte": qu.StringTOBsonId(lteid),
+		},
+	}
+	mdb := Mogdbs["extract"]
+	sess := mdb.Mongodb.GetMgoConn()
+	defer mdb.Mongodb.DestoryMongoConn(sess)
+	query := sess.DB(mdb.Name).C(mdb.Coll).Find(q).Iter()
+	index := 0
+	wg := &sync.WaitGroup{}
+	for tmp := make(map[string]interface{}); query.Next(tmp); index++ {
+		//清洗数据流开始
+		ClearThread <- true
+		go func(data map[string]interface{}) {
+			defer func() {
+				<-ClearThread
+				wg.Done()
+			}()
+			Fields(data)
+		}(tmp)
+		tmp = make(map[string]interface{})
+		if index%1000 == 0 {
+			log.Println("清洗数据量：", index)
+		}
+	}
+	wg.Wait()
+	log.Println("清洗数据量：", index)
+	time.Sleep(5 * time.Second)
+	for {
+		if len(UpResults) < 1 {
+			break
+		}
+	}
+	//清洗完成，next node
+	if nextnodes, ok := Sysconfig["nextnode"].([]interface{}); ok {
+		for _, nextnode := range nextnodes {
+			if tmp, ok := nextnode.(map[string]interface{}); ok {
+				//todo 发送通知
+				log.Println(tmp)
+			}
+		}
+	}
+}
+
+func Fields(tmp map[string]interface{}) {
+	data := &ResultInfo{}
+	data.BidAmount(tmp)
+	//其他字段清洗...
+
+	if data.Flag > 0 { //有字段发生变化
+		ResultInfos = append(ResultInfos, data)
+	}
+}

dataclean/src/service/udpMsg.go

@@ -0,0 +1,39 @@
+// udpMsg
+package service
+
+import (
+	"encoding/json"
+	"log"
+	mu "mfw/util"
+	"net"
+)
+
+var (
+	UdpMsgThread = make(chan bool, 1)
+	Udpclient    mu.UdpClient //udp对象
+
+)
+
+//udp调用信号
+func ProcessUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case mu.OP_TYPE_DATA: //上个节点的数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Println("err:", err, "mapInfo:", mapInfo)
+		//todo
+		UdpMsgThread <- true
+		go func() {
+			defer func() {
+				<-UdpMsgThread
+			}()
+			//清洗数据操作
+			DataClean(mapInfo)
+		}()
+	case mu.OP_NOOP: //下个节点回应
+		ok := string(data)
+		if ok != "" {
+			log.Println("ok:", ok)
+		}
+	}
+}

dataclean/src/service/update.go

@@ -0,0 +1,108 @@
+// update
+package service
+
+import (
+	"log"
+	qu "qfw/util"
+	"time"
+)
+
+func init() {
+	go update()
+}
+
+func update() {
+	for {
+		if len(ResultInfos) > 0 {
+			UpdateLock.Lock()
+			UpResults = ResultInfos
+			ResultInfos = []*ResultInfo{}
+			UpdateLock.Unlock()
+			up()
+			UpResults = []*ResultInfo{}
+		}
+		time.Sleep(5 * time.Second)
+	}
+}
+
+func up() {
+	log.Println("开始更新")
+	binfos := []*BidInfo{}
+	einfos := []*ExtractInfo{}
+	pinfos := []*ProjectInfo{}
+	for _, v := range UpResults {
+		if v.Bidding != nil {
+			binfos = append(binfos, v.Bidding) //招标信息清洗结果
+		}
+		if v.Extract != nil {
+			einfos = append(einfos, v.Extract) //抽取清洗结果
+		}
+		if v.Project != nil {
+			pinfos = append(pinfos, v.Project) //项目清洗结果
+		}
+	}
+
+	updateB := [][]map[string]interface{}{}
+	updateE := [][]map[string]interface{}{}
+	updateP := [][]map[string]interface{}{}
+	if len(binfos) > 0 { //招标信息：拼装更新语句
+		for _, tmp := range binfos {
+			set := map[string]interface{}{}
+			for _, v := range tmp.UpParam {
+				set[v] = tmp.Data[v]
+			}
+			tmparr := []map[string]interface{}{
+				map[string]interface{}{
+					"_id": qu.StringTOBsonId(tmp.Id),
+				},
+				map[string]interface{}{"$set": set},
+			}
+			updateB = append(updateB, tmparr)
+		}
+	}
+	if len(einfos) > 0 { //抽取信息：拼装更新语句
+		for _, tmp := range einfos {
+			set := map[string]interface{}{}
+			for _, v := range tmp.UpParam {
+				set[v] = tmp.Data[v]
+			}
+			tmparr := []map[string]interface{}{
+				map[string]interface{}{
+					"_id": qu.StringTOBsonId(tmp.Id),
+				},
+				map[string]interface{}{"$set": set},
+			}
+			updateE = append(updateE, tmparr)
+		}
+	}
+	if len(pinfos) > 0 { //项目信息：拼装更新语句
+		for _, tmp := range pinfos {
+			set := map[string]interface{}{}
+			for _, v := range tmp.UpParam {
+				set[v] = tmp.Data[v]
+			}
+			tmparr := []map[string]interface{}{
+				map[string]interface{}{
+					"_id": qu.StringTOBsonId(tmp.Id),
+				},
+				map[string]interface{}{"$set": set},
+			}
+			updateP = append(updateP, tmparr)
+		}
+	}
+
+	//执行更新
+	if len(updateB) > 0 {
+		db := Mogdbs["qfw"]
+		db.Mongodb.UpSertBulk(db.Coll, updateB...)
+	}
+	if len(updateE) > 0 {
+		db := Mogdbs["extract"]
+		db.Mongodb.UpSertBulk(db.Coll, updateE...)
+	}
+	if len(updateP) > 0 {
+		db := Mogdbs["project"]
+		db.Mongodb.UpSertBulk(db.Coll, updateP...)
+	}
+	log.Println("更新完毕", "updateB:", len(updateE), "updateE:", len(updateB), "updateP:", len(updateP))
+}

dataclean/src/service/var.go

@@ -0,0 +1,47 @@
+// var
+package service
+
+import (
+	"qfw/util/mongodb"
+	"sync"
+)
+
+var (
+	Sysconfig   map[string]interface{} //配置文件
+	Mogdbs      map[string]MongoTool   //mongodb连接
+	ClearThread = make(chan bool, 10)  //清洗线程
+	ResultInfos []*ResultInfo          //清洗后的数据
+	UpResults   []*ResultInfo          //待更新数据
+	UpdateLock  = sync.Mutex{}         //更新锁
+	UpSize      = 100                  //批量更新阈值
+)
+
+type MongoTool struct {
+	Mongodb    mongodb.MongodbSim
+	Name, Coll string
+}
+
+type ResultInfo struct {
+	Bidding *BidInfo     //
+	Extract *ExtractInfo //
+	Project *ProjectInfo //
+	Flag    int          //更新标记
+}
+
+type BidInfo struct {
+	Id      string
+	Data    map[string]interface{} //原始信息
+	UpParam []string               //更新属性集
+}
+
+type ExtractInfo struct {
+	Id      string
+	Data    map[string]interface{} //原始信息
+	UpParam []string               //更新属性集
+}
+
+type ProjectInfo struct {
+	Id      string
+	Data    map[string]interface{} //原始信息
+	UpParam []string               //更新属性集
+}

fullproject/src2/city.json

@@ -1,3239 +0,0 @@
-{
-    "city": {
-        "金银川镇": "新疆",
-        "郑东新区": "河南",
-        "信阳客整所": "湖北",
-        "北屯": "新疆",
-        "鼓楼区": "江苏",
-        "辽河": "辽宁",
-        "南大港": "河北",
-        "北京": "北京",
-        "密云": "北京",
-        "延庆": "北京",
-        "天津": "天津",
-        "宁河": "天津",
-        "静海": "天津",
-        "蓟县": "天津",
-        "河北": "河北",
-        "石家庄": "河北",
-        "井陉": "河北",
-        "正定": "河北",
-        "栾城": "河北",
-        "行唐": "河北",
-        "灵寿": "河北",
-        "高邑": "河北",
-        "深泽": "河北",
-        "赞皇": "河北",
-        "无极县": "河北",
-        "平山县": "河北",
-        "元氏": "河北",
-        "赵县": "河北",
-        "辛集市": "河北",
-        "藁城": "河北",
-        "晋州": "河北",
-        "新乐": "河北",
-        "鹿泉": "河北",
-        "唐山": "河北",
-        "滦县": "河北",
-        "滦南": "河北",
-        "乐亭": "河北",
-        "迁西": "河北",
-        "玉田": "河北",
-        "唐海": "河北",
-        "遵化": "河北",
-        "迁安市": "河北",
-        "秦皇岛": "河北",
-        "青龙县": "河北",
-        "青龙满族自治县": "河北",
-        "昌黎": "河北",
-        "抚宁": "河北",
-        "卢龙": "河北",
-        "邯郸": "河北",
-        "临漳": "河北",
-        "成安": "河北",
-        "大名县": "河北",
-        "涉县": "河北",
-        "磁县": "河北",
-        "肥乡": "河北",
-        "永年": "河北",
-        "邱县": "河北",
-        "鸡泽": "河北",
-        "广平县": "河北",
-        "馆陶": "河北",
-        "魏县": "河北",
-        "曲周": "河北",
-        "武安": "河北",
-        "邢台": "河北",
-        "临城": "河北",
-        "内丘": "河北",
-        "柏乡县": "河北",
-        "隆尧": "河北",
-        "任县": "河北",
-        "南和": "河北",
-        "宁晋": "河北",
-        "巨鹿": "河北",
-        "新河县": "河北",
-        "广宗": "河北",
-        "平乡": "河北",
-        "威县": "河北",
-        "清河": "河北",
-        "临西": "河北",
-        "南宫": "河北",
-        "沙河市": "河北",
-        "保定": "河北",
-        "满城": "河北",
-        "清苑": "河北",
-        "涞水": "河北",
-        "阜平": "河北",
-        "徐水": "河北",
-        "定兴": "河北",
-        "唐县": "河北",
-        "高阳": "河北",
-        "容城": "河北",
-        "涞源": "河北",
-        "望都": "河北",
-        "安新": "河北",
-        "易县": "河北",
-        "曲阳": "河北",
-        "蠡县": "河北",
-        "顺平": "河北",
-        "博野": "河北",
-        "雄县": "河北",
-        "涿州": "河北",
-        "定州": "河北",
-        "安国": "河北",
-        "高碑店": "河北",
-        "张家口": "河北",
-        "宣化": "河北",
-        "张北": "河北",
-        "康保": "河北",
-        "沽源": "河北",
-        "尚义": "河北",
-        "蔚县": "河北",
-        "阳原": "河北",
-        "怀安": "河北",
-        "万全县": "河北",
-        "怀来": "河北",
-        "涿鹿": "河北",
-        "赤城": "河北",
-        "崇礼": "河北",
-        "承德": "河北",
-        "兴隆": "河北",
-        "平泉县": "河北",
-        "滦平": "河北",
-        "隆化": "河北",
-        "丰宁": "河北",
-        "宽城满族自治县": "河北",
-        "宽城县": "河北",
-        "围场": "河北",
-        "沧州": "河北",
-        "沧县": "河北",
-        "青县": "河北",
-        "东光县": "河北",
-        "海兴": "河北",
-        "盐山": "河北",
-        "肃宁": "河北",
-        "南皮": "河北",
-        "吴桥县": "河北",
-        "献县": "河北",
-        "孟村回族自治县": "河北",
-        "孟村县": "河北",
-        "泊头": "河北",
-        "任丘": "河北",
-        "黄骅": "河北",
-        "河间市": "河北",
-        "廊坊": "河北",
-        "固安": "河北",
-        "永清": "河北",
-        "香河": "河北",
-        "大城县": "河北",
-        "文安": "河北",
-        "大厂回族自治县": "河北",
-        "大厂县": "河北",
-        "霸州": "河北",
-        "三河": "河北",
-        "衡水": "河北",
-        "枣强": "河北",
-        "武邑": "河北",
-        "武强": "河北",
-        "饶阳": "河北",
-        "安平": "河北",
-        "故城县": "河北",
-        "景县": "河北",
-        "阜城县": "河北",
-        "冀州": "河北",
-        "深州": "河北",
-        "山西": "山西",
-        "太原": "山西",
-        "清徐": "山西",
-        "阳曲": "山西",
-        "娄烦": "山西",
-        "古交": "山西",
-        "大同市": "山西",
-        "阳高": "山西",
-        "天镇": "山西",
-        "广灵": "山西",
-        "灵丘": "山西",
-        "浑源": "山西",
-        "左云": "山西",
-        "阳泉": "山西",
-        "平定": "山西",
-        "盂县": "山西",
-        "长治": "山西",
-        "襄垣": "山西",
-        "屯留": "山西",
-        "平顺": "山西",
-        "黎城": "山西",
-        "壶关": "山西",
-        "长子": "山西",
-        "武乡": "山西",
-        "沁县": "山西",
-        "沁源": "山西",
-        "潞城": "山西",
-        "晋城": "山西",
-        "沁水": "山西",
-        "阳城": "山西",
-        "陵川": "山西",
-        "泽州": "山西",
-        "高平": "山西",
-        "朔州": "山西",
-        "山阴": "山西",
-        "应县": "山西",
-        "右玉": "山西",
-        "怀仁": "山西",
-        "晋中": "山西",
-        "榆社": "山西",
-        "左权县": "山西",
-        "和顺": "山西",
-        "昔阳": "山西",
-        "寿阳": "山西",
-        "太谷": "山西",
-        "祁县": "山西",
-        "平遥": "山西",
-        "灵石": "山西",
-        "介休": "山西",
-        "运城": "山西",
-        "临猗": "山西",
-        "万荣": "山西",
-        "闻喜": "山西",
-        "稷山": "山西",
-        "新绛": "山西",
-        "绛县": "山西",
-        "垣曲": "山西",
-        "夏县": "山西",
-        "平陆": "山西",
-        "芮城": "山西",
-        "永济": "山西",
-        "河津市": "山西",
-        "忻州": "山西",
-        "定襄": "山西",
-        "五台县": "山西",
-        "代县": "山西",
-        "繁峙": "山西",
-        "宁武": "山西",
-        "静乐": "山西",
-        "神池": "山西",
-        "五寨": "山西",
-        "岢岚": "山西",
-        "河曲": "山西",
-        "保德": "山西",
-        "偏关": "山西",
-        "原平": "山西",
-        "临汾": "山西",
-        "曲沃": "山西",
-        "翼城": "山西",
-        "襄汾": "山西",
-        "洪洞": "山西",
-        "古县": "山西",
-        "安泽": "山西",
-        "浮山": "山西",
-        "吉县": "山西",
-        "乡宁": "山西",
-        "大宁": "山西",
-        "隰县": "山西",
-        "永和": "山西",
-        "蒲县": "山西",
-        "汾西": "山西",
-        "侯马": "山西",
-        "霍州": "山西",
-        "吕梁": "山西",
-        "文水": "山西",
-        "交城": "山西",
-        "兴县": "山西",
-        "临县": "山西",
-        "柳林": "山西",
-        "石楼": "山西",
-        "岚县": "山西",
-        "方山县": "山西",
-        "中阳县": "山西",
-        "交口县": "山西",
-        "孝义市": "山西",
-        "汾阳": "山西",
-        "内蒙古": "内蒙古",
-        "呼和浩特": "内蒙古",
-        "土默特左旗": "内蒙古",
-        "托克托": "内蒙古",
-        "和林格尔": "内蒙古",
-        "清水河": "内蒙古",
-        "武川": "内蒙古",
-        "包头": "内蒙古",
-        "土默特右旗": "内蒙古",
-        "固阳": "内蒙古",
-        "达尔罕茂明安联合旗": "内蒙古",
-        "乌海": "内蒙古",
-        "赤峰": "内蒙古",
-        "阿鲁科尔沁旗": "内蒙古",
-        "巴林左旗": "内蒙古",
-        "巴林右旗": "内蒙古",
-        "林西": "内蒙古",
-        "克什克腾旗": "内蒙古",
-        "翁牛特旗": "内蒙古",
-        "喀喇沁旗": "内蒙古",
-        "宁城": "内蒙古",
-        "敖汉旗": "内蒙古",
-        "通辽": "内蒙古",
-        "科尔沁左翼中旗": "内蒙古",
-        "科尔沁左翼后旗": "内蒙古",
-        "开鲁": "内蒙古",
-        "库伦旗": "内蒙古",
-        "奈曼旗": "内蒙古",
-        "扎鲁特旗": "内蒙古",
-        "霍林郭勒": "内蒙古",
-        "鄂尔多斯": "内蒙古",
-        "达拉特旗": "内蒙古",
-        "准格尔旗": "内蒙古",
-        "鄂托克前旗": "内蒙古",
-        "鄂托克旗": "内蒙古",
-        "杭锦旗": "内蒙古",
-        "乌审旗": "内蒙古",
-        "伊金霍洛旗": "内蒙古",
-        "呼伦贝尔": "内蒙古",
-        "阿荣旗": "内蒙古",
-        "莫力达瓦达斡尔族自治旗": "内蒙古",
-        "鄂伦春自治旗": "内蒙古",
-        "鄂温克族自治旗": "内蒙古",
-        "陈巴尔虎旗": "内蒙古",
-        "新巴尔虎左旗": "内蒙古",
-        "新巴尔虎右旗": "内蒙古",
-        "满洲里": "内蒙古",
-        "牙克石": "内蒙古",
-        "扎兰屯": "内蒙古",
-        "额尔古纳": "内蒙古",
-        "根河": "内蒙古",
-        "巴彦淖尔": "内蒙古",
-        "五原": "内蒙古",
-        "磴口": "内蒙古",
-        "乌拉特前旗": "内蒙古",
-        "乌拉特中旗": "内蒙古",
-        "乌拉特后旗": "内蒙古",
-        "杭锦后旗": "内蒙古",
-        "乌兰察布": "内蒙古",
-        "卓资": "内蒙古",
-        "化德": "内蒙古",
-        "商都县": "内蒙古",
-        "兴和县": "内蒙古",
-        "凉城": "内蒙古",
-        "察哈尔右翼前旗": "内蒙古",
-        "察哈尔右翼中旗": "内蒙古",
-        "察哈尔右翼后旗": "内蒙古",
-        "四子王旗": "内蒙古",
-        "丰镇": "内蒙古",
-        "兴安盟": "内蒙古",
-        "乌兰浩特": "内蒙古",
-        "阿尔山": "内蒙古",
-        "科尔沁右翼前旗": "内蒙古",
-        "科尔沁右翼中旗": "内蒙古",
-        "扎赉特旗": "内蒙古",
-        "突泉": "内蒙古",
-        "锡林郭勒盟": "内蒙古",
-        "二连浩特": "内蒙古",
-        "锡林浩特": "内蒙古",
-        "阿巴嘎旗": "内蒙古",
-        "苏尼特左旗": "内蒙古",
-        "苏尼特右旗": "内蒙古",
-        "东乌珠穆沁旗": "内蒙古",
-        "西乌珠穆沁旗": "内蒙古",
-        "太仆寺旗": "内蒙古",
-        "镶黄旗": "内蒙古",
-        "正镶白旗": "内蒙古",
-        "正蓝旗": "内蒙古",
-        "多伦": "内蒙古",
-        "阿拉善盟": "内蒙古",
-        "阿拉善左旗": "内蒙古",
-        "阿拉善右旗": "内蒙古",
-        "额济纳旗": "内蒙古",
-        "辽宁": "辽宁",
-        "沈阳": "辽宁",
-        "辽中": "辽宁",
-        "康平": "辽宁",
-        "法库": "辽宁",
-        "新民市": "辽宁",
-        "大连": "辽宁",
-        "长海": "辽宁",
-        "瓦房店": "辽宁",
-        "普兰店": "辽宁",
-        "庄河": "辽宁",
-        "鞍山": "辽宁",
-        "台安县": "辽宁",
-        "岫岩": "辽宁",
-        "海城市": "辽宁",
-        "抚顺": "辽宁",
-        "新宾": "辽宁",
-        "清原": "辽宁",
-        "本溪": "辽宁",
-        "桓仁": "辽宁",
-        "丹东": "辽宁",
-        "宽甸": "辽宁",
-        "东港市": "辽宁",
-        "凤城": "辽宁",
-        "锦州": "辽宁",
-        "黑山县": "辽宁",
-        "义县": "辽宁",
-        "凌海": "辽宁",
-        "北镇": "辽宁",
-        "营口": "辽宁",
-        "盖州": "辽宁",
-        "大石桥市": "辽宁",
-        "阜新": "辽宁",
-        "彰武": "辽宁",
-        "辽阳": "辽宁",
-        "灯塔市": "辽宁",
-        "盘锦": "辽宁",
-        "大洼": "辽宁",
-        "盘山": "辽宁",
-        "铁岭": "辽宁",
-        "西丰": "辽宁",
-        "昌图": "辽宁",
-        "调兵山": "辽宁",
-        "开原": "辽宁",
-        "朝阳市": "辽宁",
-        "建平": "辽宁",
-        "喀喇沁": "辽宁",
-        "北票": "辽宁",
-        "凌源": "辽宁",
-        "葫芦岛": "辽宁",
-        "绥中": "辽宁",
-        "建昌": "辽宁",
-        "兴城": "辽宁",
-        "吉林": "吉林",
-        "长春": "吉林",
-        "农安": "吉林",
-        "九台": "吉林",
-        "榆树": "吉林",
-        "德惠": "吉林",
-        "吉林": "吉林",
-        "永吉": "吉林",
-        "蛟河": "吉林",
-        "桦甸": "吉林",
-        "舒兰": "吉林",
-        "磐石": "吉林",
-        "四平": "吉林",
-        "梨树县": "吉林",
-        "伊通": "吉林",
-        "公主岭": "吉林",
-        "双辽": "吉林",
-        "辽源": "吉林",
-        "东丰": "吉林",
-        "东辽": "吉林",
-        "通化": "吉林",
-        "辉南": "吉林",
-        "柳河": "吉林",
-        "梅河口": "吉林",
-        "集安": "吉林",
-        "白山市": "吉林",
-        "抚松": "吉林",
-        "靖宇": "吉林",
-        "长白": "吉林",
-        "临江": "吉林",
-        "松原": "吉林",
-        "前郭尔罗斯": "吉林",
-        "长岭": "吉林",
-        "乾安": "吉林",
-        "扶余": "吉林",
-        "白城": "吉林",
-        "镇赉": "吉林",
-        "通榆": "吉林",
-        "洮南": "吉林",
-        "大安市": "吉林",
-        "延边州": "吉林",
-        "延吉": "吉林",
-        "图们": "吉林",
-        "敦化": "吉林",
-        "珲春": "吉林",
-        "龙井市": "吉林",
-        "和龙": "吉林",
-        "汪清": "吉林",
-        "安图县": "吉林",
-        "黑龙江": "黑龙江",
-        "哈尔滨": "黑龙江",
-        "依兰县": "黑龙江",
-        "方正县": "黑龙江",
-        "宾县": "黑龙江",
-        "巴彦": "黑龙江",
-        "木兰县": "黑龙江",
-        "通河县": "黑龙江",
-        "延寿": "黑龙江",
-        "双城区": "黑龙江",
-        "尚志": "黑龙江",
-        "五常市": "黑龙江",
-        "齐齐哈尔": "黑龙江",
-        "龙江县": "黑龙江",
-        "依安": "黑龙江",
-        "泰来": "黑龙江",
-        "甘南": "黑龙江",
-        "富裕县": "黑龙江",
-        "克山县": "黑龙江",
-        "克东县": "黑龙江",
-        "拜泉": "黑龙江",
-        "讷河": "黑龙江",
-        "鸡西市": "黑龙江",
-        "鸡东县": "黑龙江",
-        "虎林": "黑龙江",
-        "密山": "黑龙江",
-        "鹤岗": "黑龙江",
-        "萝北": "黑龙江",
-        "绥滨": "黑龙江",
-        "双鸭山": "黑龙江",
-        "集贤": "黑龙江",
-        "友谊县": "黑龙江",
-        "宝清": "黑龙江",
-        "饶河": "黑龙江",
-        "大庆": "黑龙江",
-        "肇州": "黑龙江",
-        "肇源": "黑龙江",
-        "林甸": "黑龙江",
-        "杜尔伯特": "黑龙江",
-        "伊春市": "黑龙江",
-        "嘉荫": "黑龙江",
-        "铁力": "黑龙江",
-        "佳木斯": "黑龙江",
-        "桦南": "黑龙江",
-        "桦川": "黑龙江",
-        "汤原": "黑龙江",
-        "抚远": "黑龙江",
-        "同江市": "黑龙江",
-        "富锦": "黑龙江",
-        "七台河": "黑龙江",
-        "勃利": "黑龙江",
-        "牡丹江": "黑龙江",
-        "东宁": "黑龙江",
-        "林口": "黑龙江",
-        "绥芬河": "黑龙江",
-        "海林": "黑龙江",
-        "宁安": "黑龙江",
-        "穆棱": "黑龙江",
-        "黑河": "黑龙江",
-        "嫩江": "黑龙江",
-        "逊克": "黑龙江",
-        "孙吴": "黑龙江",
-        "北安": "黑龙江",
-        "五大连池": "黑龙江",
-        "绥化": "黑龙江",
-        "望奎": "黑龙江",
-        "兰西": "黑龙江",
-        "青冈": "黑龙江",
-        "庆安": "黑龙江",
-        "明水": "黑龙江",
-        "绥棱": "黑龙江",
-        "安达": "黑龙江",
-        "肇东": "黑龙江",
-        "海伦": "黑龙江",
-        "呼玛": "黑龙江",
-        "塔河": "黑龙江",
-        "漠河": "黑龙江",
-        "上海": "上海",
-        "崇明": "上海",
-        "江苏": "江苏",
-        "南京": "江苏",
-        "溧水": "江苏",
-        "高淳": "江苏",
-        "无锡": "江苏",
-        "江阴": "江苏",
-        "宜兴": "江苏",
-        "徐州": "江苏",
-        "丰县": "江苏",
-        "沛县": "江苏",
-        "铜山": "江苏",
-        "睢宁": "江苏",
-        "新沂": "江苏",
-        "邳州": "江苏",
-        "常州": "江苏",
-        "溧阳": "江苏",
-        "金坛": "江苏",
-        "苏州": "江苏",
-        "常熟": "江苏",
-        "张家港": "江苏",
-        "昆山": "江苏",
-        "吴江": "江苏",
-        "太仓": "江苏",
-        "南通": "江苏",
-        "海安": "江苏",
-        "如东": "江苏",
-        "启东": "江苏",
-        "如皋": "江苏",
-        "通州": "江苏",
-        "海门": "江苏",
-        "连云港": "江苏",
-        "赣榆": "江苏",
-        "东海": "江苏",
-        "灌云": "江苏",
-        "灌南": "江苏",
-        "淮安": "江苏",
-        "涟水": "江苏",
-        "洪泽区": "江苏",
-        "洪泽县": "江苏",
-        "盱眙": "江苏",
-        "金湖": "江苏",
-        "盐城": "江苏",
-        "响水": "江苏",
-        "滨海县": "江苏",
-        "阜宁": "江苏",
-        "射阳": "江苏",
-        "建湖": "江苏",
-        "东台": "江苏",
-        "大丰": "江苏",
-        "扬州": "江苏",
-        "宝应": "江苏",
-        "仪征": "江苏",
-        "高邮": "江苏",
-        "江都": "江苏",
-        "镇江": "江苏",
-        "丹阳": "江苏",
-        "扬中": "江苏",
-        "句容": "江苏",
-        "泰州": "江苏",
-        "兴化": "江苏",
-        "靖江": "江苏",
-        "泰兴市": "江苏",
-        "泰兴镇": "四川",
-        "姜堰": "江苏",
-        "宿迁": "江苏",
-        "沭阳": "江苏",
-        "泗阳": "江苏",
-        "泗洪": "江苏",
-        "浙江": "浙江",
-        "杭州": "浙江",
-        "桐庐": "浙江",
-        "淳安": "浙江",
-        "建德": "浙江",
-        "富阳区": "浙江",
-        "临安": "浙江",
-        "宁波": "浙江",
-        "象山": "浙江",
-        "宁海": "浙江",
-        "余姚": "浙江",
-        "慈溪": "浙江",
-        "奉化": "浙江",
-        "温州": "浙江",
-        "洞头": "浙江",
-        "永嘉": "浙江",
-        "平阳": "浙江",
-        "苍南": "浙江",
-        "文成县": "浙江",
-        "泰顺": "浙江",
-        "瑞安": "浙江",
-        "乐清": "浙江",
-        "嘉兴": "浙江",
-        "嘉善": "浙江",
-        "海盐": "浙江",
-        "海宁": "浙江",
-        "平湖": "浙江",
-        "桐乡": "浙江",
-        "湖州": "浙江",
-        "德清": "浙江",
-        "长兴": "浙江",
-        "安吉": "浙江",
-        "绍兴": "浙江",
-        "新昌": "浙江",
-        "诸暨": "浙江",
-        "上虞": "浙江",
-        "嵊州": "浙江",
-        "金华": "浙江",
-        "武义": "浙江",
-        "浦江": "浙江",
-        "磐安": "浙江",
-        "兰溪": "浙江",
-        "义乌": "浙江",
-        "东阳": "浙江",
-        "永康": "浙江",
-        "衢州": "浙江",
-        "常山": "浙江",
-        "开化": "浙江",
-        "龙游": "浙江",
-        "江山市": "浙江",
-        "舟山": "浙江",
-        "岱山": "浙江",
-        "嵊泗": "浙江",
-        "台州": "浙江",
-        "玉环": "浙江",
-        "三门县": "浙江",
-        "天台县": "浙江",
-        "仙居": "浙江",
-        "温岭": "浙江",
-        "临海": "浙江",
-        "丽水": "浙江",
-        "青田": "浙江",
-        "缙云": "浙江",
-        "遂昌": "浙江",
-        "松阳": "浙江",
-        "云和": "浙江",
-        "庆元": "浙江",
-        "景宁": "浙江",
-        "龙泉市": "浙江",
-        "安徽": "安徽",
-        "合肥": "安徽",
-        "长丰": "安徽",
-        "肥东": "安徽",
-        "肥西": "安徽",
-        "芜湖": "安徽",
-        "繁昌": "安徽",
-        "南陵": "安徽",
-        "蚌埠": "安徽",
-        "怀远": "安徽",
-        "五河": "安徽",
-        "固镇": "安徽",
-        "淮南": "安徽",
-        "凤台": "安徽",
-        "马鞍山": "安徽",
-        "当涂": "安徽",
-        "含山": "安徽",
-        "和县": "安徽",
-        "淮北": "安徽",
-        "濉溪": "安徽",
-        "铜陵": "安徽",
-        "安庆": "安徽",
-        "怀宁": "安徽",
-        "枞阳": "安徽",
-        "潜山": "安徽",
-        "太湖县": "安徽",
-        "宿松": "安徽",
-        "望江": "安徽",
-        "岳西": "安徽",
-        "桐城": "安徽",
-        "黄山": "安徽",
-        "歙县": "安徽",
-        "休宁": "安徽",
-        "黟县": "安徽",
-        "祁门": "安徽",
-        "滁州": "安徽",
-        "来安": "安徽",
-        "全椒": "安徽",
-        "定远": "安徽",
-        "凤阳": "安徽",
-        "天长市": "安徽",
-        "明光市": "安徽",
-        "阜阳": "安徽",
-        "临泉": "安徽",
-        "太和县": "安徽",
-        "阜南": "安徽",
-        "颍上": "安徽",
-        "界首": "安徽",
-        "宿州": "安徽",
-        "砀山": "安徽",
-        "萧县": "安徽",
-        "灵璧": "安徽",
-        "泗县": "安徽",
-        "巢湖": "安徽",
-        "庐江": "安徽",
-        "无为县": "安徽",
-        "六安": "安徽",
-        "寿县": "安徽",
-        "霍邱": "安徽",
-        "舒城": "安徽",
-        "金寨": "安徽",
-        "霍山": "安徽",
-        "亳州": "安徽",
-        "涡阳": "安徽",
-        "蒙城": "安徽",
-        "利辛": "安徽",
-        "池州": "安徽",
-        "东至县": "安徽",
-        "石台": "安徽",
-        "青阳": "安徽",
-        "宣城": "安徽",
-        "郎溪": "安徽",
-        "广德": "安徽",
-        "泾县": "安徽",
-        "绩溪": "安徽",
-        "旌德": "安徽",
-        "宁国市": "安徽",
-        "福建": "福建",
-        "福州": "福建",
-        "闽侯": "福建",
-        "连江": "福建",
-        "罗源": "福建",
-        "闽清": "福建",
-        "永泰县": "福建",
-        "平潭": "福建",
-        "福清": "福建",
-        "长乐": "福建",
-        "厦门": "福建",
-        "莆田": "福建",
-        "仙游": "福建",
-        "三明": "福建",
-        "明溪": "福建",
-        "清流": "福建",
-        "宁化": "福建",
-        "大田": "福建",
-        "尤溪": "福建",
-        "沙县": "福建",
-        "将乐": "福建",
-        "泰宁": "福建",
-        "建宁": "福建",
-        "永安": "福建",
-        "泉州": "福建",
-        "惠安县": "福建",
-        "安溪": "福建",
-        "永春": "福建",
-        "德化": "福建",
-        "金门": "福建",
-        "石狮": "福建",
-        "晋江": "福建",
-        "南安": "福建",
-        "漳州": "福建",
-        "云霄": "福建",
-        "漳浦": "福建",
-        "诏安": "福建",
-        "长泰": "福建",
-        "东山县": "福建",
-        "南靖": "福建",
-        "平和县": "福建",
-        "华安": "福建",
-        "龙海": "福建",
-        "南平": "福建",
-        "顺昌": "福建",
-        "浦城": "福建",
-        "光泽": "福建",
-        "松溪": "福建",
-        "政和县": "福建",
-        "邵武": "福建",
-        "武夷山": "福建",
-        "建瓯": "福建",
-        "建阳": "福建",
-        "龙岩": "福建",
-        "长汀": "福建",
-        "永定": "福建",
-        "上杭": "福建",
-        "武平": "福建",
-        "连城": "福建",
-        "漳平": "福建",
-        "宁德": "福建",
-        "霞浦": "福建",
-        "古田": "福建",
-        "屏南": "福建",
-        "寿宁": "福建",
-        "周宁": "福建",
-        "柘荣": "福建",
-        "福安": "福建",
-        "福鼎": "福建",
-        "江西": "江西",
-        "南昌": "江西",
-        "新建区": "江西",
-        "安义": "江西",
-        "进贤": "江西",
-        "景德镇": "江西",
-        "浮梁": "江西",
-        "乐平市": "江西",
-        "萍乡": "江西",
-        "莲花": "江西",
-        "上栗": "江西",
-        "芦溪": "江西",
-        "九江市": "江西",
-        "武宁": "江西",
-        "修水县": "江西",
-        "永修": "江西",
-        "德安": "江西",
-        "星子县": "江西",
-        "都昌": "江西",
-        "湖口县": "江西",
-        "彭泽": "江西",
-        "瑞昌": "江西",
-        "新余": "江西",
-        "分宜": "江西",
-        "鹰潭": "江西",
-        "余江": "江西",
-        "贵溪": "江西",
-        "赣州": "江西",
-        "赣县": "江西",
-        "信丰": "江西",
-        "大余": "江西",
-        "上犹": "江西",
-        "崇义": "江西",
-        "安远": "江西",
-        "龙南县": "江西",
-        "定南": "江西",
-        "全南": "江西",
-        "宁都": "江西",
-        "于都": "江西",
-        "兴国县": "江西",
-        "会昌": "江西",
-        "寻乌": "江西",
-        "石城": "江西",
-        "瑞金": "江西",
-        "南康": "江西",
-        "吉安": "江西",
-        "吉水": "江西",
-        "峡江": "江西",
-        "新干": "江西",
-        "永丰": "江西",
-        "泰和县": "江西",
-        "遂川": "江西",
-        "万安县": "江西",
-        "安福县": "江西",
-        "永新县": "江西",
-        "井冈山": "江西",
-        "宜春": "江西",
-        "奉新": "江西",
-        "万载县": "江西",
-        "上高": "江西",
-        "宜丰": "江西",
-        "靖安": "江西",
-        "铜鼓": "江西",
-        "丰城": "江西",
-        "樟树": "江西",
-        "高安": "江西",
-        "抚州": "江西",
-        "南城": "江西",
-        "黎川": "江西",
-        "南丰": "江西",
-        "崇仁": "江西",
-        "乐安": "江西",
-        "宜黄": "江西",
-        "金溪": "江西",
-        "资溪": "江西",
-        "东乡县": "江西",
-        "广昌": "江西",
-        "上饶": "江西",
-        "广丰": "江西",
-        "玉山": "江西",
-        "铅山": "江西",
-        "横峰": "江西",
-        "弋阳": "江西",
-        "余干": "江西",
-        "鄱阳": "江西",
-        "万年县": "江西",
-        "婺源": "江西",
-        "德兴": "江西",
-        "山东": "山东",
-        "济南": "山东",
-        "平阴": "山东",
-        "济阳": "山东",
-        "商河县": "山东",
-        "章丘": "山东",
-        "青岛": "山东",
-        "胶州": "山东",
-        "即墨": "山东",
-        "平度市": "山东",
-        "胶南": "山东",
-        "莱西": "山东",
-        "淄博": "山东",
-        "桓台": "山东",
-        "高青": "山东",
-        "沂源": "山东",
-        "枣庄": "山东",
-        "滕州": "山东",
-        "东营市": "山东",
-        "东营区": "山东",
-        "垦利": "山东",
-        "利津": "山东",
-        "广饶": "山东",
-        "烟台": "山东",
-        "长岛县": "山东",
-        "龙口市": "山东",
-        "莱阳": "山东",
-        "莱州": "山东",
-        "蓬莱": "山东",
-        "招远": "山东",
-        "栖霞市": "山东",
-        "海阳": "山东",
-        "潍坊": "山东",
-        "临朐": "山东",
-        "昌乐": "山东",
-        "青州": "山东",
-        "诸城": "山东",
-        "寿光": "山东",
-        "安丘": "山东",
-        "高密": "山东",
-        "昌邑市": "山东",
-        "济宁": "山东",
-        "微山": "山东",
-        "鱼台": "山东",
-        "金乡": "山东",
-        "嘉祥": "山东",
-        "汶上": "山东",
-        "泗水": "山东",
-        "梁山": "山东",
-        "曲阜": "山东",
-        "兖州": "山东",
-        "邹城": "山东",
-        "泰安": "山东",
-        "宁阳": "山东",
-        "东平": "山东",
-        "新泰": "山东",
-        "肥城": "山东",
-        "威海": "山东",
-        "文登": "山东",
-        "荣成": "山东",
-        "乳山": "山东",
-        "日照": "山东",
-        "五莲": "山东",
-        "莒县": "山东",
-        "莱芜": "山东",
-        "临沂": "山东",
-        "沂南": "山东",
-        "郯城": "山东",
-        "沂水": "山东",
-        "苍山": "山东",
-        "费县": "山东",
-        "平邑": "山东",
-        "莒南": "山东",
-        "蒙阴": "山东",
-        "临沭": "山东",
-        "德州": "山东",
-        "陵县": "山东",
-        "宁津": "山东",
-        "庆云": "山东",
-        "临邑": "山东",
-        "齐河": "山东",
-        "平原": "山东",
-        "夏津": "山东",
-        "武城": "山东",
-        "乐陵": "山东",
-        "禹城": "山东",
-        "聊城": "山东",
-        "阳谷": "山东",
-        "莘县": "山东",
-        "茌平": "山东",
-        "东阿": "山东",
-        "冠县": "山东",
-        "高唐": "山东",
-        "临清": "山东",
-        "滨州": "山东",
-        "惠民": "山东",
-        "阳信": "山东",
-        "无棣": "山东",
-        "沾化": "山东",
-        "博兴": "山东",
-        "邹平": "山东",
-        "菏泽": "山东",
-        "曹县": "山东",
-        "单县": "山东",
-        "成武": "山东",
-        "巨野": "山东",
-        "郓城": "山东",
-        "鄄城": "山东",
-        "定陶": "山东",
-        "东明县": "山东",
-        "河南": "河南",
-        "郑州": "河南",
-        "管城": "河南",
-        "中牟": "河南",
-        "巩义": "河南",
-        "荥阳": "河南",
-        "新密": "河南",
-        "新郑": "河南",
-        "登封": "河南",
-        "开封": "河南",
-        "杞县": "河南",
-        "通许": "河南",
-        "尉氏": "河南",
-        "兰考": "河南",
-        "洛阳": "河南",
-        "孟津": "河南",
-        "新安": "河南",
-        "栾川": "河南",
-        "嵩县": "河南",
-        "汝阳": "河南",
-        "宜阳": "河南",
-        "洛宁": "河南",
-        "伊川": "河南",
-        "偃师": "河南",
-        "平顶山": "河南",
-        "宝丰": "河南",
-        "叶县": "河南",
-        "鲁山": "河南",
-        "郏县": "河南",
-        "舞钢": "河南",
-        "汝州": "河南",
-        "安阳": "河南",
-        "汤阴": "河南",
-        "滑县": "河南",
-        "内黄": "河南",
-        "林州": "河南",
-        "鹤壁": "河南",
-        "浚县": "河南",
-        "淇县": "河南",
-        "新乡": "河南",
-        "获嘉": "河南",
-        "原阳": "河南",
-        "延津": "河南",
-        "封丘": "河南",
-        "长垣": "河南",
-        "卫辉": "河南",
-        "辉市": "河南",
-        "焦作": "河南",
-        "修武": "河南",
-        "博爱": "河南",
-        "武陟": "河南",
-        "温县": "河南",
-        "沁阳": "河南",
-        "孟州": "河南",
-        "濮阳": "河南",
-        "清丰": "河南",
-        "南乐": "河南",
-        "范县": "河南",
-        "台前": "河南",
-        "许昌": "河南",
-        "鄢陵": "河南",
-        "襄城县": "河南",
-        "禹州": "河南",
-        "长葛": "河南",
-        "漯河": "河南",
-        "舞阳": "河南",
-        "临颍": "河南",
-        "三门峡": "河南",
-        "渑池": "河南",
-        "陕县": "河南",
-        "卢氏": "河南",
-        "义马": "河南",
-        "灵宝": "河南",
-        "南阳": "河南",
-        "南召": "河南",
-        "方城": "河南",
-        "西峡": "河南",
-        "镇平县": "河南",
-        "内乡": "河南",
-        "淅川": "河南",
-        "社旗": "河南",
-        "唐河": "河南",
-        "新野": "河南",
-        "桐柏": "河南",
-        "邓州": "河南",
-        "商丘": "河南",
-        "民权": "河南",
-        "睢县": "河南",
-        "宁陵": "河南",
-        "柘城": "河南",
-        "虞城": "河南",
-        "夏邑": "河南",
-        "永城": "河南",
-        "信阳": "河南",
-        "罗山": "河南",
-        "光山": "河南",
-        "新县": "河南",
-        "商城": "河南",
-        "固始": "河南",
-        "潢川": "河南",
-        "淮滨": "河南",
-        "息县": "河南",
-        "周口": "河南",
-        "扶沟": "河南",
-        "西华": "河南",
-        "商水": "河南",
-        "沈丘": "河南",
-        "郸城": "河南",
-        "淮阳": "河南",
-        "太康": "河南",
-        "鹿邑": "河南",
-        "项城": "河南",
-        "驻马店": "河南",
-        "西平": "河南",
-        "上蔡": "河南",
-        "平舆": "河南",
-        "正阳": "河南",
-        "确山": "河南",
-        "泌阳": "河南",
-        "汝南": "河南",
-        "遂平": "河南",
-        "新蔡": "河南",
-        "济源": "河南",
-        "湖北": "湖北",
-        "武汉": "湖北",
-        "黄石市": "湖北",
-        "阳新县": "湖北",
-        "大冶": "湖北",
-        "十堰": "湖北",
-        "郧县": "湖北",
-        "郧西": "湖北",
-        "竹山": "湖北",
-        "竹溪": "湖北",
-        "房县": "湖北",
-        "丹江口": "湖北",
-        "宜昌": "湖北",
-        "远安": "湖北",
-        "兴山": "湖北",
-        "秭归": "湖北",
-        "长阳": "湖北",
-        "五峰": "湖北",
-        "宜都": "湖北",
-        "当阳": "湖北",
-        "枝江": "湖北",
-        "襄阳": "湖北",
-        "南漳": "湖北",
-        "谷城": "湖北",
-        "保康": "湖北",
-        "老河口": "湖北",
-        "枣阳": "湖北",
-        "宜城": "湖北",
-        "鄂州": "湖北",
-        "荆门": "湖北",
-        "京山县": "湖北",
-        "沙洋": "湖北",
-        "钟祥": "湖北",
-        "屈家岭": "湖北",
-        "孝感": "湖北",
-        "孝昌": "湖北",
-        "大悟": "湖北",
-        "云梦": "湖北",
-        "应城": "湖北",
-        "安陆": "湖北",
-        "汉川": "湖北",
-        "荆州": "湖北",
-        "公安县": "湖北",
-        "监利": "湖北",
-        "江陵": "湖北",
-        "石首": "湖北",
-        "洪湖": "湖北",
-        "松滋": "湖北",
-        "黄冈市": "湖北",
-        "团风县": "湖北",
-        "红安": "湖北",
-        "罗田": "湖北",
-        "英山": "湖北",
-        "浠水": "湖北",
-        "蕲春": "湖北",
-        "黄梅": "湖北",
-        "麻城": "湖北",
-        "武穴": "湖北",
-        "咸宁": "湖北",
-        "嘉鱼": "湖北",
-        "通城": "湖北",
-        "崇阳": "湖北",
-        "通山": "湖北",
-        "赤壁": "湖北",
-        "随州": "湖北",
-        "随县": "湖北",
-        "广水": "湖北",
-        "恩施州": "湖北",
-        "恩施": "湖北",
-        "利川": "湖北",
-        "建始县": "湖北",
-        "巴东": "湖北",
-        "宣恩": "湖北",
-        "咸丰": "湖北",
-        "来凤": "湖北",
-        "鹤峰": "湖北",
-        "仙桃": "湖北",
-        "潜江": "湖北",
-        "天门市": "湖北",
-        "神农架": "湖北",
-        "江汉油田": "湖北",
-        "湖南": "湖南",
-        "长沙": "湖南",
-        "望城": "湖南",
-        "宁乡": "湖南",
-        "浏阳": "湖南",
-        "株洲": "湖南",
-        "攸县": "湖南",
-        "茶陵": "湖南",
-        "炎陵": "湖南",
-        "醴陵": "湖南",
-        "湘潭": "湖南",
-        "湘乡": "湖南",
-        "韶山": "湖南",
-        "衡阳": "湖南",
-        "衡南": "湖南",
-        "衡山": "湖南",
-        "衡东": "湖南",
-        "祁东": "湖南",
-        "耒阳": "湖南",
-        "常宁": "湖南",
-        "邵阳": "湖南",
-        "邵东": "湖南",
-        "新邵": "湖南",
-        "隆回": "湖南",
-        "洞口县": "湖南",
-        "绥宁": "湖南",
-        "新宁县": "湖南",
-        "城步": "湖南",
-        "武冈": "湖南",
-        "岳阳": "湖南",
-        "华容县": "湖南",
-        "湘阴": "湖南",
-        "平江": "湖南",
-        "汨罗": "湖南",
-        "临湘": "湖南",
-        "常德": "湖南",
-        "安乡县": "湖南",
-        "汉寿": "湖南",
-        "澧县": "湖南",
-        "临澧": "湖南",
-        "桃源县": "湖南",
-        "石门": "湖南",
-        "津市市": "湖南",
-        "张家界": "湖南",
-        "慈利": "湖南",
-        "桑植": "湖南",
-        "益阳": "湖南",
-        "南县": "湖南",
-        "桃江": "湖南",
-        "安化": "湖南",
-        "沅江": "湖南",
-        "郴州": "湖南",
-        "桂阳": "湖南",
-        "宜章": "湖南",
-        "永兴": "湖南",
-        "嘉禾": "湖南",
-        "临武": "湖南",
-        "汝城": "湖南",
-        "桂东": "湖南",
-        "安仁": "湖南",
-        "资兴": "湖南",
-        "永州": "湖南",
-        "祁阳": "湖南",
-        "东安县": "湖南",
-        "双牌": "湖南",
-        "道县": "湖南",
-        "江永": "湖南",
-        "宁远": "湖南",
-        "蓝山": "湖南",
-        "新田": "湖南",
-        "江华": "湖南",
-        "怀化": "湖南",
-        "中方县": "湖南",
-        "沅陵": "湖南",
-        "辰溪": "湖南",
-        "溆浦": "湖南",
-        "会同": "湖南",
-        "麻阳": "湖南",
-        "新晃": "湖南",
-        "芷江": "湖南",
-        "靖州": "湖南",
-        "通道县": "湖南",
-        "通道侗族自治县": "湖南",
-        "洪江": "湖南",
-        "娄底": "湖南",
-        "双峰": "湖南",
-        "新化": "湖南",
-        "冷水江": "湖南",
-        "涟源": "湖南",
-        "湘西": "湖南",
-        "吉首": "湖南",
-        "泸溪": "湖南",
-        "凤凰县": "湖南",
-        "花垣": "湖南",
-        "保靖": "湖南",
-        "古丈": "湖南",
-        "永顺": "湖南",
-        "龙山县": "湖南",
-        "广东": "广东",
-        "广州": "广东",
-        "增城": "广东",
-        "从化": "广东",
-        "韶关": "广东",
-        "始兴": "广东",
-        "仁化": "广东",
-        "翁源": "广东",
-        "乳源": "广东",
-        "新丰": "广东",
-        "乐昌": "广东",
-        "南雄": "广东",
-        "深圳": "广东",
-        "珠海": "广东",
-        "汕头": "广东",
-        "南澳": "广东",
-        "佛山": "广东",
-        "江门市": "广东",
-        "台山市": "广东",
-        "开平市": "广东",
-        "鹤山市": "广东",
-        "恩平": "广东",
-        "湛江": "广东",
-        "遂溪": "广东",
-        "徐闻": "广东",
-        "廉江": "广东",
-        "雷州": "广东",
-        "吴川": "广东",
-        "茂名": "广东",
-        "电白": "广东",
-        "高州": "广东",
-        "化州": "广东",
-        "信宜": "广东",
-        "肇庆": "广东",
-        "广宁": "广东",
-        "怀集": "广东",
-        "封开": "广东",
-        "德庆": "广东",
-        "高要区": "广东",
-        "四会市": "广东",
-        "惠州": "广东",
-        "博罗": "广东",
-        "惠东": "广东",
-        "龙门县": "广东",
-        "梅州": "广东",
-        "梅县": "广东",
-        "大埔": "广东",
-        "丰顺": "广东",
-        "五华县": "广东",
-        "平远": "广东",
-        "蕉岭": "广东",
-        "兴宁市": "广东",
-        "汕尾": "广东",
-        "海丰": "广东",
-        "陆河": "广东",
-        "陆丰": "广东",
-        "河源": "广东",
-        "紫金县": "广东",
-        "龙川": "广东",
-        "连平": "广东",
-        "和平县": "广东",
-        "东源": "广东",
-        "阳江市": "广东",
-        "阳西": "广东",
-        "阳东": "广东",
-        "阳春": "广东",
-        "清远": "广东",
-        "佛冈": "广东",
-        "阳山": "广东",
-        "连山县": "广东",
-        "连山壮族瑶族自治县": "广东",
-        "连南": "广东",
-        "清新区": "广东",
-        "英德": "广东",
-        "连州": "广东",
-        "东莞": "广东",
-        "中山市": "广东",
-        "潮州": "广东",
-        "潮安": "广东",
-        "饶平": "广东",
-        "揭阳": "广东",
-        "揭东": "广东",
-        "揭西": "广东",
-        "惠来": "广东",
-        "普宁": "广东",
-        "云浮": "广东",
-        "新兴": "广东",
-        "郁南": "广东",
-        "云安": "广东",
-        "罗定": "广东",
-        "广西": "广西",
-        "南宁": "广西",
-        "武鸣": "广西",
-        "隆安": "广西",
-        "马山": "广西",
-        "上林": "广西",
-        "宾阳": "广西",
-        "横县": "广西",
-        "柳州": "广西",
-        "柳江": "广西",
-        "柳城": "广西",
-        "鹿寨": "广西",
-        "融安": "广西",
-        "融水": "广西",
-        "三江侗族自治县": "广西",
-        "三江县": "广西",
-        "桂林": "广西",
-        "阳朔": "广西",
-        "临桂": "广西",
-        "灵川": "广西",
-        "全州": "广西",
-        "兴安": "广西",
-        "永福": "广西",
-        "灌阳": "广西",
-        "龙胜": "广西",
-        "资源县": "广西",
-        "平乐": "广西",
-        "荔蒲": "广西",
-        "恭城": "广西",
-        "梧州": "广西",
-        "苍梧": "广西",
-        "藤县": "广西",
-        "蒙山": "广西",
-        "岑溪": "广西",
-        "北海市": "广西",
-        "北海经济开发区": "山东",
-        "合浦": "广西",
-        "防城港": "广西",
-        "上思": "广西",
-        "东兴市": "广西",
-        "钦州": "广西",
-        "灵山县": "广西",
-        "浦北": "广西",
-        "贵港": "广西",
-        "平南": "广西",
-        "桂平": "广西",
-        "玉林": "广西",
-        "容县": "广西",
-        "陆川": "广西",
-        "博白": "广西",
-        "兴业县": "广西",
-        "北流": "广西",
-        "百色": "广西",
-        "田阳": "广西",
-        "田东": "广西",
-        "平果县": "广西",
-        "德保": "广西",
-        "靖西": "广西",
-        "那坡": "广西",
-        "凌云县": "广西",
-        "乐业县": "广西",
-        "田林县": "广西",
-        "西林": "广西",
-        "隆林": "广西",
-        "贺州": "广西",
-        "昭平": "广西",
-        "钟山县": "广西",
-        "富川": "广西",
-        "河池": "广西",
-        "南丹": "广西",
-        "天峨": "广西",
-        "凤山": "广西",
-        "东兰": "广西",
-        "罗城": "广西",
-        "环江": "广西",
-        "巴马": "广西",
-        "都安": "广西",
-        "大化": "广西",
-        "宜州": "广西",
-        "来宾": "广西",
-        "忻城": "广西",
-        "象州": "广西",
-        "武宣": "广西",
-        "金秀": "广西",
-        "合山": "广西",
-        "崇左": "广西",
-        "扶绥": "广西",
-        "宁明": "广西",
-        "龙州": "广西",
-        "大新县": "广西",
-        "天等": "广西",
-        "凭祥": "广西",
-        "海南": "海南",
-        "三沙": "海南",
-        "海口": "海南",
-        "三亚": "海南",
-        "五指山": "海南",
-        "琼海": "海南",
-        "儋州": "海南",
-        "文昌": "海南",
-        "万宁": "海南",
-        "东方市": "海南",
-        "定安": "海南",
-        "屯昌": "海南",
-        "澄迈": "海南",
-        "临高": "海南",
-        "白沙县": "海南",
-        "白沙黎族自治县": "海南",
-        "昌江自治县": "海南",
-        "乐东": "海南",
-        "陵水": "海南",
-        "保亭": "海南",
-        "琼中": "海南",
-        "重庆": "重庆",
-        "綦江": "重庆",
-        "潼南": "重庆",
-        "铜梁": "重庆",
-        "大足": "重庆",
-        "荣昌": "重庆",
-        "璧山": "重庆",
-        "梁平": "重庆",
-        "城口县": "重庆",
-        "丰都": "重庆",
-        "垫江": "重庆",
-        "武隆": "重庆",
-        "忠县": "重庆",
-        "开县": "重庆",
-        "云阳": "重庆",
-        "奉节": "重庆",
-        "巫山": "重庆",
-        "巫溪": "重庆",
-        "石柱土家族自治": "重庆",
-        "秀山土家族苗族自治": "重庆",
-        "酉阳土家族苗族自治": "重庆",
-        "彭水苗族土家族自治": "重庆",
-        "四川": "四川",
-        "成都": "四川",
-        "金堂": "四川",
-        "双流": "四川",
-        "郫县": "四川",
-        "大邑": "四川",
-        "蒲江": "四川",
-        "新津": "四川",
-        "都江堰": "四川",
-        "彭州": "四川",
-        "邛崃": "四川",
-        "崇州": "四川",
-        "自贡": "四川",
-        "荣县": "四川",
-        "富顺": "四川",
-        "攀枝花": "四川",
-        "米易": "四川",
-        "盐边": "四川",
-        "泸州": "四川",
-        "泸县": "四川",
-        "合江": "四川",
-        "叙永": "四川",
-        "古蔺": "四川",
-        "南部县": "四川",
-        "德阳": "四川",
-        "中江": "四川",
-        "罗江": "四川",
-        "广汉": "四川",
-        "什邡": "四川",
-        "绵竹": "四川",
-        "绵阳": "四川",
-        "三台县": "四川",
-        "盐亭": "四川",
-        "安县": "四川",
-        "梓潼": "四川",
-        "北川羌族自治": "四川",
-        "平武": "四川",
-        "江油": "四川",
-        "广元": "四川",
-        "旺苍": "四川",
-        "青川": "四川",
-        "剑阁": "四川",
-        "苍溪": "四川",
-        "遂宁": "四川",
-        "蓬溪": "四川",
-        "射洪": "四川",
-        "大英": "四川",
-        "内江": "四川",
-        "威远": "四川",
-        "资中": "四川",
-        "隆昌": "四川",
-        "乐山": "四川",
-        "犍为": "四川",
-        "井研": "四川",
-        "夹江": "四川",
-        "沐川": "四川",
-        "峨边彝族自治": "四川",
-        "马边彝族自治": "四川",
-        "峨眉山": "四川",
-        "南充": "四川",
-        "营山": "四川",
-        "蓬安": "四川",
-        "仪陇": "四川",
-        "西充": "四川",
-        "阆中": "四川",
-        "眉山": "四川",
-        "仁寿": "四川",
-        "彭山": "四川",
-        "洪雅": "四川",
-        "丹棱": "四川",
-        "青神": "四川",
-        "宜宾": "四川",
-        "南溪": "四川",
-        "江安": "四川",
-        "长宁县": "四川",
-        "高县": "四川",
-        "珙县": "四川",
-        "筠连": "四川",
-        "兴文": "四川",
-        "屏山": "四川",
-        "广安": "四川",
-        "岳池": "四川",
-        "武胜": "四川",
-        "邻水": "四川",
-        "华蓥": "四川",
-        "达州": "四川",
-        "达县": "四川",
-        "宣汉": "四川",
-        "开江县": "四川",
-        "大竹县": "四川",
-        "渠县": "四川",
-        "万源市": "四川",
-        "雅安市": "四川",
-        "名山县": "四川",
-        "荥经": "四川",
-        "汉源": "四川",
-        "石棉县": "四川",
-        "天全": "四川",
-        "芦山": "四川",
-        "宝兴": "四川",
-        "巴中": "四川",
-        "通江": "四川",
-        "南江县": "四川",
-        "平昌": "四川",
-        "资阳": "四川",
-        "安岳": "四川",
-        "乐至县": "四川",
-        "简阳": "四川",
-        "阿坝州": "四川",
-        "汶川": "四川",
-        "理县": "四川",
-        "茂县": "四川",
-        "松潘": "四川",
-        "九寨沟": "四川",
-        "金川县": "四川",
-        "小金县": "四川",
-        "黑水县": "四川",
-        "马尔康": "四川",
-        "壤塘": "四川",
-        "阿坝": "四川",
-        "若尔盖": "四川",
-        "红原": "四川",
-        "甘孜": "四川",
-        "康定": "四川",
-        "泸定": "四川",
-        "丹巴": "四川",
-        "九龙县": "四川",
-        "雅江": "四川",
-        "道孚": "四川",
-        "炉霍": "四川",
-        "新龙": "四川",
-        "德格": "四川",
-        "白玉县": "四川",
-        "石渠": "四川",
-        "色达": "四川",
-        "理塘": "四川",
-        "巴塘": "四川",
-        "乡城县": "四川",
-        "稻城": "四川",
-        "得荣": "四川",
-        "凉山": "四川",
-        "西昌": "四川",
-        "木里": "四川",
-        "盐源": "四川",
-        "德昌": "四川",
-        "会理": "四川",
-        "会东": "四川",
-        "宁南": "四川",
-        "普格": "四川",
-        "布拖": "四川",
-        "金阳": "四川",
-        "昭觉": "四川",
-        "喜德": "四川",
-        "冕宁": "四川",
-        "越西": "四川",
-        "甘洛": "四川",
-        "美姑": "四川",
-        "雷波": "四川",
-        "贵州": "贵州",
-        "贵阳": "贵州",
-        "开阳": "贵州",
-        "息烽": "贵州",
-        "修文": "贵州",
-        "清镇": "贵州",
-        "六盘水": "贵州",
-        "水城": "贵州",
-        "盘县": "贵州",
-        "遵义": "贵州",
-        "桐梓": "贵州",
-        "绥阳": "贵州",
-        "正安": "贵州",
-        "道真": "贵州",
-        "务川": "贵州",
-        "凤冈": "贵州",
-        "湄潭": "贵州",
-        "余庆": "贵州",
-        "习水": "贵州",
-        "赤水": "贵州",
-        "仁怀": "贵州",
-        "安顺": "贵州",
-        "平坝": "贵州",
-        "普定": "贵州",
-        "镇宁": "贵州",
-        "关岭": "贵州",
-        "紫云县": "贵州",
-        "紫云苗族布依族自治县": "贵州",
-        "毕节": "贵州",
-        "大方县": "贵州",
-        "黔西": "贵州",
-        "金沙县": "贵州",
-        "织金": "贵州",
-        "纳雍": "贵州",
-        "威宁": "贵州",
-        "赫章": "贵州",
-        "铜仁": "贵州",
-        "江口县": "贵州",
-        "玉屏": "贵州",
-        "石阡": "贵州",
-        "思南": "贵州",
-        "印江": "贵州",
-        "德江": "贵州",
-        "沿河": "贵州",
-        "松桃": "贵州",
-        "黔西南": "贵州",
-        "兴义": "贵州",
-        "兴仁": "贵州",
-        "普安": "贵州",
-        "晴隆": "贵州",
-        "贞丰": "贵州",
-        "望谟": "贵州",
-        "册亨": "贵州",
-        "安龙县": "贵州",
-        "黔东南": "贵州",
-        "凯里市": "贵州",
-        "黄平县": "贵州",
-        "施秉": "贵州",
-        "三穗": "贵州",
-        "镇远县": "贵州",
-        "岑巩": "贵州",
-        "天柱": "贵州",
-        "锦屏": "贵州",
-        "剑河": "贵州",
-        "台江县": "贵州",
-        "台江区": "福建",
-        "黎平": "贵州",
-        "榕江": "贵州",
-        "从江": "贵州",
-        "雷山": "贵州",
-        "麻江": "贵州",
-        "丹寨": "贵州",
-        "黔南布": "贵州",
-        "都匀": "贵州",
-        "福泉": "贵州",
-        "荔波": "贵州",
-        "贵定": "贵州",
-        "瓮安": "贵州",
-        "独山县": "贵州",
-        "平塘": "贵州",
-        "罗甸": "贵州",
-        "长顺": "贵州",
-        "龙里": "贵州",
-        "惠水": "贵州",
-        "三都": "贵州",
-        "云南": "云南",
-        "昆明": "云南",
-        "呈贡": "云南",
-        "晋宁": "云南",
-        "富民县": "云南",
-        "宜良": "云南",
-        "石林": "云南",
-        "嵩明": "云南",
-        "禄劝": "云南",
-        "寻甸": "云南",
-        "安宁": "云南",
-        "曲靖": "云南",
-        "马龙": "云南",
-        "陆良": "云南",
-        "师宗": "云南",
-        "罗平县": "云南",
-        "富源": "云南",
-        "会泽": "云南",
-        "沾益": "云南",
-        "宣威": "云南",
-        "玉溪": "云南",
-        "江川": "云南",
-        "澄江": "云南",
-        "通海": "云南",
-        "华宁": "云南",
-        "易门": "云南",
-        "峨山": "云南",
-        "新平": "云南",
-        "元江": "云南",
-        "保山": "云南",
-        "施甸": "云南",
-        "腾冲": "云南",
-        "龙陵": "云南",
-        "昌宁": "云南",
-        "昭通": "云南",
-        "鲁甸": "云南",
-        "巧家": "云南",
-        "盐津": "云南",
-        "大关县": "云南",
-        "永善": "云南",
-        "绥江": "云南",
-        "镇雄": "云南",
-        "彝良": "云南",
-        "威信": "云南",
-        "水富": "云南",
-        "丽江": "云南",
-        "玉龙县": "云南",
-        "玉龙纳西族自治县": "云南",
-        "永胜": "云南",
-        "华坪": "云南",
-        "宁蒗": "云南",
-        "普洱": "云南",
-        "宁洱": "云南",
-        "墨江": "云南",
-        "景东县": "云南",
-        "景东彝族自治县": "云南",
-        "景谷": "云南",
-        "镇沅": "云南",
-        "江城自治县": "云南",
-        "江城县": "云南",
-        "孟连": "云南",
-        "澜沧": "云南",
-        "西盟": "云南",
-        "临沧": "云南",
-        "凤庆": "云南",
-        "云县": "云南",
-        "永德": "云南",
-        "镇康": "云南",
-        "双江": "云南",
-        "耿马": "云南",
-        "沧源": "云南",
-        "楚雄": "云南",
-        "双柏": "云南",
-        "牟定": "云南",
-        "南华": "云南",
-        "姚安": "云南",
-        "大姚": "云南",
-        "永仁": "云南",
-        "元谋": "云南",
-        "武定": "云南",
-        "禄丰": "云南",
-        "红河": "云南",
-        "个旧": "云南",
-        "开远": "云南",
-        "蒙自": "云南",
-        "屏边": "云南",
-        "建水": "云南",
-        "石屏": "云南",
-        "弥勒": "云南",
-        "泸西": "云南",
-        "元阳": "云南",
-        "金平自治县": "云南",
-        "金平县": "云南",
-        "绿春": "云南",
-        "河口自治县": "云南",
-        "文山": "云南",
-        "砚山": "云南",
-        "西畴": "云南",
-        "麻栗坡": "云南",
-        "马关县": "云南",
-        "丘北县": "云南",
-        "广南": "云南",
-        "富宁": "云南",
-        "西双版纳": "云南",
-        "景洪": "云南",
-        "勐海": "云南",
-        "勐腊": "云南",
-        "大理白族自治州": "云南",
-        "大理州": "云南",
-        "大理市": "云南",
-        "漾濞": "云南",
-        "祥云": "云南",
-        "宾川": "云南",
-        "弥渡": "云南",
-        "南涧": "云南",
-        "巍山": "云南",
-        "永平": "云南",
-        "云龙": "云南",
-        "洱源": "云南",
-        "剑川": "云南",
-        "鹤庆": "云南",
-        "德宏": "云南",
-        "瑞丽": "云南",
-        "潞西": "云南",
-        "梁河": "云南",
-        "盈江": "云南",
-        "陇川": "云南",
-        "怒江": "云南",
-        "泸水": "云南",
-        "福贡": "云南",
-        "贡山": "云南",
-        "兰坪": "云南",
-        "迪庆": "云南",
-        "香格里拉": "云南",
-        "德钦": "云南",
-        "维西": "云南",
-        "西藏": "西藏",
-        "拉萨": "西藏",
-        "林周": "西藏",
-        "当雄": "西藏",
-        "尼木": "西藏",
-        "曲水": "西藏",
-        "堆龙德庆": "西藏",
-        "达孜": "西藏",
-        "墨竹工卡": "西藏",
-        "昌都": "西藏",
-        "江达": "西藏",
-        "贡觉": "西藏",
-        "类乌齐": "西藏",
-        "丁青": "西藏",
-        "察雅": "西藏",
-        "八宿": "西藏",
-        "左贡": "西藏",
-        "芒康": "西藏",
-        "洛隆": "西藏",
-        "边坝": "西藏",
-        "乃东": "西藏",
-        "扎囊": "西藏",
-        "贡嘎": "西藏",
-        "桑日": "西藏",
-        "琼结": "西藏",
-        "曲松": "西藏",
-        "措美": "西藏",
-        "洛扎": "西藏",
-        "加查": "西藏",
-        "隆子": "西藏",
-        "错那": "西藏",
-        "浪卡子": "西藏",
-        "日喀则": "西藏",
-        "南木林": "西藏",
-        "江孜": "西藏",
-        "定日": "西藏",
-        "萨迦": "西藏",
-        "拉孜": "西藏",
-        "昂仁": "西藏",
-        "谢通门": "西藏",
-        "白朗": "西藏",
-        "仁布": "西藏",
-        "康马": "西藏",
-        "定结": "西藏",
-        "仲巴": "西藏",
-        "亚东": "西藏",
-        "吉隆": "西藏",
-        "聂拉木": "西藏",
-        "萨嘎": "西藏",
-        "岗巴": "西藏",
-        "那曲": "西藏",
-        "嘉黎": "西藏",
-        "比如县": "西藏",
-        "聂荣县": "西藏",
-        "安多县": "西藏",
-        "申扎": "西藏",
-        "索县": "西藏",
-        "班戈": "西藏",
-        "巴青": "西藏",
-        "尼玛": "西藏",
-        "普兰": "西藏",
-        "札达": "西藏",
-        "噶尔": "西藏",
-        "日土": "西藏",
-        "革吉": "西藏",
-        "改则": "西藏",
-        "措勤": "西藏",
-        "林芝市": "西藏",
-        "林芝县": "西藏",
-        "工布江达": "西藏",
-        "米林": "西藏",
-        "墨脱": "西藏",
-        "波密": "西藏",
-        "察隅": "西藏",
-        "朗县": "西藏",
-        "陕西": "陕西",
-        "西安": "陕西",
-        "蓝田": "陕西",
-        "周至": "陕西",
-        "户县": "陕西",
-        "高陵": "陕西",
-        "铜川": "陕西",
-        "宜君": "陕西",
-        "宝鸡": "陕西",
-        "凤翔县": "陕西",
-        "岐山": "陕西",
-        "扶风": "陕西",
-        "眉县": "陕西",
-        "陇县": "陕西",
-        "千阳": "陕西",
-        "麟游": "陕西",
-        "凤县": "陕西",
-        "太白县": "陕西",
-        "咸阳": "陕西",
-        "三原县": "陕西",
-        "泾阳": "陕西",
-        "乾县": "陕西",
-        "礼泉": "陕西",
-        "永寿": "陕西",
-        "彬县": "陕西",
-        "长武": "陕西",
-        "旬邑": "陕西",
-        "淳化": "陕西",
-        "武功县": "陕西",
-        "兴平": "陕西",
-        "渭南": "陕西",
-        "潼关": "陕西",
-        "大荔": "陕西",
-        "合阳": "陕西",
-        "澄城": "陕西",
-        "蒲城": "陕西",
-        "白水": "陕西",
-        "富平": "陕西",
-        "韩城": "陕西",
-        "华阴": "陕西",
-        "延安": "陕西",
-        "延长县": "陕西",
-        "延川": "陕西",
-        "子长": "陕西",
-        "安塞": "陕西",
-        "志丹": "陕西",
-        "吴起": "陕西",
-        "甘泉": "陕西",
-        "富县": "陕西",
-        "洛川": "陕西",
-        "宜川": "陕西",
-        "黄龙": "陕西",
-        "黄陵": "陕西",
-        "汉中": "陕西",
-        "南郑县": "陕西",
-        "城固": "陕西",
-        "洋县": "陕西",
-        "西乡": "陕西",
-        "勉县": "陕西",
-        "宁强": "陕西",
-        "略阳": "陕西",
-        "镇巴": "陕西",
-        "留坝": "陕西",
-        "佛坪": "陕西",
-        "榆林": "陕西",
-        "神木": "陕西",
-        "府谷": "陕西",
-        "横山": "陕西",
-        "靖边": "陕西",
-        "定边": "陕西",
-        "绥德": "陕西",
-        "米脂": "陕西",
-        "佳县": "陕西",
-        "吴堡": "陕西",
-        "清涧": "陕西",
-        "子洲": "陕西",
-        "安康": "陕西",
-        "汉阴": "陕西",
-        "石泉": "陕西",
-        "宁陕": "陕西",
-        "紫阳": "陕西",
-        "岚皋": "陕西",
-        "平利": "陕西",
-        "镇坪": "陕西",
-        "旬阳": "陕西",
-        "白河县": "陕西",
-        "商洛": "陕西",
-        "洛南": "陕西",
-        "丹凤": "陕西",
-        "商南": "陕西",
-        "山阳": "陕西",
-        "镇安县": "陕西",
-        "柞水": "陕西",
-        "甘肃": "甘肃",
-        "兰州": "甘肃",
-        "东乡自治县": "甘肃",
-        "永登": "甘肃",
-        "皋兰": "甘肃",
-        "榆中": "甘肃",
-        "嘉峪关": "甘肃",
-        "金昌": "甘肃",
-        "永昌": "甘肃",
-        "白银市": "甘肃",
-        "靖远": "甘肃",
-        "会宁": "甘肃",
-        "景泰县": "甘肃",
-        "天水": "甘肃",
-        "清水县": "甘肃",
-        "秦安": "甘肃",
-        "甘谷": "甘肃",
-        "武山": "甘肃",
-        "张家川": "甘肃",
-        "武威": "甘肃",
-        "民勤": "甘肃",
-        "古浪": "甘肃",
-        "天祝": "甘肃",
-        "张掖": "甘肃",
-        "肃南": "甘肃",
-        "民乐县": "甘肃",
-        "临泽": "甘肃",
-        "高台": "甘肃",
-        "山丹": "甘肃",
-        "平凉": "甘肃",
-        "泾川": "甘肃",
-        "灵台": "甘肃",
-        "崇信": "甘肃",
-        "华亭": "甘肃",
-        "庄浪": "甘肃",
-        "静宁": "甘肃",
-        "酒泉": "甘肃",
-        "金塔": "甘肃",
-        "瓜州": "甘肃",
-        "肃北": "甘肃",
-        "阿克塞": "甘肃",
-        "玉门": "甘肃",
-        "敦煌": "甘肃",
-        "庆阳": "甘肃",
-        "庆城": "甘肃",
-        "环县": "甘肃",
-        "华池": "甘肃",
-        "合水县": "甘肃",
-        "正宁": "甘肃",
-        "宁县": "甘肃",
-        "镇原县": "甘肃",
-        "定西": "甘肃",
-        "通渭": "甘肃",
-        "陇西": "甘肃",
-        "渭源": "甘肃",
-        "临洮": "甘肃",
-        "漳县": "甘肃",
-        "岷县": "甘肃",
-        "陇南": "甘肃",
-        "成县": "甘肃",
-        "文县": "甘肃",
-        "宕昌": "甘肃",
-        "康县": "甘肃",
-        "西和": "甘肃",
-        "礼县": "甘肃",
-        "徽县": "甘肃",
-        "两当县": "甘肃",
-        "临夏": "甘肃",
-        "康乐县": "甘肃",
-        "永靖": "甘肃",
-        "广河": "甘肃",
-        "和政": "甘肃",
-        "积石山": "甘肃",
-        "合作市": "甘肃",
-        "临潭": "甘肃",
-        "卓尼": "甘肃",
-        "舟曲": "甘肃",
-        "迭部": "甘肃",
-        "玛曲": "甘肃",
-        "碌曲": "甘肃",
-        "夏河": "甘肃",
-        "青海": "青海",
-        "共和县": "青海",
-        "西宁": "青海",
-        "大通自治县": "青海",
-        "大通县": "青海",
-        "湟中": "青海",
-        "湟源": "青海",
-        "平安县": "青海",
-        "民和": "青海",
-        "乐都": "青海",
-        "河南蒙古族自治县": "青海",
-        "互助土族自治县": "青海",
-        "互助县": "青海",
-        "化隆": "青海",
-        "循化": "青海",
-        "海北州": "青海",
-        "海北藏族自治州": "青海",
-        "门源": "青海",
-        "祁连": "青海",
-        "海晏": "青海",
-        "刚察": "青海",
-        "黄南": "青海",
-        "同仁": "青海",
-        "尖扎": "青海",
-        "泽库": "青海",
-        "同德": "青海",
-        "贵德": "青海",
-        "兴海": "青海",
-        "贵南": "青海",
-        "果洛": "青海",
-        "玛沁": "青海",
-        "班玛": "青海",
-        "甘德": "青海",
-        "达日": "青海",
-        "久治": "青海",
-        "玛多": "青海",
-        "玉树": "青海",
-        "杂多": "青海",
-        "称多": "青海",
-        "治多": "青海",
-        "囊谦": "青海",
-        "曲麻莱": "青海",
-        "海西州": "青海",
-        "海西蒙古族藏族自治州": "青海",
-        "格尔木": "青海",
-        "德令哈": "青海",
-        "乌兰": "青海",
-        "都兰": "青海",
-        "天峻": "青海",
-        "宁夏": "宁夏",
-        "银川": "宁夏",
-        "永宁": "宁夏",
-        "贺兰": "宁夏",
-        "灵武": "宁夏",
-        "石嘴山": "宁夏",
-        "平罗": "宁夏",
-        "吴忠": "宁夏",
-        "盐池": "宁夏",
-        "同心县": "宁夏",
-        "青铜峡": "宁夏",
-        "固原": "宁夏",
-        "西吉": "宁夏",
-        "隆德": "宁夏",
-        "泾源": "宁夏",
-        "彭阳": "宁夏",
-        "中卫": "宁夏",
-        "中宁": "宁夏",
-        "海原": "宁夏",
-        "新疆": "新疆",
-        "乌鲁木齐": "新疆",
-        "克拉玛依": "新疆",
-        "吐鲁番": "新疆",
-        "鄯善": "新疆",
-        "托克逊": "新疆",
-        "哈密": "新疆",
-        "巴里坤": "新疆",
-        "伊吾": "新疆",
-        "昌吉": "新疆",
-        "阜康": "新疆",
-        "呼图壁": "新疆",
-        "玛纳斯": "新疆",
-        "奇台": "新疆",
-        "吉木萨尔": "新疆",
-        "木垒哈萨克": "新疆",
-        "博尔塔拉": "新疆",
-        "博乐": "新疆",
-        "精河": "新疆",
-        "温泉县": "新疆",
-        "巴音郭楞": "新疆",
-        "库尔勒": "新疆",
-        "轮台": "新疆",
-        "尉犁": "新疆",
-        "若羌": "新疆",
-        "且末": "新疆",
-        "焉耆": "新疆",
-        "和静": "新疆",
-        "和硕": "新疆",
-        "博湖": "新疆",
-        "阿克苏": "新疆",
-        "温宿": "新疆",
-        "库车": "新疆",
-        "沙雅": "新疆",
-        "新和县": "新疆",
-        "拜城": "新疆",
-        "乌什": "新疆",
-        "阿瓦提": "新疆",
-        "柯坪": "新疆",
-        "克孜勒": "新疆",
-        "阿图什": "新疆",
-        "阿克陶": "新疆",
-        "阿合奇": "新疆",
-        "乌恰": "新疆",
-        "喀什": "新疆",
-        "疏附": "新疆",
-        "疏勒": "新疆",
-        "英吉沙": "新疆",
-        "泽普": "新疆",
-        "莎车": "新疆",
-        "叶城": "新疆",
-        "麦盖提": "新疆",
-        "岳普湖": "新疆",
-        "伽师": "新疆",
-        "巴楚": "新疆",
-        "塔什": "新疆",
-        "和田": "新疆",
-        "墨玉": "新疆",
-        "皮山": "新疆",
-        "洛浦": "新疆",
-        "策勒": "新疆",
-        "于田县": "新疆",
-        "民丰": "新疆",
-        "伊犁": "新疆",
-        "伊宁": "新疆",
-        "奎屯": "新疆",
-        "察布查尔锡伯": "新疆",
-        "霍城": "新疆",
-        "巩留": "新疆",
-        "新源": "新疆",
-        "昭苏": "新疆",
-        "特克斯": "新疆",
-        "尼勒克": "新疆",
-        "塔城": "新疆",
-        "乌苏": "新疆",
-        "额敏": "新疆",
-        "沙湾": "新疆",
-        "托里": "新疆",
-        "裕民": "新疆",
-        "和布克赛尔": "新疆",
-        "阿勒泰": "新疆",
-        "布尔津": "新疆",
-        "富蕴": "新疆",
-        "福海": "新疆",
-        "哈巴河": "新疆",
-        "青河": "新疆",
-        "吉木乃": "新疆",
-        "石河子": "新疆",
-        "阿拉尔": "新疆",
-        "图木舒克": "新疆",
-        "五家渠": "新疆",
-        "台湾": "台湾",
-        "台北": "台湾",
-        "高雄": "台湾",
-        "基隆": "台湾",
-        "台中市": "台湾",
-        "台南": "台湾",
-        "新竹": "台湾",
-        "嘉义": "台湾",
-        "宜兰": "台湾",
-        "桃园市": "台湾",
-        "苗栗": "台湾",
-        "彰化": "台湾",
-        "南投": "台湾",
-        "云林": "台湾",
-        "屏东": "台湾",
-        "台东": "台湾",
-        "花莲": "台湾",
-        "澎湖": "台湾",
-        "香港": "香港",
-        "澳门": "澳门",
-        "东城区": "北京",
-        "西城区": "北京",
-        "崇文": "北京",
-        "宣武": "北京",
-        "丰台": "北京",
-        "石景山": "北京",
-        "海淀": "北京",
-        "门头沟": "北京",
-        "房山": "北京",
-        "顺义": "北京",
-        "昌平": "北京",
-        "大兴区": "北京",
-        "怀柔": "北京",
-        "平谷": "北京",
-        "河西区": "天津",
-        "滨海新区": "天津",
-        "南开区": "天津",
-        "红桥区": "天津",
-        "塘沽": "天津",
-        "汉沽": "天津",
-        "大港": "天津",
-        "东丽": "天津",
-        "西青": "天津",
-        "津南": "天津",
-        "北辰": "天津",
-        "武清": "天津",
-        "宝坻": "天津",
-        "井陉矿": "河北",
-        "裕华": "河北",
-        "路南区": "河北",
-        "路北区": "河北",
-        "古冶": "河北",
-        "丰南": "河北",
-        "丰润": "河北",
-        "海港区": "河北",
-        "山海关": "河北",
-        "北戴河": "河北",
-        "邯山": "河北",
-        "丛台": "河北",
-        "复兴区": "河北",
-        "峰峰矿": "河北",
-        "北市区": "河北",
-        "南市区": "河北",
-        "下花园": "河北",
-        "双滦": "河北",
-        "鹰手营子矿": "河北",
-        "运河区": "河北",
-        "安次": "河北",
-        "广阳": "河北",
-        "桃城": "河北",
-        "小店区": "山西",
-        "迎泽": "山西",
-        "杏花岭": "山西",
-        "尖草坪": "山西",
-        "万柏林": "山西",
-        "晋源": "山西",
-        "南郊区": "山西",
-        "新荣": "山西",
-        "朔城": "山西",
-        "平鲁": "山西",
-        "榆次": "山西",
-        "盐湖": "山西",
-        "忻府": "山西",
-        "尧都": "山西",
-        "离石": "山西",
-        "玉泉": "内蒙古",
-        "赛罕": "内蒙古",
-        "东河区": "内蒙古",
-        "昆都仑": "内蒙古",
-        "石拐": "内蒙古",
-        "白云矿": "内蒙古",
-        "九原": "内蒙古",
-        "海勃湾": "内蒙古",
-        "乌达": "内蒙古",
-        "红山区": "内蒙古",
-        "元宝山": "内蒙古",
-        "松山": "内蒙古",
-        "科尔沁": "内蒙古",
-        "东胜": "内蒙古",
-        "海拉尔": "内蒙古",
-        "临河": "内蒙古",
-        "集宁": "内蒙古",
-        "沈河": "辽宁",
-        "大东区": "辽宁",
-        "皇姑区": "辽宁",
-        "苏家屯": "辽宁",
-        "东陵区": "辽宁",
-        "沈北新": "辽宁",
-        "于洪": "辽宁",
-        "西岗区": "辽宁",
-        "沙河口": "辽宁",
-        "甘井子": "辽宁",
-        "旅顺口": "辽宁",
-        "金州": "辽宁",
-        "立山": "辽宁",
-        "千山": "辽宁",
-        "新抚": "辽宁",
-        "东洲": "辽宁",
-        "望花": "辽宁",
-        "顺城": "辽宁",
-        "溪湖": "辽宁",
-        "明山区": "辽宁",
-        "南芬": "辽宁",
-        "元宝区": "辽宁",
-        "振兴区": "辽宁",
-        "振安": "辽宁",
-        "古塔区": "辽宁",
-        "凌河": "辽宁",
-        "站前区": "辽宁",
-        "西市区": "辽宁",
-        "鲅鱼圈": "辽宁",
-        "老边": "辽宁",
-        "新邱": "辽宁",
-        "太平区": "辽宁",
-        "清河门": "辽宁",
-        "细河": "辽宁",
-        "白塔": "辽宁",
-        "文圣区": "辽宁",
-        "宏伟区": "辽宁",
-        "弓长岭": "辽宁",
-        "太子河": "辽宁",
-        "双台子": "辽宁",
-        "兴隆台": "辽宁",
-        "银州": "辽宁",
-        "双塔区": "辽宁",
-        "龙城区": "辽宁",
-        "龙港区": "辽宁",
-        "南票区": "辽宁",
-        "杨家杖子开发": "辽宁",
-        "南关区": "吉林",
-        "二道区": "吉林",
-        "绿园区": "吉林",
-        "双阳区": "吉林",
-        "龙潭区": "吉林",
-        "船营": "吉林",
-        "丰满区": "吉林",
-        "东昌区": "吉林",
-        "东昌府区": "山东",
-        "二道江": "吉林",
-        "八道江": "吉林",
-        "江源区": "吉林",
-        "宁江": "吉林",
-        "洮北": "吉林",
-        "道里": "黑龙江",
-        "南岗区": "黑龙江",
-        "道外区": "黑龙江",
-        "平房区": "黑龙江",
-        "松北": "黑龙江",
-        "香坊": "黑龙江",
-        "呼兰": "黑龙江",
-        "阿城": "黑龙江",
-        "龙沙": "黑龙江",
-        "建华区": "黑龙江",
-        "铁锋": "黑龙江",
-        "昂昂溪": "黑龙江",
-        "富拉尔基": "黑龙江",
-        "碾子山": "黑龙江",
-        "梅里斯达斡尔族": "黑龙江",
-        "鸡冠区": "黑龙江",
-        "恒山": "黑龙江",
-        "滴道": "黑龙江",
-        "城子河": "黑龙江",
-        "麻山": "黑龙江",
-        "工农区": "黑龙江",
-        "尖山": "黑龙江",
-        "岭东": "黑龙江",
-        "四方台区": "黑龙江",
-        "萨尔图": "黑龙江",
-        "龙凤区": "黑龙江",
-        "让胡路": "黑龙江",
-        "红岗区": "黑龙江",
-        "南岔": "黑龙江",
-        "友好区": "黑龙江",
-        "翠峦": "黑龙江",
-        "新青区": "黑龙江",
-        "美溪": "黑龙江",
-        "金山屯": "黑龙江",
-        "五营区": "黑龙江",
-        "乌马河": "黑龙江",
-        "汤旺河": "黑龙江",
-        "带岭": "黑龙江",
-        "乌伊岭": "黑龙江",
-        "上甘岭": "黑龙江",
-        "东风区": "黑龙江",
-        "桃山": "黑龙江",
-        "茄子河": "黑龙江",
-        "阳明区": "黑龙江",
-        "爱民区": "黑龙江",
-        "爱辉区": "黑龙江",
-        "北林区": "黑龙江",
-        "大兴安岭": "黑龙江",
-        "加格达奇": "黑龙江",
-        "松岭区": "黑龙江",
-        "新林区": "黑龙江",
-        "呼中": "黑龙江",
-        "黄浦": "上海",
-        "徐汇": "上海",
-        "静安": "上海",
-        "闸北": "上海",
-        "虹口": "上海",
-        "杨浦": "上海",
-        "闵行": "上海",
-        "嘉定": "上海",
-        "浦东新": "上海",
-        "金山区 ": "上海",
-        "松江": "上海",
-        "青浦": "上海",
-        "奉贤": "上海",
-        "玄武": "江苏",
-        "白下": "江苏",
-        "秦淮": "江苏",
-        "建邺": "江苏",
-        "下关区": "江苏",
-        "浦口区": "江苏",
-        "雨花台": "江苏",
-        "江宁": "江苏",
-        "六合": "江苏",
-        "崇安": "江苏",
-        "南长": "江苏",
-        "北塘": "江苏",
-        "锡山": "江苏",
-        "惠山": "江苏",
-        "滨湖": "江苏",
-        "九里": "江苏",
-        "贾汪": "江苏",
-        "泉山": "江苏",
-        "天宁": "江苏",
-        "钟楼": "江苏",
-        "戚墅堰": "江苏",
-        "新北": "江苏",
-        "武进": "江苏",
-        "沧浪": "江苏",
-        "金阊": "江苏",
-        "虎丘": "江苏",
-        "吴中": "江苏",
-        "相城": "江苏",
-        "崇川": "江苏",
-        "港闸": "江苏",
-        "新浦": "江苏",
-        "楚州": "江苏",
-        "淮阴区": "江苏",
-        "清浦": "江苏",
-        "亭湖": "江苏",
-        "盐都": "江苏",
-        "广陵": "江苏",
-        "邗江": "江苏",
-        "维扬": "江苏",
-        "京口": "江苏",
-        "润州": "江苏",
-        "丹徒": "江苏",
-        "海陵": "江苏",
-        "高港": "江苏",
-        "宿城": "江苏",
-        "宿豫": "江苏",
-        "上城区": "浙江",
-        "下城区": "浙江",
-        "江干区": "浙江",
-        "拱墅": "浙江",
-        "滨江区": "浙江",
-        "萧山": "浙江",
-        "余杭": "浙江",
-        "海曙": "浙江",
-        "江东区": "浙江",
-        "北仑": "浙江",
-        "镇海": "浙江",
-        "鄞州": "浙江",
-        "鹿城": "浙江",
-        "龙湾区": "浙江",
-        "瓯海": "浙江",
-        "南湖区": "浙江",
-        "秀洲": "浙江",
-        "吴兴": "浙江",
-        "南浔": "浙江",
-        "越城": "浙江",
-        "婺城": "浙江",
-        "金东": "浙江",
-        "柯城": "浙江",
-        "衢江": "浙江",
-        "定海": "浙江",
-        "椒江": "浙江",
-        "黄岩": "浙江",
-        "路桥区": "浙江",
-        "莲都": "浙江",
-        "瑶海": "安徽",
-        "庐阳": "安徽",
-        "蜀山": "安徽",
-        "新站": "安徽",
-        "包河": "安徽",
-        "镜湖区": "安徽",
-        "镜湖新区": "浙江",
-        "弋江": "安徽",
-        "鸠江": "安徽",
-        "三山区": "安徽",
-        "蚌山": "安徽",
-        "禹会": "安徽",
-        "淮上": "安徽",
-        "田家庵": "安徽",
-        "谢家集": "安徽",
-        "八公山": "安徽",
-        "潘集": "安徽",
-        "金家庄": "安徽",
-        "花山区": "安徽",
-        "雨山": "安徽",
-        "博望新": "安徽",
-        "杜集区": "安徽",
-        "相山": "安徽",
-        "烈山": "安徽",
-        "铜官山": "安徽",
-        "狮子山": "安徽",
-        "迎江": "安徽",
-        "大观区": "安徽",
-        "宜秀": "安徽",
-        "屯溪": "安徽",
-        "徽州": "安徽",
-        "琅琊区": "安徽",
-        "南谯": "安徽",
-        "颍州": "安徽",
-        "颍东": "安徽",
-        "颍泉": "安徽",
-        "埇桥": "安徽",
-        "居巢区": "安徽",
-        "金安": "安徽",
-        "裕安": "安徽",
-        "谯城": "安徽",
-        "贵池": "安徽",
-        "宣州": "安徽",
-        "仓山": "福建",
-        "马尾": "福建",
-        "晋安": "福建",
-        "思明区": "福建",
-        "海沧": "福建",
-        "湖里": "福建",
-        "集美区": "福建",
-        "同安区": "福建",
-        "同安县": "福建",
-        "翔安区": "福建",
-        "城厢": "福建",
-        "涵江": "福建",
-        "荔城": "福建",
-        "秀屿区": "福建",
-        "梅列": "福建",
-        "三元区": "福建",
-        "鲤城": "福建",
-        "丰泽区": "福建",
-        "洛江": "福建",
-        "泉港": "福建",
-        "芗城": "福建",
-        "龙文": "福建",
-        "延平": "福建",
-        "新罗": "福建",
-        "蕉城": "福建",
-        "东湖区": "江西",
-        "青云谱": "江西",
-        "湾里区": "江西",
-        "青山湖": "江西",
-        "珠山": "江西",
-        "安源": "江西",
-        "湘东": "江西",
-        "庐山": "江西",
-        "浔阳": "江西",
-        "渝水": "江西",
-        "月湖区": "江西",
-        "章贡": "江西",
-        "吉州": "江西",
-        "青原区": "江西",
-        "袁州": "江西",
-        "临川": "江西",
-        "信州": "江西",
-        "历下": "山东",
-        "槐荫": "山东",
-        "天桥区": "山东",
-        "历城": "山东",
-        "长清": "山东",
-        "市南区": "山东",
-        "市北区": "山东",
-        "四方区": "山东",
-        "黄岛": "山东",
-        "崂山": "山东",
-        "李沧": "山东",
-        "城阳": "山东",
-        "淄川": "山东",
-        "张店": "山东",
-        "博山": "山东",
-        "临淄": "山东",
-        "周村": "山东",
-        "薛城": "山东",
-        "峄城": "山东",
-        "台儿庄": "山东",
-        "山亭": "山东",
-        "芝罘": "山东",
-        "福山": "山东",
-        "牟平": "山东",
-        "莱山": "山东",
-        "潍城": "山东",
-        "寒亭": "山东",
-        "坊子": "山东",
-        "奎文": "山东",
-        "任城": "山东",
-        "泰山区": "山东",
-        "岱岳": "山东",
-        "环翠": "山东",
-        "岚山": "山东",
-        "莱城": "山东",
-        "钢城区": "山东",
-        "兰山": "山东",
-        "罗庄区": "山东",
-        "德城区": "山东",
-        "东昌府": "山东",
-        "滨城区": "山东",
-        "牡丹区": "山东",
-        "中原区": "河南",
-        "二七区": "河南",
-        "金水区": "河南",
-        "上街": "河南",
-        "惠济区": "河南",
-        "龙亭区": "河南",
-        "顺河回族": "河南",
-        "禹王台": "河南",
-        "金明": "河南",
-        "西工区": "河南",
-        "瀍河回族": "河南",
-        "涧西": "河南",
-        "吉利区": "河南",
-        "洛龙": "河南",
-        "卫东区": "河南",
-        "石龙": "河南",
-        "湛河": "河南",
-        "文峰": "河南",
-        "北关区": "河南",
-        "殷都": "河南",
-        "龙安": "河南",
-        "山城区": "河南",
-        "淇滨": "河南",
-        "红旗区": "河南",
-        "卫滨": "河南",
-        "凤泉": "河南",
-        "牧野区": "河南",
-        "中站区": "河南",
-        "马村区": "河南",
-        "华龙区": "河南",
-        "魏都": "河南",
-        "源汇": "河南",
-        "郾城": "河南",
-        "召陵": "河南",
-        "湖滨": "河南",
-        "宛城": "河南",
-        "卧龙区": "河南",
-        "梁园": "河南",
-        "睢阳": "河南",
-        "浉河": "河南",
-        "平桥区": "河南",
-        "川汇": "河南",
-        "驿城": "河南",
-        "江岸区": "湖北",
-        "江汉区": "湖北",
-        "硚口": "湖北",
-        "汉阳": "湖北",
-        "武昌": "湖北",
-        "洪山": "湖北",
-        "东西湖": "湖北",
-        "汉南": "湖北",
-        "蔡甸": "湖北",
-        "江夏": "湖北",
-        "黄陂": "湖北",
-        "新洲区": "湖北",
-        "东湖开发": "湖北",
-        "武汉经济开发": "湖北",
-        "黄石港": "湖北",
-        "西塞山": "湖北",
-        "下陆": "湖北",
-        "铁山区": "湖北",
-        "黄石经济开发": "湖北",
-        "茅箭": "湖北",
-        "张湾": "湖北",
-        "十堰经济开发": "湖北",
-        "武当山旅游经济特": "湖北",
-        "西陵": "湖北",
-        "伍家岗": "湖北",
-        "点军": "湖北",
-        "猇亭": "湖北",
-        "夷陵": "湖北",
-        "樊城": "湖北",
-        "隆中风景": "湖北",
-        "高新开发": "湖北",
-        "鱼梁洲开发": "湖北",
-        "梁子湖": "湖北",
-        "鄂城": "湖北",
-        "葛店开发": "湖北",
-        "鄂州经济开发": "湖北",
-        "东宝": "湖北",
-        "掇刀": "湖北",
-        "荆门经济开发": "湖北",
-        "孝南": "湖北",
-        "沙市": "湖北",
-        "荆州经济开发": "湖北",
-        "黄州": "湖北",
-        "龙感湖管理": "湖北",
-        "咸安": "湖北",
-        "曾都": "湖北",
-        "芙蓉区": "湖南",
-        "天心区": "湖南",
-        "岳麓": "湖南",
-        "开福": "湖南",
-        "雨花区": "湖南",
-        "荷塘区": "湖南",
-        "芦淞": "湖南",
-        "石峰": "湖南",
-        "天元区": "湖南",
-        "雨湖": "湖南",
-        "岳塘": "湖南",
-        "珠晖": "湖南",
-        "雁峰": "湖南",
-        "石鼓": "湖南",
-        "蒸湘": "湖南",
-        "南岳": "湖南",
-        "双清": "湖南",
-        "大祥": "湖南",
-        "北塔区": "湖南",
-        "岳阳楼": "湖南",
-        "云溪": "湖南",
-        "君山": "湖南",
-        "武陵": "湖南",
-        "鼎城": "湖南",
-        "武陵源": "湖南",
-        "赫山": "湖南",
-        "北湖": "湖南",
-        "苏仙": "湖南",
-        "零陵": "湖南",
-        "冷水滩": "湖南",
-        "鹤城": "湖南",
-        "娄星": "湖南",
-        "荔湾": "广东",
-        "越秀": "广东",
-        "海珠": "广东",
-        "天河区": "广东",
-        "黄埔": "广东",
-        "番禺": "广东",
-        "花都区": "广东",
-        "南沙区": "广东",
-        "萝岗": "广东",
-        "武江": "广东",
-        "浈江": "广东",
-        "曲江": "广东",
-        "罗湖": "广东",
-        "福田": "广东",
-        "宝安": "广东",
-        "龙岗": "广东",
-        "盐田": "广东",
-        "香洲": "广东",
-        "斗门区": "广东",
-        "斗门镇": "广东",
-        "金湾区": "广东",
-        "龙湖区": "广东",
-        "濠江": "广东",
-        "潮阳": "广东",
-        "潮南": "广东",
-        "澄海": "广东",
-        "禅城": "广东",
-        "南海区": "广东",
-        "顺德": "广东",
-        "三水": "广东",
-        "高明区": "广东",
-        "蓬江": "广东",
-        "江海区": "广东",
-        "新会": "广东",
-        "赤坎": "广东",
-        "霞山": "广东",
-        "坡头区": "广东",
-        "麻章": "广东",
-        "茂南": "广东",
-        "茂港": "广东",
-        "端州": "广东",
-        "鼎湖": "广东",
-        "惠城": "广东",
-        "惠阳": "广东",
-        "梅江": "广东",
-        "源城": "广东",
-        "清城": "广东",
-        "枫溪": "广东",
-        "湘桥": "广东",
-        "榕城": "广东",
-        "云城": "广东",
-        "青秀": "广西",
-        "江南区": "广西",
-        "西乡塘": "广西",
-        "良庆": "广西",
-        "邕宁": "广西",
-        "鱼峰": "广西",
-        "柳南": "广西",
-        "柳北": "广西",
-        "秀峰区": "广西",
-        "叠彩": "广西",
-        "七星区": "广西",
-        "雁山区": "广西",
-        "万秀区": "广西",
-        "蝶山": "广西",
-        "长洲": "广西",
-        "银海": "广西",
-        "铁山港": "广西",
-        "港口区": "广西",
-        "防城": "广西",
-        "钦南": "广西",
-        "钦北": "广西",
-        "港北": "广西",
-        "港南": "广西",
-        "覃塘": "广西",
-        "玉州": "广西",
-        "右江区": "广西",
-        "八步": "广西",
-        "金城江": "广西",
-        "兴宾": "广西",
-        "江洲": "广西",
-        "秀英区": "海南",
-        "龙华区": "海南",
-        "琼山": "海南",
-        "美兰": "海南",
-        "洋浦经济开发": "海南",
-        "万州": "重庆",
-        "涪陵": "重庆",
-        "渝中": "重庆",
-        "大渡口区": "重庆",
-        "沙坪坝": "重庆",
-        "九龙坡": "重庆",
-        "南岸区": "重庆",
-        "北碚": "重庆",
-        "万盛区": "重庆",
-        "渝北": "重庆",
-        "巴南": "重庆",
-        "黔江": "重庆",
-        "长寿区": "重庆",
-        "江津区": "重庆",
-        "合川": "重庆",
-        "南川": "重庆",
-        "永川": "重庆",
-        "北部新": "重庆",
-        "锦江": "四川",
-        "青羊": "四川",
-        "金牛": "四川",
-        "武侯": "四川",
-        "成华": "四川",
-        "龙泉驿": "四川",
-        "青白江": "四川",
-        "新都区": "四川",
-        "温江区": "四川",
-        "自流井": "四川",
-        "贡井": "四川",
-        "沿滩区": "四川",
-        "仁和区": "四川",
-        "江阳区": "四川",
-        "纳溪": "四川",
-        "龙马潭": "四川",
-        "旌阳": "四川",
-        "涪城": "四川",
-        "游仙": "四川",
-        "元坝": "四川",
-        "朝天区": "四川",
-        "船山": "四川",
-        "安居区": "四川",
-        "五通桥": "四川",
-        "金口河": "四川",
-        "顺庆": "四川",
-        "高坪": "四川",
-        "嘉陵": "四川",
-        "东坡区": "四川",
-        "翠屏": "四川",
-        "通川": "四川",
-        "雨城": "四川",
-        "巴州": "四川",
-        "雁江": "四川",
-        "南明": "贵州",
-        "云岩": "贵州",
-        "花溪": "贵州",
-        "乌当": "贵州",
-        "小河区": "贵州",
-        "金阳新": "贵州",
-        "六枝特": "贵州",
-        "红花岗": "贵州",
-        "汇川": "贵州",
-        "新浦新": "贵州",
-        "西秀": "贵州",
-        "黄果树开发": "贵州",
-        "七星关": "贵州",
-        "碧江": "贵州",
-        "万山特": "贵州",
-        "凯里经济开发": "贵州",
-        "盘龙": "云南",
-        "官渡区": "云南",
-        "西山区": "云南",
-        "东川区": "云南",
-        "麒麟区": "云南",
-        "红塔区": "云南",
-        "隆阳": "云南",
-        "昭阳": "云南",
-        "古城区": "云南",
-        "思茅": "云南",
-        "临翔": "云南",
-        "山南市": "西藏",
-        "阿里地区": "西藏",
-        "碑林": "陕西",
-        "莲湖": "陕西",
-        "灞桥": "陕西",
-        "未央": "陕西",
-        "雁塔": "陕西",
-        "阎良": "陕西",
-        "临潼": "陕西",
-        "沣渭新": "陕西",
-        "王益": "陕西",
-        "印台": "陕西",
-        "耀州": "陕西",
-        "渭滨": "陕西",
-        "金台": "陕西",
-        "陈仓": "陕西",
-        "秦都": "陕西",
-        "渭城": "陕西",
-        "临渭": "陕西",
-        "宝塔区": "陕西",
-        "汉台": "陕西",
-        "榆阳": "陕西",
-        "汉滨": "陕西",
-        "商州": "陕西",
-        "杨凌示范": "陕西",
-        "七里河区": "甘肃",
-        "西固": "甘肃",
-        "红古": "甘肃",
-        "平川": "甘肃",
-        "秦州": "甘肃",
-        "麦积": "甘肃",
-        "凉州": "甘肃",
-        "甘州": "甘肃",
-        "崆峒": "甘肃",
-        "肃州": "甘肃",
-        "西峰": "甘肃",
-        "安定": "甘肃",
-        "武都": "甘肃",
-        "海东": "青海",
-        "兴庆": "宁夏",
-        "西夏区": "宁夏",
-        "金凤": "宁夏",
-        "大武口": "宁夏",
-        "惠农": "宁夏",
-        "利通": "宁夏",
-        "红寺堡": "宁夏",
-        "原州": "宁夏",
-        "沙坡头": "宁夏",
-        "天山区": "新疆",
-        "沙依巴克": "新疆",
-        "水磨沟": "新疆",
-        "头屯河": "新疆",
-        "达坂城": "新疆",
-        "米东": "新疆",
-        "独山子": "新疆",
-        "白碱滩": "新疆",
-        "乌尔禾": "新疆",
-        "中西区": "香港",
-        "九龙城": "香港",
-        "观塘": "香港",
-        "深水埗": "香港",
-        "湾仔区": "香港",
-        "黄大仙": "香港",
-        "油尖旺": "香港",
-        "离岛": "香港",
-        "葵青": "香港",
-        "西贡": "香港",
-        "沙田": "香港",
-        "屯门": "香港",
-        "荃湾": "香港",
-        "元朗": "香港",
-        "花地玛堂": "澳门",
-        "圣安多尼堂": "澳门",
-        "大堂": "澳门",
-        "望德堂": "澳门",
-        "风顺堂": "澳门",
-        "嘉模堂": "澳门",
-        "圣方济各堂": "澳门",
-        "大学": "大学",
-        "学院": "学院",
-        "医院": "医院",
-        "中学": "中学",
-        "小学": "小学",
-        "幼儿园": "幼儿园",
-        "中心": "中心",
-        "公司": "公司",
-        "集团": "集团",
-        "办事处": "办事处",
-        "研究院": "研究院",
-        "研究所": "研究所",
-        "移动": "移动",
-        "联通": "联通",
-        "电信": "电信",
-        "工商": "工商",
-        "公安": "公安",
-        "检察院": "检察院",
-        "检查院": "检查院",
-		"法院":"法院",
-		"司法":"司法",
-		"采购中心":"采购中心",
-		"办公室":"办公室",
-        "地税": "地税",
-        "国税": "国税",
-        "包钢": "包钢",
-        "镇": "镇",
-        "乡": "乡",
-        "市": "市",
-		"局":"局",
-		"厅":"厅",
-		"滇":"滇",
-		"辽":"辽",
-		"沪":"沪",
-		"浙":"浙",
-		"皖":"皖",
-		"闽":"闽",
-		"赣":"赣",
-		"鲁":"鲁",
-		"豫":"豫",
-		"鄂":"鄂",
-		"湘":"湘",
-		"粤":"粤",
-		"桂":"桂",
-		"琼":"琼",
-		"渝":"渝",
-		"川":"川",
-		"藏":"藏",
-		"陕":"陕",
-		"甘":"甘",
-		"黑":"黑",
-		"蒙":"蒙",
-		"吉":"吉",
-		"澳":"澳",
-		"湾":"湾",
-		"山":"山",
-		"河":"河",
-        "首钢": "首钢",
-        "太钢": "太钢",
-        "天钢": "天钢",
-        "国丰": "国丰",
-        "临钢": "临钢",
-        "文丰": "文丰",
-        "天铁": "天铁",
-        "普阳": "普阳",
-        "德龙": "德龙",
-        "港陆": "港陆",
-        "新金": "新金",
-        "宝钢": "宝钢",
-        "南钢": "南钢",
-        "梅钢": "梅钢",
-        "马钢": "马钢",
-        "济钢": "济钢",
-        "莱钢": "莱钢",
-        "沙钢": "沙钢",
-        "新钢": "新钢",
-        "泰钢": "泰钢",
-        "萍钢": "萍钢",
-        "华伟": "华伟",
-        "兆泰": "兆泰",
-        "西城": "西城",
-        "宁钢": "宁钢",
-        "春冶": "春冶",
-        "兆顺": "兆顺",
-        "长达": "长达",
-        "飞达": "飞达 ",
-        "武钢": "武钢",
-        "安钢": "安钢",
-        "涟钢": "涟钢",
-        "湘钢": "湘钢",
-        "鄂钢": "鄂钢",
-        "韶钢": "韶钢",
-        "柳钢": "柳钢",
-        "鞍钢": "鞍钢",
-        "本钢": "本钢",
-        "通钢": "通钢",
-        "北台": "北台",
-        "攀钢": "攀钢",
-        "重钢": "重钢",
-        "昆钢": "昆钢",
-        "包钢": "包钢",
-        "酒钢": "酒钢",
-        "八钢": "八钢"
-    }
-}

fullproject/src2/cleareids.go

@@ -1,110 +0,0 @@
-// cleareids
-package main
-
-import (
-	"encoding/json"
-	"log"
-	"qfw/util"
-	"qfw/util/redis"
-	"sync"
-
-	"github.com/robfig/cron"
-)
-
-var (
-	c            *cron.Cron
-	projectcycle int64
-)
-
-func clearedis() {
-	if credis, ok := Sysconfig["clearedis"].(map[string]interface{}); ok {
-		clearcron := util.ObjToString(credis["clearcron"])
-		projectcycle = util.Int64All(credis["projectcycle"])
-		log.Println(credis)
-		if credis["open"].(bool) {
-			c = cron.New()
-			c.AddFunc(clearcron, clearPKey)
-			c.Start()
-		}
-	}
-}
-
-func clearPKey() {
-	log.Println("开始清理")
-	nowtime := currentMegerTime //time.Now().Unix() int64(1461204000)
-	wg := sync.WaitGroup{}
-	for _, pncb := range []*KeyMap{PNKey, PCKey, PBKey} {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			clearPNCBKey(pncb, nowtime)
-		}()
-	}
-	wg.Wait()
-	log.Println("清理结束")
-}
-
-func clearPNCBKey(pncb *KeyMap, nowtime int64) {
-	delkey := clearIdsKeys(pncb, nowtime)
-	pncb.Lock.Lock()
-	for _, k := range delkey {
-		delete(pncb.Map, k)
-	}
-	pncb.Lock.Unlock()
-}
-
-func clearIdsKeys(pKey *KeyMap, nowtime int64) []string {
-	defer util.Catch()
-	delkey := []string{}
-	pKey.Lock.Lock()
-	for k, ma := range pKey.Map {
-		ids := ma.Arr
-		delids := []interface{}{}
-		res := redis.Mget(REDISIDS, *ids)
-		for _, b1 := range res {
-			if b1 != nil {
-				var info ProjectInfo
-				err := json.Unmarshal(b1.([]byte), &info)
-				if err != nil {
-					log.Println(err)
-					continue
-				}
-				publistime := info.Publistime[len(info.Publistime)-1]
-				if projectcycle < (nowtime-publistime)/86400 { //项目周期超时
-					delids = append(delids, info.Id)
-				}
-			}
-		}
-		if len(delids) < 1 {
-			continue
-		}
-		//删除redis相关信息
-		b := redis.Del(REDISIDS, delids...)
-		if b {
-			for _, id := range delids {
-				ids = deleteSliceId(*ids, id.(string))
-			}
-			//log.Println(ids, *ma.Arr)
-			ma.Arr = ids
-			if len(*ma.Arr) < 1 {
-				redis.Del(REDISKEYS, k)
-				delkey = append(delkey, k)
-			} else {
-				//更新REDISKEYS
-				redis.Put(REDISKEYS, k, *ma.Arr, 0)
-			}
-		}
-	}
-	pKey.Lock.Unlock()
-	return delkey
-}
-
-func deleteSliceId(a []string, id string) *[]string {
-	ret := make([]string, 0, len(a))
-	for _, val := range a {
-		if val != id {
-			ret = append(ret, val)
-		}
-	}
-	return &ret
-}

fullproject/src2/compare.go

@@ -1,427 +0,0 @@
-package main
-
-import (
-	"qfw/util"
-	"qfw/util/redis"
-	"regexp"
-	"sort"
-	"strings"
-	"time"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-/**
-发布时间（结合信息类型-->）
-信息类型（拟建、招标(变更)、结果(成交、中标、流标、废标、变更)、其他(合同、验收、违规、变更)）
-项目代码（长度）
-采购单位（结合省份）
-招标机构
-所属省份
-信息标题（含有二次、标段、包的情况，结合分包情况处理）
-是否分包：
-**/
-type ProjectInfo struct {
-	Id            string                 `json:"id"`
-	Publistime    []int64                `json:"publistime"` //多条信息的发布时间、跨度
-	InfoType      [][]string             `json:"infotype"`   //多条信息内的 toptype、subtype
-	Ids           []string               `json:"ids"`
-	Topscopeclass []string               `json:"topscopeclass"`
-	Subscopeclass []string               `json:"subscopeclass"`
-	Winners       []string               `json:"winners"`
-	ProjectName   string                 `json:"projectname"`
-	ProjectCode   string                 `json:"projectcode"` //项目代码唯一（纯数字的权重低）
-	Buyer         string                 `json:"buyer"`       //采购单位唯一
-	Buyerperson   string                 `json:"buyerperson"`
-	Buyertel      string                 `json:"buyertel"`
-	Agency        string                 `json:"agency"`     //代理机构唯一
-	Area          string                 `json:"area"`       //地区唯一
-	City          string                 `json:"city"`       //地市
-	HasPackage    bool                   `json:"haspackage"` //是否有分包
-	Package       map[string]interface{} `json:"package"`    //分包的对比对象
-
-	Buyerclass  string   `json:"buyerclass"`  //采购单位分类
-	Bidopentime int64    `json:"bidopentime"` //开标时间
-	District    string   `json:"district"`    //区县
-	Winnerorder []string //中标候选人
-}
-
-type CompareOne struct {
-	ProjectNameType string //项目名称对比结果分类 A相等 B包含 C不相等 Dthis存在对比不存在 Ethis不存在
-	ProjectCodeType string //同上
-	BuyerType       string //同上
-	AreaType        string //同上
-	AgencyType      string //同上
-	PublistimeType  int    //1在时间范围  2不在
-	PackageType     int    //1都是多包 2都不是多包 3招标 4新信息是结果
-	Score           int
-	Parent          *CompareInfo
-	Pinfo           *ProjectInfo
-	Pos             int
-
-	Cresult string //对比结果
-}
-
-type CompareInfo struct {
-	Field  string        //对比属性 pn/pc/pb
-	Key    string        //存放rediskey
-	Scores []*CompareOne //对比分值 pinfo索引对应分值
-	Bfind  bool          //是否查找到
-	IdArr  []string
-	K      *Key
-	KeyMap *KeyMap
-}
-
-func NewCompareInfo(field, key string, KeyMap *KeyMap) *CompareInfo {
-	return &CompareInfo{
-		Field:  field,
-		Key:    key,
-		Scores: []*CompareOne{},
-		KeyMap: KeyMap,
-	}
-
-}
-
-var numreg2 = regexp.MustCompile("^[0-9]+$")
-var TitleReg = regexp.MustCompile("([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ、\\-~至]+(子|合同|分|施工|监理|标)?[包标段][号段]?[、]?)+|((子|合同|分|施工|监理|标)?[包标段][号段]?[一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ、\\-~至]+[、]?)+|(子|合同|分|施工|监理|标)?[包标段][号段]?[a-zA-Z0-9]+[\\-~－至、](子|合同|分|施工|监理|标)?[包标段][号段]?[a-zA-Z0-9]+")
-
-//没有在redis中查找到，新增项目
-func InsertProject(new_pn string, tmp, mess map[string]interface{}, pici int64, thisinfo *Info) string {
-	e := InitEL("")
-	set := map[string]interface{}{
-		"pici": pici,
-	}
-	set["s_projectname"] = new_pn
-	e.fieldpriority(&tmp, nil, &set)
-	set["extractpos"] = e.GetVal()
-	set["createtime"] = time.Now().Unix()
-	set["sourceinfoid"] = util.BsonIdToSId(tmp["_id"])
-	set["sourceinfourl"] = tmp["href"]
-	set["topscopeclass"] = thisinfo.Topscopeclass
-	set["subscopeclass"] = thisinfo.Subscopeclass
-	if thisinfo.Buyerperson != "" {
-		set["buyerperson"] = thisinfo.Buyerperson
-	}
-	if thisinfo.Buyertel != "" {
-		set["buyertel"] = thisinfo.Buyertel
-	}
-	if thisinfo.Buyerclass != "" {
-		set["buyertel"] = thisinfo.Buyerclass
-	}
-	if thisinfo.District != "" {
-		set["district"] = thisinfo.District
-	}
-	if thisinfo.Bidopentime > 0 {
-		set["bidopentime"] = thisinfo.Bidopentime
-	}
-	if len(thisinfo.Winnerorder) > 0 {
-		set["winnerorder"] = thisinfo.Winnerorder
-	}
-	s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
-	set["s_subscopeclass"] = s_subscopeclass
-	s_winner := strings.Join(thisinfo.Winners, ",")
-	set["s_winner"] = s_winner
-	if tmp["package"] != nil {
-		set["package"] = tmp["package"] //没定义优先级
-	}
-	push := NewPushInfo(tmp)
-	for tkey, _ := range extractpos {
-		if tmp[tkey] != nil {
-			push[tkey] = tmp[tkey]
-		}
-	}
-	set["list"] = []bson.M{
-		push,
-	}
-	if mess != nil { //项目标识
-		for k, v := range mess {
-			set[k] = v
-		}
-	}
-	return MQFW.Save(projectColl, set)
-}
-
-//生成存放在redis数组中的对象
-func NewPinfo(id string, thisinfo *Info) ProjectInfo {
-	p1 := ProjectInfo{
-		Publistime:    []int64{thisinfo.Publishtime},
-		InfoType:      [][]string{[]string{thisinfo.TopType, thisinfo.SubType}},
-		Id:            id,
-		Ids:           []string{thisinfo.Id},
-		Topscopeclass: thisinfo.Topscopeclass,
-		Subscopeclass: thisinfo.Subscopeclass,
-		Winners:       thisinfo.Winners,
-		ProjectName:   thisinfo.ProjectName,
-		ProjectCode:   thisinfo.ProjectCode,
-		Buyer:         thisinfo.Buyer,
-		Agency:        thisinfo.Agency,
-		Area:          thisinfo.Area,
-		HasPackage:    thisinfo.HasPackage,
-		Package:       map[string]interface{}{},
-		Buyerclass:    thisinfo.Buyerclass,
-		Bidopentime:   thisinfo.Bidopentime,
-		District:      thisinfo.District,
-		Winnerorder:   thisinfo.Winnerorder,
-	}
-	for k4, _ := range thisinfo.Package {
-		p1.Package[k4] = ""
-	}
-	return p1
-}
-
-//每一个字段的打分map,获取最高分
-func GetMaxScore(scoremap map[int]int) (find bool, index int) {
-	ks := []int{}
-	vs := []int{}
-	for k, v := range scoremap {
-		mk, mv := -1, -100
-		bf := false
-		for mk, mv = range vs {
-			if mv > v {
-				bf = true
-				break
-			}
-		}
-		if mk == -1 || !bf {
-			vs = append(vs, v)
-			ks = append(ks, k)
-		} else {
-			vs = append(append(vs[:mk], v), vs[mk:]...)
-			ks = append(append(ks[:mk], k), ks[mk:]...)
-		}
-	}
-	if len(ks) > 0 {
-		index = ks[len(ks)-1]
-		find = true
-	}
-	return
-}
-
-//往对应redis中更新信息
-func updateinfo(thisinfo *Info, tmp map[string]interface{}, pInfo *ProjectInfo, pici int64) string {
-	updateid := pInfo.Id
-	if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
-		return updateid
-	}
-	set := map[string]interface{}{
-		"pici": pici,
-	}
-	res, bres := MQFW.FindById(projectColl, pInfo.Id, `{"list":0}`)
-	EqInfoUpdate(thisinfo, pInfo)
-	if bres && res != nil && *res != nil {
-		set["topscopeclass"] = pInfo.Topscopeclass
-		set["subscopeclass"] = pInfo.Subscopeclass
-		s_subscopeclass := strings.Join(pInfo.Subscopeclass, ",")
-		if len(s_subscopeclass) > 0 {
-			s_subscopeclass = "," + s_subscopeclass + ","
-		}
-		set["s_subscopeclass"] = s_subscopeclass
-		s_winner := strings.Join(pInfo.Winners, ",")
-		if len(s_winner) > 0 {
-			s_winner = "," + s_winner + ","
-		}
-		set["s_winner"] = s_winner
-		if pInfo.Buyerperson != "" && pInfo.Buyertel != "" {
-			set["buyerperson"] = pInfo.Buyerperson
-			set["buyertel"] = pInfo.Buyertel
-		}
-		if pInfo.Buyerclass != "" {
-			set["buyerclass"] = pInfo.Buyerclass
-		}
-		if pInfo.District != "" {
-			set["district"] = pInfo.District
-		}
-		if pInfo.Bidopentime > 0 {
-			set["bidopentime"] = pInfo.Bidopentime
-		}
-		if len(pInfo.Winnerorder) > 0 {
-			set["winnerorder"] = pInfo.Winnerorder
-		}
-		if thisinfo.HasPackage {
-			set["multipackage"] = 1
-		} else {
-			set["multipackage"] = 0
-		}
-		e := InitEL(util.ObjToString((*res)["extractpos"]))
-		if thisinfo.dealtype == 1 {
-			var sonpackage map[string]interface{}
-			for _, obj := range tmp["package"].(map[string]interface{}) {
-				sonpackage, _ = obj.(map[string]interface{})
-			}
-			for _, v2 := range []string{"budget", "budget_w", "winner", "winner_w", "bidstatus", "bidstatus_w"} {
-				if sonpackage[v2] != nil {
-					tmp[v2] = sonpackage[v2]
-				}
-			}
-		}
-		e.fieldpriority(&tmp, res, &set)
-		set["extractpos"] = e.GetVal()
-		if thisinfo.HasPackage { //多包处理
-			p1, _ := (*res)["package"].(map[string]interface{})
-			p2, _ := tmp["package"].(map[string]interface{})
-			if p2 != nil {
-				if p1 != nil {
-					for pk2, pv2 := range p2 {
-						if p1[pk2] != nil { //合并
-							item1, _ := p1[pk2].(map[string]interface{})
-							item2, _ := pv2.(map[string]interface{})
-							if item1 != nil && item2 != nil { //原始项
-								for ik1, iv1 := range item2 {
-									if item1[ik1] == nil {
-										item1[ik1] = iv1
-									}
-								}
-							}
-						} else {
-							p1[pk2] = pv2
-						}
-					}
-				} else {
-					p1 = p2
-				}
-			}
-			set["package"] = p1
-		}
-		//中标候选人合并
-
-		update := map[string]interface{}{}
-		if len(set) > 0 {
-			update["$set"] = set
-		}
-		//保留原数据吧
-		push := NewPushInfo(tmp)
-		for tkey, _ := range extractpos {
-			if tmp[tkey] != nil {
-				push[tkey] = tmp[tkey]
-			}
-		}
-		update["$push"] = map[string]interface{}{
-			"list": push,
-		}
-		if len(update) > 0 {
-			MQFW.Update(projectColl, map[string]interface{}{
-				"_id": util.StringTOBsonId(pInfo.Id),
-			}, &update, false, false)
-		}
-	}
-	//再往redis中放 index
-	//往队列中增加时间 -------------->start
-	redis.Put(REDISIDS, updateid, pInfo, 0)
-	return updateid
-}
-
-func EqInfoUpdate(thisinfo *Info, pInfo *ProjectInfo) {
-	var tk int
-	bf1 := false
-	for _k, tv := range pInfo.Publistime {
-		tk = _k
-		if tv > thisinfo.Publishtime {
-			bf1 = true
-			break
-		}
-	}
-	if bf1 {
-		pInfo.Publistime = append(append(pInfo.Publistime[:tk], thisinfo.Publishtime), pInfo.Publistime[tk:]...)
-		pInfo.InfoType = append(append(pInfo.InfoType[:tk], []string{thisinfo.TopType, thisinfo.SubType}), pInfo.InfoType[tk:]...)
-		pInfo.Ids = append(append(pInfo.Ids[:tk], thisinfo.Id), pInfo.Ids[tk:]...)
-	} else {
-		pInfo.Publistime = append(pInfo.Publistime, thisinfo.Publishtime)
-		pInfo.InfoType = append(pInfo.InfoType, []string{thisinfo.TopType, thisinfo.SubType})
-		pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
-	}
-	//增加发布时间结束----------------->end
-
-	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
-		pInfo.Buyer = thisinfo.Buyer
-	}
-	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
-		pInfo.Agency = thisinfo.Agency
-	}
-	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
-		pInfo.ProjectCode = thisinfo.ProjectCode
-	}
-
-	if thisinfo.Buyerperson != "" && thisinfo.Buyertel != "" && len([]rune(thisinfo.Buyertel)) > 6 {
-		pInfo.Buyerperson = thisinfo.Buyerperson
-		pInfo.Buyertel = thisinfo.Buyertel
-	}
-	if thisinfo.Buyerclass != "" {
-		pInfo.Buyerclass = thisinfo.Buyerclass
-	}
-	if thisinfo.District != "" {
-		pInfo.District = thisinfo.District
-	}
-	if thisinfo.Bidopentime > 0 {
-		pInfo.Bidopentime = thisinfo.Bidopentime
-	}
-	if len(thisinfo.Topscopeclass) > 0 {
-		sort.Strings(pInfo.Topscopeclass)
-		for _, k := range thisinfo.Topscopeclass {
-			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
-				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
-				sort.Strings(pInfo.Topscopeclass)
-			}
-		}
-	}
-
-	if len(thisinfo.Subscopeclass) > 0 {
-		sort.Strings(pInfo.Subscopeclass)
-		for _, k := range thisinfo.Subscopeclass {
-			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
-				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
-				sort.Strings(pInfo.Subscopeclass)
-			}
-		}
-	}
-	//winner
-	if len(thisinfo.Winners) > 0 {
-		sort.Strings(pInfo.Winners)
-		for _, k := range thisinfo.Winners {
-			if BinarySearch(pInfo.Winners, k) == -1 {
-				pInfo.Winners = append(pInfo.Winners, k)
-				sort.Strings(pInfo.Winners)
-			}
-		}
-	}
-	//winnerorder
-	if len(thisinfo.Winnerorder) > 0 {
-		sort.Strings(pInfo.Winnerorder)
-		for _, k := range thisinfo.Winnerorder {
-			if BinarySearch(pInfo.Winnerorder, k) == -1 {
-				pInfo.Winnerorder = append(pInfo.Winnerorder, k)
-				sort.Strings(pInfo.Winnerorder)
-			}
-		}
-	}
-	//return pInfo
-}
-
-//二分字符串查找
-func BinarySearch(s []string, k string) int {
-	sort.Strings(s)
-	lo, hi := 0, len(s)-1
-	for lo <= hi {
-		m := (lo + hi) >> 1
-		if s[m] < k {
-			lo = m + 1
-		} else if s[m] > k {
-			hi = m - 1
-		} else {
-			return m
-		}
-	}
-	return -1
-}
-
-//移除数组中重复的元素
-func RemoveDup(arr []string) (newarr []string) {
-	m1 := map[string]bool{}
-	newarr = []string{}
-	for _, k := range arr {
-		if !m1[k] {
-			m1[k] = true
-			newarr = append(newarr, k)
-		}
-	}
-	return
-}

fullproject/src2/config.json

@@ -1,36 +0,0 @@
-{
-    "mongodbServers": "192.168.3.207:27082",
-    "mongodbPoolSize": 10,
-    "mongodbName": "extract_v3",
-    "jkmail": {
-        "to": "zhangjinkun@topnet.net.cn",
-        "api": "http://10.171.112.160:19281/_send/_mail"
-    },
-    "thread": 1,
-    "extractColl": "bidding20190521",
-    "projectColl": "projectset",
-    "lenprojectname": 18,
-    "redisPoolSize": 60,
-    "redisaddrs": "ids=192.168.3.18:3379,keys=192.168.3.18:3379,info=192.168.3.18:3379",
-    "clearedis": {
-        "open": true,
-        "clearcron": "0 10 15 ? * 4",
-        "projectcycle": 180
-    },
-    "megerfields": {
-        "projectlen": 5,
-        "projectcodelen": 8
-    },
-    "taskstock": {
-        "open": true,
-        "endate": "2019-06-30"
-    },
-    "udpport": ":1482",
-    "nextNode": [
-        {
-            "addr": "127.0.0.1",
-            "port": 1483,
-            "memo": "创建项目索引"
-        }
-    ]
-}

fullproject/src2/datamonitor.go

@@ -1,50 +0,0 @@
-package main
-
-import (
-	"sync"
-	"time"
-)
-
-/**
-增加数据监控服务，每15分钟保存一次抽取情况
-**/
-var IS = &InfoStatus{
-	time.Now().Unix(), 0, map[string]int{}, &sync.Mutex{}}
-
-func init() {
-	go IS.Save()
-}
-
-type InfoStatus struct {
-	Starttime int64
-	Endtime   int64
-	Val       map[string]int
-	Lock      *sync.Mutex
-}
-
-func (is *InfoStatus) Add(t string) {
-	is.Lock.Lock()
-	is.Val[t]++
-	is.Lock.Unlock()
-}
-
-func (is *InfoStatus) Save() {
-	is.Lock.Lock()
-	is.Endtime = time.Now().Unix()
-	save := map[string]interface{}{}
-	all := 0
-	for _, v := range is.Val {
-		all += v
-	}
-	if all > 0 {
-		save["receive"] = all
-		save["starttime"] = is.Starttime
-		save["endtime"] = is.Endtime
-		save["flag"] = "projectser"
-		is.Val = map[string]int{}
-		go MQFW.Save("datamonitor", save)
-	}
-	is.Starttime = is.Endtime
-	is.Lock.Unlock()
-	time.AfterFunc(15*time.Minute, is.Save)
-}

fullproject/src2/extractarea.go

@@ -1,79 +0,0 @@
-package main
-
-import (
-	"qfw/util"
-)
-
-var CityConfig map[string]interface{}
-var AreaGet DFA //敏感词
-
-func initarea() {
-	util.ReadConfig("city.json", &CityConfig)
-	cp := CityConfig["city"].(map[string]interface{})
-	citys := []string{}
-	for k, _ := range cp {
-		citys = append(citys, k)
-	}
-	AreaGet = DFA{}
-	AreaGet.AddWord(citys...)
-}
-
-type DFA struct {
-	Link map[string]interface{}
-}
-
-func (d *DFA) AddWordAll(haskey bool, keys ...string) {
-	if d.Link == nil {
-		d.Link = make(map[string]interface{})
-	}
-	for _, key := range keys {
-		nowMap := &d.Link
-		for i := 0; i < len(key); i++ {
-			kc := key[i : i+1]
-			if v, ok := (*nowMap)[kc]; ok {
-				nowMap, _ = v.(*map[string]interface{})
-			} else {
-				newMap := map[string]interface{}{}
-				newMap["YN"] = "0"
-				(*nowMap)[kc] = &newMap
-				nowMap = &newMap
-			}
-			if i == len(key)-1 {
-				(*nowMap)["YN"] = "1"
-				if haskey {
-					(*nowMap)["K"] = key
-				}
-			}
-		}
-	}
-}
-
-func (d *DFA) AddWord(keys ...string) {
-	d.AddWordAll(false, keys...)
-}
-
-//适合一次查找
-func (d *DFA) CheckSensitiveWord(src string) (b bool) {
-	pos := 0
-	nowMap := &d.Link
-	for i := 0; i < len(src); i++ {
-		word := src[i : i+1]
-		nowMap, _ = (*nowMap)[word].(*map[string]interface{})
-		if nowMap != nil { // 存在，则判断是否为最后一个
-			if pos == 0 {
-				pos = i
-			}
-			if "1" == util.ObjToString((*nowMap)["YN"]) { // 如果为最后一个匹配规则,结束循环，返回匹配标识数
-				b = true
-				break
-			}
-		} else {
-			nowMap = &d.Link
-			if pos > 0 {
-				i = pos
-				pos = 0
-			}
-		}
-	}
-	return
-}

fullproject/src2/fulldata.go

@@ -1,126 +0,0 @@
-package main
-
-import (
-	"log"
-	"qfw/util"
-	"qfw/util/mongodb"
-
-	"qfw/util/redis"
-	"sync"
-	"time"
-)
-
-var FullCount = 0
-
-func RunFullData() {
-	defer util.Catch()
-	var wg = sync.WaitGroup{}
-	startTime := int64(1325347200) //2012-01-01
-	ps := 3
-	pool := make(chan *task, ps)
-	day := 0
-	endChan := make(chan bool, 1)
-	go func() {
-		now := time.Now().Unix()
-		bComplete := false
-		for {
-			if startTime > now || bComplete {
-				log.Println("任务结束")
-				endChan <- true
-				break
-			}
-			endTime := startTime + 86400
-			q := map[string]interface{}{
-				"publishtime": map[string]interface{}{
-					"$gt":  startTime,
-					"$lte": endTime,
-				},
-			}
-
-			//数据正序处理
-			sess := MQFW.GetMgoConn()
-			var result []map[string]interface{}
-			sess.DB(MQFW.DbName).C(extractColl).Find(q).All(&result)
-			MQFW.DestoryMongoConn(sess)
-			pool <- &task{result}
-			wg.Add(1)
-			startTime = endTime
-			day++
-			log.Println("day====", day)
-			if day > 0 && day%ps == 0 {
-				wg.Wait()
-				MQFW.Destory()
-				MQFW = mongodb.MongodbSim{
-					MongodbAddr: Sysconfig["mongodbServers"].(string),
-					Size:        2 * ps,
-					DbName:      Sysconfig["mongodbName"].(string),
-				}
-				MQFW.InitPool()
-			}
-
-		}
-	}()
-
-	for {
-		select {
-		case t := <-pool:
-			t.query()
-			t.result = nil
-			t = nil
-			wg.Done()
-		case <-endChan:
-			return
-		}
-	}
-
-}
-
-type task struct {
-	result []map[string]interface{}
-}
-
-func (t *task) query() {
-	index := 0
-	wg := &sync.WaitGroup{}
-	for _, tmp := range t.result {
-		if index%10000 == 0 {
-			log.Println(index, tmp["_id"])
-		}
-		index++
-		if util.IntAll(tmp["repeat"]) == 1 {
-			continue
-		}
-		pt := util.Int64All(tmp["publishtime"])
-		if pt > currentMegerTime {
-			currentMegerTime = pt
-		}
-		currentMegerCount++
-		if currentMegerCount > 300000 {
-			log.Println("执行清理", currentMegerTime)
-			clearPKey()
-			currentMegerCount = 0
-		}
-		wg.Add(1)
-		MultiThread <- true
-		go func(tmp map[string]interface{}) {
-			defer func() {
-				<-MultiThread
-				wg.Done()
-			}()
-			thisid := util.BsonIdToSId(tmp["_id"])
-			info := PreThisInfo(tmp)
-			if info != nil {
-				lockPNCBMap(info)
-				startProjectMerge(info, tmp)
-				redis.Put(INFOID, thisid, 1, INFOTIMEOUT)
-				currentMegerTime = info.Publishtime
-				unlockPNCBMap(info)
-			}
-		}(tmp)
-	}
-	wg.Wait()
-	FullCount += index
-
-	log.Println("currentFull", FullCount)
-
-}

fullproject/src2/handleproject.go

@@ -1,205 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"qfw/util"
-	"regexp"
-	"strings"
-)
-
-var pnreg = regexp.MustCompile("^(及编号[：:])|(项目|采购|招标|中标|成交|结果|[_]|公告)$")
-var numreg = regexp.MustCompile("^[0-9]$")
-
-/**
-0 项目名称不是多包结构,直接按项目名称查找
-1 项目名称多包,截去多包查找，再按不截取的查找
-2 非招标项目名称含单个子包,截取多包查找，再搂
-**/
-func handleprojectname(pn, buyer, toptype string) (string, int) {
-	for i := 0; i < 4; i++ {
-		opn := pn
-		pn = pnreg.ReplaceAllString(opn, "")
-		if opn == pn {
-			break
-		}
-	}
-	//return pn
-	if len([]rune(pn)) > lenprojectname || strings.Index(pn, buyer) > -1 || AreaGet.CheckSensitiveWord(pn) {
-		return pn, 0
-	} else {
-		return buyer + pn, 0
-	}
-}
-
-//对字段的优先级处理
-type ExtractLevel struct {
-	Strarr []string
-}
-
-func InitEL(v string) (e *ExtractLevel) {
-	if v == "" {
-		v = "0999999999"
-		//第1位一下标为0，不用
-	}
-	Strarr := strings.Split(v, "")
-	e = &ExtractLevel{Strarr}
-	return
-}
-
-func (e *ExtractLevel) PutVal(field, val string) {
-	e.Strarr[extractpos[field]] = val
-}
-
-func (e *ExtractLevel) GetVal() string {
-	return strings.Join(e.Strarr, "")
-}
-
-//字段优先级处理
-func (e *ExtractLevel) fieldpriority(info, project, set *map[string]interface{}) {
-	defer util.Catch()
-	for _k, v := range extractpos {
-		breplace := false
-		var _infov interface{}
-		_infopos := fmt.Sprintf("%d", (*info)[_k+"_w"])
-		if !numreg.MatchString(_infopos) {
-			_infopos = "9"
-		}
-		if project == nil {
-			breplace = true
-			_infov = (*info)[_k]
-		} else {
-			ty := cextractpos[_k]
-			switch ty {
-			case 's':
-				_v := util.ObjToString((*project)[_k])
-				_tv := util.ObjToString((*info)[_k])
-				_infov = _tv
-				if _v == "" && _tv != "" {
-					breplace = true
-				} else if _tv != "" { //对比
-					if e.Strarr[v] > _infopos {
-						breplace = true
-					}
-				}
-			case 'l':
-				_v := util.Float64All((*project)[_k])
-				_tv := util.Float64All((*info)[_k])
-				_infov = _tv
-				if _v == 0 && _tv > 0 {
-					breplace = true
-				} else if _tv > 0 {
-					if e.Strarr[v] > _infopos {
-						breplace = true
-					}
-				}
-
-			}
-		}
-		if breplace {
-			(*set)[_k] = _infov
-			e.PutVal(_k, _infopos)
-		}
-	}
-	otherfieldpriority(info, project, set)
-}
-
-//抽取字段
-var cextractpos = map[string]rune{
-	"projectname":  's',
-	"projectcode":  's',
-	"buyer":        's',
-	"winner":       's',
-	"budget":       'l',
-	"bidamount":    'l',
-	"bidstatus":    's',
-	"agency":       's',
-	"projectscope": 's',
-}
-var extractpos = map[string]int{
-	"projectname":  0,
-	"projectcode":  1,
-	"buyer":        2,
-	"winner":       3,
-	"budget":       4,
-	"bidamount":    5,
-	"bidstatus":    6,
-	"agency":       7,
-	"projectscope": 8,
-}
-
-//合并字段
-//"area","city","topscopeclass","subscopeclass","bidtype","zbtime","jgtime","buyerclass"
-var bidtype = map[string]string{
-	"招标": "招标",
-	"邀标": "邀标",
-	"询价": "询价",
-	"竞谈": "竞谈",
-	"单一": "单一",
-	"竞价": "竞价",
-}
-
-func otherfieldpriority(info, project, set *map[string]interface{}) {
-	area2 := util.ObjToString((*info)["area"])
-	infoformat := util.IntAll((*info)["infoformat"])
-	toptype := util.ObjToString((*info)["toptype"])
-	topscopeclass := util.ObjToString((*info)["topscopeclass"])
-	subscopeclass := util.ObjToString((*info)["subscopeclass"])
-	bt := bidtype[util.ObjToString((*info)["subtype"])]
-	if project == nil {
-		if area2 == "" || area2 == "全国" {
-			area2 = "A"
-		}
-		(*set)["area"] = area2
-		(*set)["city"] = util.ObjToString((*info)["city"])
-		(*set)["topscopeclass"] = topscopeclass
-		(*set)["subscopeclass"] = subscopeclass
-		if bt == "" {
-			bt = "招标"
-		}
-		(*set)["bidtype"] = bt
-		if infoformat == 2 {
-			(*set)["njtime"] = (*info)["publishtime"]
-		} else {
-			if toptype == "招标" {
-				(*set)["zbtime"] = (*info)["publishtime"]
-			} else if toptype == "结果" {
-				(*set)["jgtime"] = (*info)["publishtime"]
-			}
-		}
-		if (*set)["zbtime"] == nil {
-			(*set)["zbtime"] = (*info)["publishtime"]
-		}
-	} else {
-		area := util.ObjToString((*project)["area"])
-		if area == "A" && (area2 != "" && area2 != "A" && area2 != "全国") {
-			(*set)["area"] = area2
-		}
-		publishtime := util.Int64All((*info)["publishtime"])
-		if infoformat == 2 {
-			//拟建信息，先不判断
-			if publishtime > 0 {
-				(*set)["njtime"] = publishtime
-			}
-		} else {
-			//招标信息
-			if toptype == "招标" {
-				zbtime := util.Int64All((*project)["zbtime"])
-				if publishtime > 0 && publishtime < zbtime {
-					(*set)["zbtime"] = publishtime
-				}
-			} else if toptype == "结果" {
-				jgtime := util.Int64All((*project)["jgtime"])
-				if publishtime > 0 && publishtime > jgtime {
-					(*set)["jgtime"] = publishtime
-				}
-			}
-		}
-		if subscopeclass != "" {
-			(*set)["topscopeclass"] = topscopeclass
-			(*set)["subscopeclass"] = subscopeclass
-		}
-		if bt != "" && bt != "招标" {
-			(*set)["bidtype"] = bt
-		}
-	}
-}

fullproject/src2/main.go

@@ -1,101 +0,0 @@
-package main
-
-import (
-	"qfw/util"
-	"qfw/util/mongodb"
-	"qfw/util/redis"
-	"sync"
-	"time"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-const (
-	REDISIDS    = "ids"
-	REDISKEYS   = "keys"
-	INFOID      = "info"
-	INFOTIMEOUT = 86400 * 30
-)
-
-var (
-	Sysconfig                map[string]interface{}
-	MQFW                     mongodb.MongodbSim
-	extractColl, projectColl string
-	lenprojectname           int
-	MultiThread              chan bool
-	IdLock                   = &sync.Mutex{}
-	PncbMayLock              = &sync.Mutex{}
-	MegerFieldsLen           *MegerFields
-	//三组lock,对应的（PNKey）key为项目名称,值对应的是此项目名称对应的项目id数组
-	PNKey, PCKey, PBKey          = NewKeyMap(), NewKeyMap(), NewKeyMap()
-	PNKeyMap, PCKeyMap, PBKeyMap = sync.Map{}, sync.Map{}, sync.Map{}
-
-	currentMegerTime  int64 //合并项目的时间位置，用来清理几个月之前的项目
-	currentMegerCount int   //合并项目的计数，用来定时清理
-)
-
-type MegerFields struct {
-	ProjectNamelen int
-	ProjectCodelen int
-}
-
-type KeyMap struct {
-	Lock sync.Mutex
-	Map  map[string]*Key
-}
-
-type Key struct {
-	Arr  *[]string
-	Lock *sync.Mutex
-}
-
-func NewKeyMap() *KeyMap {
-	return &KeyMap{
-		Map: map[string]*Key{},
-	}
-}
-
-func init() {
-	initarea()
-
-	util.ReadConfig(&Sysconfig)
-	MultiThread = make(chan bool, util.IntAllDef(Sysconfig["thread"], 200))
-	lenprojectname = util.IntAllDef(Sysconfig["lenprojectname"], 20) - 1
-	megerfields, _ := Sysconfig["megerfields"].(map[string]interface{})
-	MegerFieldsLen = &MegerFields{
-		ProjectNamelen: util.IntAllDef(megerfields["projectlen"], 5),
-		ProjectCodelen: util.IntAllDef(megerfields["projectcodelen"], 8),
-	}
-	redis.InitRedisBySize(Sysconfig["redisaddrs"].(string), util.IntAllDef(Sysconfig["redisPoolSize"], 100), 30, 300)
-	MQFW = mongodb.MongodbSim{
-		MongodbAddr: Sysconfig["mongodbServers"].(string),
-		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
-		DbName:      Sysconfig["mongodbName"].(string),
-	}
-	MQFW.InitPool()
-	extractColl = Sysconfig["extractColl"].(string)
-	projectColl = Sysconfig["projectColl"].(string)
-}
-
-func main() {
-	RunFullData()
-	time.Sleep(99999 * time.Hour)
-}
-
-func NewPushInfo(tmp map[string]interface{}) bson.M {
-	return bson.M{
-		"comeintime":  tmp["comeintime"],
-		"publishtime": tmp["publishtime"],
-		"title":       tmp["title"],
-		"toptype":     tmp["toptype"],
-		"subtype":     tmp["subtype"],
-		"infoformat":  tmp["infoformat"],
-		"infoid":      util.BsonIdToSId(tmp["_id"]),
-		"href":        tmp["href"],
-		"area":        tmp["area"],
-		"city":        tmp["city"],
-
-		"cresult": tmp["cresult"],
-		"score":   tmp["score"],
-	}
-}

fullproject/src2/main_test.go

@@ -1,58 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"testing"
-)
-
-func Test_main(t *testing.T) {
-	m1 := map[string]*[]string{}
-	s := m1["dd"]
-	log.Println("aaaaa1")
-	func() {
-		if s == nil {
-			defer func() {
-				log.Println("defer")
-			}()
-			s = &[]string{}
-			m1["dd"] = s
-			log.Println("aaa3")
-		}
-	}()
-	n := 0
-	for {
-		defer func(n int) {
-			log.Println("for", n)
-		}(n)
-		n++
-		log.Println(n)
-		if n > 2 {
-			break
-		}
-	}
-	log.Println("aaa2")
-	*s = append(*s, "aa")
-	log.Println("---m1[dd]:", m1["dd"])
-
-	//arr := []string{"aa", "bb", "cc", "dd", "ee", "abdsfsdfsd"}
-	//log.Println("二分查找", binarySearch(arr, "bb"))
-}
-func Test_3t2(t *testing.T) {
-	pcb := [][]string{
-		[]string{"A", "B"},
-		[]string{"A", "B"},
-		[]string{"A", "B"},
-	}
-	for _, v1 := range pcb[0] {
-		tmp := []string{}
-		for _, v2 := range pcb[1] {
-			for _, v3 := range pcb[1] {
-				key := fmt.Sprintf("%s%s%s", v1, v2, v3)
-				tmp = append(tmp, key)
-			}
-		}
-		log.Println(tmp)
-	}
-
-}

fullproject/src2/projectmeger.go

@@ -1,615 +0,0 @@
-// projectmeger
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	du "jy/util"
-	"log"
-	qu "qfw/util"
-	"qfw/util/redis"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-)
-
-//有效值三选一、三选二
-var ThreeToTow, ThreeToOne map[string]bool
-
-func init() {
-	ThreeToTow = map[string]bool{}
-	ThreeToOne = map[string]bool{
-		"AAA": true,
-		"AAB": true,
-	}
-	tows := [][][]string{
-		[][]string{
-			[]string{"A", "B"},
-			[]string{"A", "B"},
-			[]string{"A", "B"},
-		},
-		[][]string{
-			[]string{"A", "B"},
-			[]string{"D", "E"},
-			[]string{"A", "B"},
-		},
-		[][]string{
-			[]string{"A", "B"},
-			[]string{"A", "B"},
-			[]string{"D", "E"},
-		},
-		[][]string{
-			[]string{"D", "E"},
-			[]string{"A", "B"},
-			[]string{"A", "B"},
-		},
-	}
-	for _, tow := range tows {
-		for _, v1 := range tow[0] {
-			for _, v2 := range tow[1] {
-				for _, v3 := range tow[2] {
-					key := fmt.Sprintf("%s%s%s", v1, v2, v3)
-					log.Println(key)
-					ThreeToTow[key] = true
-				}
-			}
-		}
-	}
-}
-
-func startProjectMerge(thisinfo *Info, tmp map[string]interface{}) {
-	bNormalScore := false
-	pcbv := PCBVal(tmp)
-	if checkInfoAlter(tmp) { //进入变更信息流程
-		if pcbv.Val > 1 { //判断三项至少包含两项
-			bNormalScore = true
-		} else {
-			extInfoTag("invalid", qu.BsonIdToSId(tmp["_id"])) //无效信息，打标记
-			//go IS.Add("invalid")                              //数据统计使用
-			return
-		}
-	} else {
-		bNormalScore = true
-	}
-	//合并流程
-	if bNormalScore {
-		PNKeyMap.Store(thisinfo.PNKey, true)
-		PBKeyMap.Store(thisinfo.PBKey, true)
-		PCKeyMap.Store(thisinfo.PCKey, true)
-		if pcbv.Buyer { //有采购单位
-			hasBuyer(pcbv, thisinfo, tmp)
-		} else { //无采购单位
-			noBuyer(pcbv, thisinfo, tmp)
-		}
-	}
-}
-
-//判断信息是否是变更
-func checkInfoAlter(tmp map[string]interface{} /*新信息*/) bool {
-	toptype := qu.ObjToString(tmp["toptype"])
-	subtype := qu.ObjToString(tmp["subtype"])
-	title := qu.ObjToString(tmp["title"])
-	if subtype == "变更" || strings.Index(title, "变更公告") > -1 || strings.Index(title, "更正公告") > -1 {
-		//当信息类型是变更或标题中含变更时
-		if toptype == "招标" {
-			//招标的变更公告，不作处理
-		} else if toptype == "结果" {
-			subtype = "变更"
-		}
-	}
-	return subtype == "变更"
-}
-
-//有采购单位
-func hasBuyer(p PCBV, thisinfo *Info, tmp map[string]interface{}) {
-	var pncb []*CompareInfo
-	var res []interface{}
-	sflag := ""
-	pici := time.Now().Unix()
-	if p.Pname || p.Pcode { //有项目名称或项目编号
-		//获取对比项目数组
-		res, pncb = getComeperProjects(p, thisinfo)
-		//三选二打分
-		scores := score3Select2(p, thisinfo, tmp, res, pncb)
-		//项目合并
-		sflag = mergeProject(tmp, thisinfo, scores, pncb)
-	} else {
-		//生成项目，不参与后续对比
-		sflag = "alone"
-		mess := map[string]interface{}{
-			"meger_mess":  "有采购单位,不满足三选二",
-			"meger_sflag": sflag,
-		}
-		newProject(tmp, mess, pici, thisinfo)
-	}
-	extInfoTag(sflag, thisinfo.Id)
-	//go IS.Add(sflag) //数据统计使用
-}
-
-//无采购单位
-func noBuyer(p PCBV, thisinfo *Info, tmp map[string]interface{}) {
-	var pncb []*CompareInfo
-	var res []interface{}
-	sflag := ""
-	pici := time.Now().Unix()
-	if p.Pname { //有项目名称
-		//获取对比项目数组
-		res, pncb = getComeperProjects(p, thisinfo)
-		if p.Pcode { //有项目编号
-			//三选二打分
-			scores := score3Select2(p, thisinfo, tmp, res, pncb)
-			//项目合并
-			sflag = mergeProject(tmp, thisinfo, scores, pncb)
-		} else { //无项目编号
-			if p.PnameLen > MegerFieldsLen.ProjectNamelen {
-				//三选一打分
-				scores := score3Select1(p, thisinfo, tmp, res, pncb)
-				//项目合并
-				sflag = mergeProject(tmp, thisinfo, scores, pncb)
-			} else {
-				//生成项目，不参与后续对比
-				sflag = "alone"
-				mess := map[string]interface{}{
-					"meger_mess":  "无采购单位,不满足三选一",
-					"meger_sflag": sflag,
-				}
-				newProject(tmp, mess, pici, thisinfo)
-			}
-		}
-	} else { //无项目名称
-		if p.Pcode {
-			//获取对比项目数组
-			res, pncb = getComeperProjects(p, thisinfo)
-			if p.PcodeLen > MegerFieldsLen.ProjectCodelen {
-				if p.Area && p.City && p.Agency { //有省市代理机构
-					//三选一打分
-					scores := score3Select1(p, thisinfo, tmp, res, pncb)
-					//项目合并
-					sflag = mergeProject(tmp, thisinfo, scores, pncb)
-				} else {
-					//生成项目，不参与后续对比
-					sflag = "alone"
-					mess := map[string]interface{}{
-						"meger_mess":  "无采购单位,不满足三选一",
-						"meger_sflag": sflag,
-					}
-					newProject(tmp, mess, pici, thisinfo)
-				}
-			} else {
-				//生成项目，不参与后续对比
-				sflag = "alone"
-				mess := map[string]interface{}{
-					"meger_mess":  "无采购单位,不满足三选一",
-					"meger_sflag": sflag,
-				}
-				newProject(tmp, mess, pici, thisinfo)
-			}
-		} else {
-			//信息无效,打标记
-			sflag = "invalid"
-		}
-	}
-	extInfoTag(sflag, thisinfo.Id)
-	//go IS.Add(sflag) //数据统计使用
-}
-
-//3选2打分
-func score3Select2(p PCBV, thisinfo *Info, tmp map[string]interface{}, res []interface{}, pncb []*CompareInfo) (scores []*CompareOne) {
-	defer qu.Catch()
-	if len(res) > 0 {
-		//对比打分
-		for k, tmps := range res { //遍历对象数组
-			if tmp, ok := tmps.([]interface{}); ok {
-				for _, b1 := range tmp {
-					if b1 != nil {
-						var info ProjectInfo
-						err := json.Unmarshal(b1.([]byte), &info)
-						if err != nil {
-							log.Println(err)
-						}
-						cone := &CompareOne{
-							Parent: pncb[k],
-							Pinfo:  &info,
-						}
-						cone.BuyerType, cone.Score = fieldPCBScore(thisinfo.Buyer, info.Buyer, cone.BuyerType, cone.Score)
-						if p.Buyer {
-							cone.ProjectNameType, cone.Score = fieldPCBScore(thisinfo.ProjectName, info.ProjectName, cone.ProjectNameType, cone.Score)
-							cone.ProjectCodeType, cone.Score = fieldPCBScore(thisinfo.ProjectCode, info.ProjectCode, cone.ProjectCodeType, cone.Score)
-						} else { //无采购单位，打分考虑长度
-							if len([]rune(thisinfo.ProjectName)) > MegerFieldsLen.ProjectNamelen {
-								cone.ProjectNameType, cone.Score = fieldPCBScore(thisinfo.ProjectName, info.ProjectName, cone.ProjectNameType, cone.Score)
-							} else {
-								cone.ProjectNameType = "D"
-							}
-							if len(thisinfo.ProjectCode) > MegerFieldsLen.ProjectCodelen {
-								cone.ProjectCodeType, cone.Score = fieldPCBScore(thisinfo.ProjectCode, info.ProjectCode, cone.ProjectCodeType, cone.Score)
-							} else {
-								cone.ProjectCodeType = "D"
-							}
-						}
-						//省市打分
-						if thisinfo.Area != "A" && thisinfo.Area != "全国" && info.Area != "A" && info.Area != "全国" {
-							if thisinfo.Area == info.Area && thisinfo.City == info.City {
-								cone.Score += 2
-							} else {
-								cone.Score -= 1
-							}
-						} else {
-							cone.Score += 1
-						}
-						//代理机构打分
-						if len([]rune(info.Agency)) > 0 {
-							if thisinfo.Agency == info.Agency { //A
-								cone.Score += 2
-							} else if strings.Index(info.Agency, thisinfo.Agency) > -1 || strings.Index(thisinfo.Agency, info.Agency) > -1 { //B
-								cone.Score += 1
-							} else {
-								if len(thisinfo.Agency) < 1 { //E
-									cone.Score -= 1
-								} else { //C
-									cone.Score -= 2
-								}
-							}
-						} else { //D不计分
-							//
-						}
-						skey := fmt.Sprintf("%s%s%s", cone.BuyerType, cone.ProjectNameType, cone.ProjectCodeType)
-						cone.Cresult = skey
-						if ThreeToTow[skey] {
-							scores = append(scores, cone)
-						}
-					}
-				}
-			}
-		}
-	}
-	return
-}
-
-//3选1打分
-func score3Select1(p PCBV, thisinfo *Info, tmp map[string]interface{}, res []interface{}, pncb []*CompareInfo) (scores []*CompareOne) {
-	defer qu.Catch()
-	//对比打分
-	if len(res) > 0 { //找到项目名称、项目编号或采购单位相同时
-		for k, tmps := range res { //遍历对象数组
-			if tmp, ok := tmps.([]interface{}); ok {
-				for _, b1 := range tmp {
-					if b1 != nil {
-						var info ProjectInfo
-						err := json.Unmarshal(b1.([]byte), &info)
-						if err != nil {
-							log.Println(err)
-						}
-						cone := &CompareOne{
-							Parent: pncb[k],
-							Pinfo:  &info,
-						}
-						if pncb[k].Field == "pn" { //比较字段项目名称
-							if len(info.ProjectName) > 0 {
-								if thisinfo.ProjectName == info.ProjectName { //A
-									cone.Score += 2
-									cone.ProjectNameType = "A"
-								} else if strings.Index(info.ProjectName, thisinfo.ProjectName) > -1 || strings.Index(thisinfo.ProjectName, info.ProjectName) > -1 { //B
-									cone.Score += 1
-									cone.ProjectNameType = "B"
-								} else { //C
-									cone.Score -= 2
-									cone.ProjectNameType = "C"
-								}
-							} else { //D不计分
-								cone.ProjectNameType = "D"
-							}
-						}
-						if pncb[k].Field == "pc" { //比较字段项目编号
-							if len(info.ProjectCode) > 0 {
-								if thisinfo.ProjectCode == info.ProjectCode { //A
-									cone.Score += 2
-									cone.ProjectCodeType = "A"
-								} else if strings.Index(info.ProjectCode, thisinfo.ProjectCode) > -1 || strings.Index(thisinfo.ProjectCode, info.ProjectCode) > -1 { //B
-									cone.Score += 1
-									cone.ProjectCodeType = "B"
-								} else { //C
-									cone.Score -= 2
-									cone.ProjectCodeType = "C"
-								}
-							} else { //D不计分
-								cone.ProjectCodeType = "D"
-							}
-						}
-						if thisinfo.Area != "A" && thisinfo.Area != "全国" && info.Area != "A" && info.Area != "全国" {
-							if thisinfo.Area == info.Area && thisinfo.City == info.City {
-								cone.Score += 2
-								cone.AreaType = "A"
-							} else {
-								cone.Score -= 1
-								cone.AreaType = "C"
-							}
-						} else {
-							cone.Score += 1
-							cone.AreaType = "B"
-						}
-						if len([]rune(info.Agency)) > 0 {
-							if thisinfo.Agency == info.Agency { //A
-								cone.Score += 2
-								cone.AgencyType = "A"
-							} else if strings.Index(info.Agency, thisinfo.Agency) > -1 || strings.Index(thisinfo.Agency, info.Agency) > -1 { //B
-								cone.Score += 1
-								cone.AgencyType = "B"
-							} else {
-								if len(thisinfo.Agency) < 1 { //E
-									cone.Score -= 1
-									cone.AgencyType = "E"
-								} else { //C
-									cone.Score -= 2
-									cone.AgencyType = "C"
-								}
-							}
-						} else { //D不计分
-							cone.AgencyType = "D"
-						}
-						skey := fmt.Sprintf("%s%s%s", cone.BuyerType, cone.ProjectNameType, cone.ProjectCodeType)
-						cone.Cresult = skey
-						if ThreeToOne[skey] {
-							scores = append(scores, cone)
-						}
-					}
-				}
-			}
-		}
-	}
-	return
-}
-
-//获取对比项目数组
-func getComeperProjects(p PCBV, thisinfo *Info) (res []interface{}, pncb []*CompareInfo) {
-	if p.PnameLen > 0 {
-		pn := NewCompareInfo("pn", thisinfo.PNKey, PNKey)
-		pncb = append(pncb, pn)
-	}
-	if p.PcodeLen > 0 {
-		pc := NewCompareInfo("pc", thisinfo.PCKey, PCKey)
-		pncb = append(pncb, pc)
-	}
-	if p.BuyerLen > 0 {
-		pb := NewCompareInfo("pb", thisinfo.PBKey, PBKey)
-		pncb = append(pncb, pb)
-	}
-	repeatId := map[string]bool{}
-	IdLock.Lock() //此处加id锁，会引进多线程的死锁，对比三个大map数组，找到key相同的项目id数组,并去重
-	for _, pv := range pncb {
-		if pv != nil {
-			pv.KeyMap.Lock.Lock()
-			K := pv.KeyMap.Map[pv.Key]
-			if K == nil {
-				K = &Key{&[]string{}, &sync.Mutex{}}
-				pv.KeyMap.Map[pv.Key] = K
-			}
-			pv.K = K
-			pv.K.Lock.Lock()
-			pv.KeyMap.Lock.Unlock()
-			defer pv.K.Lock.Unlock()
-			newarr := []string{}
-			for _, id := range *K.Arr {
-				if !repeatId[id] {
-					newarr = append(newarr, id)
-					repeatId[id] = true
-				}
-			}
-			pv.IdArr = newarr
-		}
-	}
-	IdLock.Unlock()
-	for _, pv := range pncb {
-		if len(pv.IdArr) > 0 {
-			res = append(res, redis.Mget(REDISIDS, pv.IdArr))
-		}
-	}
-	return
-}
-
-//合并项目
-func mergeProject(tmp map[string]interface{}, thisinfo *Info, scores []*CompareOne, pncb []*CompareInfo) (sflag string) {
-	var id = ""
-	//分值排序
-	sort.Slice(scores, func(i, j int) bool {
-		return scores[i].Score > scores[j].Score
-	})
-	if len(scores) > 0 {
-		//分值排序
-		sort.Slice(scores, func(i, j int) bool {
-			return scores[i].Score > scores[j].Score
-		})
-		max := scores[0]
-		if max.Score > 0 {
-			sflag = "repeat"
-			max.Parent.Bfind = true
-			tmp["cresult"] = max.Cresult
-			tmp["score"] = max.Score
-			id = updateinfo(thisinfo, tmp, max.Pinfo, time.Now().Unix())
-			//log.Println(sflag, id, max.Cresult)
-			if len(scores) > 1 && (scores[0].Score == scores[1].Score) {
-				//通知检查,异常数据存库
-				checkNotice(thisinfo.Id, id, scores)
-			}
-		} else {
-			sflag = "invalidscore"
-		}
-	} else {
-		//生成项目
-		sflag = "normal"
-		mess := map[string]interface{}{
-			"meger_sflag": sflag,
-		}
-		id = newProject(tmp, mess, time.Now().Unix(), thisinfo)
-	}
-	if id != "" { //更新REDISKEYS redis
-		put := []interface{}{}
-		for _, pv := range pncb {
-			if pv != nil && !pv.Bfind {
-				if BinarySearch(*pv.K.Arr, id) == -1 {
-					*(pv.K.Arr) = append(*(pv.K.Arr), id)
-					put = append(put, []interface{}{pv.Key, *(pv.K.Arr)})
-				}
-			}
-		}
-		if len(put) > 0 {
-			redis.BulkPut(REDISKEYS, 0, put...)
-		}
-	}
-	return sflag
-}
-
-//新增项目
-func newProject(tmp, mess map[string]interface{}, pipc int64, thisinfo *Info) (id string) {
-	id = InsertProject(thisinfo.NewPNKey, tmp, mess, pipc, thisinfo)
-	sflag := qu.ObjToString(mess["meger_sflag"])
-	if sflag != "alone" {
-		p1 := NewPinfo(id, thisinfo)
-		redis.PutCKV(REDISIDS, id, p1)
-	} else {
-		du.Debug("新增项目，不参与对比", id)
-	}
-	return id
-}
-
-//抽取信息打标记
-func extInfoTag(sflag, id string) {
-	MQFW.UpdateById(extractColl, id,
-		map[string]interface{}{
-			"$set": map[string]interface{}{
-				"meger_sflag": sflag,
-			},
-		})
-}
-
-//检查通知
-func checkNotice(infoid, pid string, scores []*CompareOne) {
-	MQFW.Update("project_unusual", `{"project_id":"`+pid+`"}`,
-		map[string]interface{}{
-			"$set": map[string]interface{}{
-				"project_id": pid,
-				"pici":       time.Now().Unix(),
-			},
-			"$push": map[string]interface{}{
-				"list": map[string]interface{}{
-					"infoid": infoid,
-					"pinfo1": scores[0].Pinfo.Id,
-					"pinfo2": scores[1].Pinfo.Id,
-				},
-			},
-		}, true, false)
-	du.Debug("通知检查,异常数据存库", infoid, pid)
-}
-
-//属性判断组
-func PCBVal(tmp map[string]interface{}) PCBV {
-	pcbv := PCBV{}
-	projectName := qu.ObjToString(tmp["projectname"])
-	if len([]rune(projectName)) > 1 {
-		pcbv.Val += 1
-		pcbv.Pname = true
-		pcbv.PnameLen = len([]rune(projectName))
-	}
-	projectCode := qu.ObjToString(tmp["projectcode"])
-	if len([]rune(projectCode)) > 1 {
-		pcbv.Val += 1
-		pcbv.Pcode = true
-		pcbv.PcodeLen = len(projectCode)
-	}
-	buyer := qu.ObjToString(tmp["buyer"])
-	if len([]rune(buyer)) > 1 {
-		pcbv.Val += 1
-		pcbv.Buyer = true
-		pcbv.BuyerLen = len([]rune(buyer))
-	}
-	//省市代理机构
-	if qu.ObjToString(tmp["area"]) != "" {
-		pcbv.Area = true
-	}
-	if qu.ObjToString(tmp["city"]) != "" {
-		pcbv.City = true
-	}
-	if qu.ObjToString(tmp["agency"]) != "" {
-		pcbv.Agency = true
-	}
-	return pcbv
-}
-
-//项目名称、项目编号、采购单位打分
-func fieldPCBScore(this, info, ctype string, score int) (string, int) {
-	if len(info) > 0 {
-		if this == info { //A
-			score += 5
-			ctype = "A"
-		} else if strings.Index(info, this) > -1 || strings.Index(this, info) > -1 { //B
-			score += 2
-			ctype = "B"
-		} else {
-			if len(this) < 1 { //E
-				score -= 1
-				ctype = "E"
-			} else { //C
-				score -= 2
-				ctype = "C"
-			}
-		}
-	} else { //D不计分
-		ctype = "D"
-	}
-	return ctype, score
-}
-
-//pncbMap加锁
-func lockPNCBMap(thisinfo *Info) {
-	for { //等待其他任务完成
-		ok := true
-		if len(thisinfo.PBKey) > 3 {
-			if _, b := PBKeyMap.Load(thisinfo.PBKey); b {
-				ok = false
-			}
-		}
-		if ok && len(thisinfo.PNKey) > 3 {
-			PNKeyMap.Range(func(k, v interface{}) bool {
-				if strings.Contains(k.(string), thisinfo.PNKey) || strings.Contains(thisinfo.PNKey, k.(string)) {
-					ok = false
-					return false
-				}
-				return true
-			})
-			//			if _, b := PNKeyMap.Load(thisinfo.PNKey); b {
-			//				ok = false
-			//			}
-		}
-		if ok && len(thisinfo.PCKey) > 3 {
-			PCKeyMap.Range(func(k, v interface{}) bool {
-				if strings.Contains(k.(string), thisinfo.PCKey) || strings.Contains(thisinfo.PCKey, k.(string)) {
-					ok = false
-					return false
-				}
-				return true
-			})
-			//			if _, b := PCKeyMap.Load(thisinfo.PCKey); b {
-			//				ok = false
-			//			}
-		}
-		if ok {
-			break
-		} else {
-			time.Sleep(10 * time.Millisecond)
-		}
-	}
-}
-
-//pncbMap解锁
-func unlockPNCBMap(thisinfo *Info) {
-	//if len(thisinfo.PNKey) > 3 {
-	PNKeyMap.Delete(thisinfo.PNKey)
-	//}
-	//if len(thisinfo.PCKey) > 3 {
-	PCKeyMap.Delete(thisinfo.PCKey)
-	//}
-	//if len(thisinfo.PBKey) > 3 {
-	PBKeyMap.Delete(thisinfo.PBKey)
-	//}
-}

fullproject/src2/thisinfo.go

@@ -1,153 +0,0 @@
-// thisinfo
-package main
-
-import (
-	"crypto/sha1"
-	"encoding/json"
-	"fmt"
-	"io"
-	du "jy/util"
-	"qfw/util"
-	"strings"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-//抽取信息映射实体类
-type Info struct {
-	Id            string                 `json:"_id"`
-	Href          string                 `json:"href"`
-	Publishtime   int64                  `json:"publishtime"`
-	Title         string                 `json:"title"`
-	TopType       string                 `json:"toptype"`
-	SubType       string                 `json:"subtype"`
-	ProjectName   string                 `json:"projectname"`
-	ProjectCode   string                 `json:"projectcode"`
-	Buyer         string                 `json:"buyer"`
-	Buyerperson   string                 `json:"buyerperson"`
-	Buyertel      string                 `json:"buyertel"`
-	Agency        string                 `json:"agency"`
-	Area          string                 `json:"area"`
-	City          string                 `json:"city"`
-	HasPackage    bool                   `json:"haspackage"`
-	Package       map[string]interface{} `json:"package"`
-	PNum          string                 `json:"pnum"`
-	Topscopeclass []string               `json:"topscopeclass"`
-	Subscopeclass []string               `json:"subscopeclass"`
-	Winners       []string
-	dealtype      int
-
-	Buyerclass  string `json:"buyerclass"`
-	Bidopentime int64  `json:"bidopentime"`
-	District    string `json:"district"`
-	Winnerorder []string
-
-	NewPNKey string
-	PNKey    string
-	PCKey    string
-	PBKey    string
-
-	AllRelatePNKeyMap map[string]*Key
-	AllRelatePCKeyMap map[string]*Key
-}
-
-//pcb三选值
-type PCBV struct {
-	Val    int  //1一项有效值,2二项有效值,3三项有效值
-	Pname  bool //有项目名称
-	Pcode  bool //有项目编号
-	Buyer  bool //有采购单位
-	Area   bool //区域
-	City   bool //市
-	Agency bool //代理机构
-
-	PnameLen int //值长度
-	PcodeLen int //值长度
-	BuyerLen int //值长度
-}
-
-func PreThisInfo(tmp map[string]interface{} /*新信息*/) *Info {
-	bys, _ := json.Marshal(tmp)
-	var thisinfo *Info
-	json.Unmarshal(bys, &thisinfo)
-	if thisinfo == nil {
-		return nil
-	}
-	if len(thisinfo.Topscopeclass) == 0 {
-		thisinfo.Topscopeclass = []string{}
-	}
-	if len(thisinfo.Subscopeclass) == 0 {
-		thisinfo.Subscopeclass = []string{}
-	}
-	//去重
-	thisinfo.Subscopeclass = RemoveDup(thisinfo.Subscopeclass)
-
-	if len(thisinfo.Package) > 0 { //信息是否分包
-		thisinfo.HasPackage = true
-	} else if thisinfo.TopType == "结果" && TitleReg.MatchString(thisinfo.Title) {
-		//当信息类型是结果时，并且标题中包含分包字样，找到包号，用以后面比较打分
-		res := TitleReg.FindAllStringSubmatch(thisinfo.Title, -1)
-		pnum := du.PackageNumberConvert(res[0][0])
-		//du.Debug(pnum, res)
-		thisinfo.PNum = pnum
-	}
-	if thisinfo.SubType == "变更" || strings.Index(thisinfo.Title, "变更公告") > -1 || strings.Index(thisinfo.Title, "更正公告") > -1 {
-		//当信息类型是变更或标题中含变更时
-		if thisinfo.TopType == "招标" {
-			//招标的变更公告过，不作处理
-			//du.Debug(thisinfo.Id, thisinfo.Href, thisinfo.ProjectName, thisinfo.ProjectCode)
-			return nil
-		} else if thisinfo.TopType == "结果" {
-			thisinfo.SubType = "变更"
-		}
-	}
-
-	//计算中标人
-	winner, _ := tmp["winner"].(string)
-	m1 := map[string]bool{}
-	winners := []string{}
-	if winner != "" {
-		m1[winner] = true
-		winners = append(winners, winner)
-	}
-	if thisinfo.HasPackage {
-		packageM, _ := tmp["package"].(bson.M)
-		for _, p := range packageM {
-			pm, _ := p.(map[string]interface{})
-			pw, _ := pm["winner"].(string)
-			if pw != "" {
-				m1[pw] = true
-				winners = append(winners, pw)
-			}
-		}
-	}
-	thisinfo.Winners = winners
-	m1 = nil
-	//中标候选人
-	winnerorder := []string{}
-	if winorders, ok := tmp["winnerorder"].([]interface{}); ok {
-		for _, wins := range winorders {
-			if win, ok := wins.(map[string]interface{}); ok {
-				entname := util.ObjToString(win["entname"])
-				if entname != "" && len([]rune(entname)) > 6 {
-					winnerorder = append(winnerorder, entname)
-				}
-			}
-		}
-	}
-	thisinfo.Winnerorder = winnerorder
-
-	new_pn, _ := handleprojectname(thisinfo.ProjectName, thisinfo.Buyer, thisinfo.TopType) //作为判断依据
-	thisinfo.NewPNKey = new_pn
-	thisinfo.PNKey = "pn_" + new_pn
-	thisinfo.PCKey = "pc_" + thisinfo.ProjectCode
-	thisinfo.PBKey = "pb_" + thisinfo.Buyer
-	return thisinfo
-}
-
-//获取hascode
-func GetHas1(key string) string {
-	t := sha1.New()
-	io.WriteString(t, key)
-	return fmt.Sprintf("%x", t.Sum(nil))
-}

fullproject/src2/udptaskmap.go

@@ -1,57 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net"
-	"net/http"
-	"sync"
-	"time"
-)
-
-var udptaskmap = &sync.Map{}
-var tomail string
-var api string
-
-type udpNode struct {
-	data      []byte
-	addr      *net.UDPAddr
-	timestamp int64
-	retry     int
-}
-
-func checkMapJob() {
-	//阿里云内网无法发送邮件
-	jkmail, _ := Sysconfig["jkmail"].(map[string]interface{})
-	if jkmail != nil {
-		tomail, _ = jkmail["to"].(string)
-		api, _ = jkmail["api"].(string)
-	}
-	log.Println("start checkMapJob", tomail, Sysconfig["jkmail"])
-	for {
-		udptaskmap.Range(func(k, v interface{}) bool {
-			now := time.Now().Unix()
-			node, _ := v.(*udpNode)
-			if now-node.timestamp > 120 {
-				node.retry++
-				if node.retry > 5 {
-					log.Println("udp重试失败", k)
-					udptaskmap.Delete(k)
-					res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, tomail, "extract-send-fail", k.(string)))
-					if err == nil {
-						defer res.Body.Close()
-						read, err := ioutil.ReadAll(res.Body)
-						log.Println("邮件发发送：", string(read), err)
-					}
-				} else {
-					log.Println("udp重发", k)
-				}
-			} else if now-node.timestamp > 10 {
-				log.Println("udp任务超时中..", k)
-			}
-			return true
-		})
-		time.Sleep(60 * time.Second)
-	}
-}

fullproject/src_dev1/bak.txt

@@ -1,470 +0,0 @@
-
-	//fmt.Println("-------------------", len(compareABCD))
-	//四个元素一致 [AB][AB][AB][AB]
-	vm = []string{"A", "B"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 2; k++ {
-				for m := 0; m < 2; m++ {
-					key := vm[i] + vm[j] + vm[k] + vm[m]
-					compareAB[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	fmt.Println("-------------------", len(compareAB))
-	//---至少两个一致，其他可能不存在
-	//[AB][ABD][AB][ABD]
-	//[AB][ABD][ABD][AB]
-	//[ABD][AB][ABD][AB]
-	//[ABD][AB][AB][ABD]
-	vm = []string{"A", "B"}
-	vm2 := []string{"A", "B", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 3; j++ {
-			for k := 0; k < 2; k++ {
-				for m := 0; m < 3; m++ {
-					key := vm[i] + vm2[j] + vm[k] + vm2[m]
-					if !compareAB[key] {
-						compareAB2D[key] = true
-						//fmt.Println(key)
-					}
-				}
-			}
-		}
-	}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 3; j++ {
-			for k := 0; k < 3; k++ {
-				for m := 0; m < 2; m++ {
-					key := vm[i] + vm2[j] + vm2[k] + vm[m]
-					if !compareAB[key] {
-						compareAB2D[key] = true
-						//fmt.Println(key)
-					}
-				}
-			}
-		}
-	}
-	for i := 0; i < 3; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 3; k++ {
-				for m := 0; m < 2; m++ {
-					key := vm2[i] + vm[j] + vm2[k] + vm[m]
-					if !compareAB[key] {
-						compareAB2D[key] = true
-						//fmt.Println(key)
-					}
-				}
-			}
-		}
-	}
-	for i := 0; i < 3; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 2; k++ {
-				for m := 0; m < 3; m++ {
-					key := vm2[i] + vm[j] + vm[k] + vm2[m]
-					if !compareAB[key] {
-						compareAB2D[key] = true
-						//fmt.Println(key)
-					}
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB2D))
-	//---至少一个一致，其他可能不存在
-	//[ABD][ABD][ABD][ABD] //已经删除DDDD
-	vm = []string{"A", "B", "D"}
-	for i := 0; i < 3; i++ {
-		for j := 0; j < 3; j++ {
-			for k := 0; k < 3; k++ {
-				for m := 0; m < 3; m++ {
-					key := vm[i] + vm[j] + vm[k] + vm[m]
-					if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] {
-						compareABD[key] = true
-						//fmt.Println(key)
-					}
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareABD))
-
-	//[AB][ABCD][AB][ABCD]
-	//[AB][ABCD][ABCD][AB]
-	//[ABCD][AB][ABCD][AB]
-	//[ABCD][AB][AB][ABCD]
-
-	vm = []string{"A", "B"}
-	vm2 = []string{"A", "B", "C", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 4; j++ {
-			for k := 0; k < 2; k++ {
-				for m := 0; m < 4; m++ {
-					key := vm[i] + vm2[j] + vm[k] + vm2[m]
-					if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-						compareAB2CD[key] = true
-						//fmt.Println(key)
-					}
-				}
-			}
-		}
-	}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 4; j++ {
-			for k := 0; k < 4; k++ {
-				for m := 0; m < 2; m++ {
-					key := vm[i] + vm2[j] + vm2[k] + vm[m]
-					if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-						compareAB2CD[key] = true
-						//fmt.Println(key)
-					}
-				}
-			}
-		}
-	}
-	for i := 0; i < 4; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 4; k++ {
-				for m := 0; m < 2; m++ {
-					key := vm2[i] + vm[j] + vm2[k] + vm[m]
-					if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-						compareAB2CD[key] = true
-						//fmt.Println(key)
-					}
-				}
-			}
-		}
-	}
-	for i := 0; i < 4; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 2; k++ {
-				for m := 0; m < 4; m++ {
-					key := vm2[i] + vm[j] + vm[k] + vm2[m]
-					if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-						compareAB2CD[key] = true
-						//fmt.Println(key)
-					}
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB2CD))
-
-	//[ABCD][ABCD][ABCD][ABCD]  //已经删除[CD][CD][CD][CD]   //这个要重点讨论
-	vm = []string{"A", "B", "C", "D"}
-	for i := 0; i < 4; i++ {
-		for j := 0; j < 4; j++ {
-			for k := 0; k < 4; k++ {
-				for m := 0; m < 4; m++ {
-					key := vm[i] + vm[j] + vm[k] + vm[m]
-					if !compareAB[key] && !compareAB2D[key] && !compareABD[key] && !compareNoPass[key] && !compareAB2CD[key] {
-						compareABCD[key] = true
-						//fmt.Println(key)
-					}
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareABCD))
-	
-	
-	
-	/*
-								if compareNoPass[compareStr] {
-									//[CD][CD][CD]
-									//没有通过
-								} else {
-									compareProject.score = score
-									compareProject.pos = k
-									if compareAB[compareStr] || compareAB2D[compareStr] || compareAB2CD[compareStr] { //项目名称、项目编号相等 四个相等
-										if compareBuyer != "C" { //采购单位相等或不存在
-											if compareBuyer == "D" { //不存在
-												if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-													if info.LenPN > 19 || info.LenPC > 12 || info.LenPTC > 12 {
-														comRes1 = append(comRes1, compareProject)
-														BFind = true
-													} else {
-														comRes2 = append(comRes2, compareProject)
-														BFind = true
-													}
-												} else {
-													if info.LenPN > 19 || info.LenPC > 12 || info.LenPTC > 12 {
-														comRes2 = append(comRes2, compareProject)
-														BFind = true
-													} else if compareCity[1:1] != "C" && compareAgency != "C" && compareTime != "D" {
-														comRes3 = append(comRes3, compareProject)
-														BFind = true
-													}
-												}
-											} else { //相等或包含
-												comRes1 = append(comRes1, compareProject)
-												BFind = true
-											}
-										} else { //采购单位不相等
-											//省市  时间  代理机构
-											if "AA" == compareCity && compareAgency != "D" && compareTime != "D" {
-												if info.LenPN > 19 || info.LenPC > 12 || info.LenPTC > 12 {
-													comRes1 = append(comRes1, compareProject)
-													BFind = true
-												} else {
-													comRes2 = append(comRes2, compareProject)
-													BFind = true
-												}
-											} else {
-												if (compareCity[1:1] != "C" && compareAgency != "C" && compareTime != "D") && (info.LenPN > 19 || info.LenPC > 12 || info.LenPTC > 12) {
-													comRes2 = append(comRes2, compareProject)
-													BFind = true
-												}
-											}
-										}
-										//} else if compareAB2D[compareStr] { //至少有两个[AB]*[AB]*相等
-									} else if compareABD[compareStr] { //至少有一个[ABD]相等
-										if compareBuyer != "C" { //采购单位相等或不存在
-											if compareBuyer == "D" { //不存在
-												if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-													if (info.LenPN > 19) && (info.LenPC > 12 || info.LenPTC > 12) {
-														comRes2 = append(comRes2, compareProject)
-														BFind = true
-													} else if info.LenPN > 19 || info.LenPC > 12 || info.LenPTC > 12 {
-														comRes3 = append(comRes3, compareProject)
-														BFind = true
-													}
-												} else {
-													if (compareCity[1:1] != "C" && compareAgency != "C" && compareTime != "D") && (info.LenPN > 19 || info.LenPC > 12 || info.LenPTC > 12) {
-														comRes3 = append(comRes3, compareProject)
-														BFind = true
-													}
-												}
-											} else { //相等或包含
-												if (info.LenPN > 19) && (info.LenPC > 12 || info.LenPTC > 12) {
-													comRes1 = append(comRes1, compareProject)
-													BFind = true
-												} else {
-													comRes2 = append(comRes2, compareProject)
-													BFind = true
-												}
-											}
-										} else { //采购单位不相等
-											//省市  时间  代理机构
-											if "AA" == compareCity && compareAgency != "D" && compareTime != "D" {
-												if (info.LenPN > 19) && (info.LenPC > 12 || info.LenPTC > 12) {
-													comRes2 = append(comRes2, compareProject)
-													BFind = true
-												} else if info.LenPN > 19 || info.LenPC > 12 || info.LenPTC > 12 {
-													comRes3 = append(comRes3, compareProject)
-													BFind = true
-												}
-											} else {
-												if (compareCity[1:1] != "C" && compareAgency != "C" && compareTime != "D") && (info.LenPN > 19) && (info.LenPC > 12 || info.LenPTC > 12) {
-													comRes3 = append(comRes3, compareProject)
-													BFind = true
-												}
-											}
-										}
-									} else if compareABCD[compareStr] { //有不相等的
-										if compareBuyer != "C" { //采购单位相等或不存在
-											if compareBuyer == "D" { //不存在
-												if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-													if (info.LenPN > 21) && (info.LenPC > 14 || info.LenPTC > 14) {
-														comRes2 = append(comRes2, compareProject)
-														BFind = true
-													} else if info.LenPN > 21 || info.LenPC > 14 || info.LenPTC > 14 {
-														comRes3 = append(comRes3, compareProject)
-														BFind = true
-													}
-												} else {
-													if (compareCity[1:1] != "C" && compareAgency != "C" && compareTime != "D") && (info.LenPN > 21 || info.LenPC > 14 || info.LenPTC > 14) {
-														comRes3 = append(comRes3, compareProject)
-														BFind = true
-													}
-												}
-											} else { //相等或包含
-												if (info.LenPN > 21) && (info.LenPC > 14 || info.LenPTC > 14) {
-													comRes1 = append(comRes1, compareProject)
-													BFind = true
-												} else {
-													comRes2 = append(comRes2, compareProject)
-													BFind = true
-												}
-											}
-										} else { //采购单位不相等
-											//省市  时间  代理机构
-											if "AA" == compareCity && compareAgency != "D" && compareTime != "D" {
-												if (info.LenPN > 24) && (info.LenPC > 16 || info.LenPTC > 16) {
-													comRes2 = append(comRes2, compareProject)
-													BFind = true
-												} else if info.LenPN > 24 || info.LenPC > 16 || info.LenPTC > 16 {
-													comRes3 = append(comRes3, compareProject)
-													BFind = true
-												}
-											} else {
-												if (compareCity[1:1] != "C" && compareAgency != "C" && compareTime != "D") && (info.LenPN > 24) && (info.LenPC > 16 || info.LenPTC > 16) {
-													comRes3 = append(comRes3, compareProject)
-													BFind = true
-												}
-											}
-										}
-									}
-								}
-
-							*/
-
-							//							if !BFind {
-							//								//解锁
-							//								IDArr[k].Lock.Unlock()
-							//	
-							
-							
-							
-							/*
-		if info.PTN != "" {
-			if info.PTN == compareProject.PTN || info.PTN == compareProject.ProjectName {
-				compareStr += "A"
-				score += 4
-				if len([]rune(info.PTN)) > 19 {
-					score += 2
-				}
-			} else if compareProject.PTN != "" {
-				if strings.Contains(info.PTN, compareProject.PTN) || strings.Contains(compareProject.PTN, info.PTN) {
-					compareStr += "B"
-					score += 3
-					if len([]rune(info.PTN)) > 19 {
-						score += 2
-					}
-				} else {
-					compareStr += "C"
-				}
-			} else if compareProject.ProjectName != "" {
-				if strings.Contains(info.PTN, compareProject.ProjectName) || strings.Contains(compareProject.ProjectName, info.PTN) {
-					compareStr += "B"
-					score += 3
-					if len([]rune(info.PTN)) > 19 {
-						score += 2
-					}
-				} else {
-					compareStr += "C"
-				}
-			} else {
-				compareStr += "D"
-			}
-		} else {
-			compareStr += "D"
-		}
-	*/
-	
-	
-	if info.ProjectCode != "" {
-		if info.ProjectCode == compareProject.ProjectCode {
-			compareStr += "A"
-			score += 4
-			if len([]rune(info.ProjectCode)) > 12 {
-				score += 2
-			}
-		} else if compareProject.ProjectCode != "" {
-			if strings.Contains(info.ProjectCode, compareProject.ProjectCode) || strings.Contains(compareProject.ProjectCode, info.ProjectCode) {
-				compareStr += "B"
-				score += 3
-				if len([]rune(info.ProjectCode)) > 12 {
-					score += 2
-				}
-			} else {
-				compareStr += "C"
-			}
-		} else {
-			compareStr += "D"
-		}
-	} else {
-		compareStr += "D"
-	}
-
-	if info.PTC != "" {
-		if info.PTC == "" { //compareProject.PTC || info.PTC == compareProject.ProjectCode {
-			compareStr += "A"
-			score += 4
-			if len([]rune(info.PTC)) > 12 {
-				score += 2
-			}
-		} else if "" != "" {
-			if strings.Contains(info.PTC, "") || strings.Contains("", info.PTC) {
-				compareStr += "B"
-				score += 3
-				if len([]rune(info.PTC)) > 12 {
-					score += 2
-				}
-			} else {
-				compareStr += "C"
-			}
-		} else if compareProject.ProjectCode != "" {
-			if strings.Contains(info.PTC, compareProject.ProjectCode) || strings.Contains(compareProject.ProjectCode, info.PTC) {
-				compareStr += "B"
-				score += 3
-				if len([]rune(info.PTC)) > 12 {
-					score += 2
-				}
-			} else {
-				compareStr += "C"
-			}
-		} else {
-			compareStr += "D"
-		}
-	} else {
-		compareStr += "D"
-	}
-	、
-	
-	
-	
-	if len(comRes1) > 0 {
-					if len(comRes1) > 1 {
-						sort.Slice(comRes1, func(i, j int) bool {
-							return comRes1[i].score > comRes1[j].score
-						})
-					}
-					UpdateProject(tmp, info, comRes1[0], 1, comStr)
-				} else if len(comRes2) > 0 {
-					if len(comRes2) > 1 {
-						sort.Slice(comRes2, func(i, j int) bool {
-							return comRes2[i].score > comRes2[j].score
-						})
-					}
-					UpdateProject(tmp, info, comRes2[0], 2, comStr)
-
-				} else if len(comRes3) > 0 {
-					if len(comRes3) > 1 {
-						sort.Slice(comRes3, func(i, j int) bool {
-							return comRes3[i].score > comRes3[j].score
-						})
-					}
-					UpdateProject(tmp, info, comRes3[0], 3, comStr)
-				} else {
-					//没有找到
-					id := NewProject(tmp, info)
-					t, _ := strconv.ParseInt(id[0:8], 16, 64)
-					AllMapLock.Lock()
-					AllIdsMap[int(t)%idsMapSize][id] = &ID{Id: id, lastTime: info.Publishtime}
-					AllMapLock.Unlock()
-					for _, m := range idsMap {
-						m.Key.Arr = append(m.Key.Arr, id)
-					}
-				}
-				
-				
-				for k, v := range str {
-		if v != "" && repeat[v] == nil {
-			index := len(v) % size
-			m := AllMap[index]
-			m.Lock.Lock()
-			ids := m.Map[v]
-			if ids == nil {
-				ids = &Key{Arr: []string{}}
-				m.Map[v] = ids
-			}
-			//所有键一样的对象加锁
-			ids.Lock.Lock()
-			m.Lock.Unlock()
-			repeat[v] = &IdsMapAndIndex{ids, k}
-		}
-	}

fullproject/src_dev1/clear.go

@@ -1,3 +0,0 @@
-package main
-
-func clearPKey() {}

fullproject/src_dev1/comparePNCB.go

@@ -1,173 +0,0 @@
-package main
-
-import (
-	"regexp"
-	"strings"
-)
-
-func comparePNCB(info *Info, compareProject *ProjectInfo) (compareStr string, score int) {
-	retv := 0
-	if info.ProjectName != "" {
-		pns := []string{}
-		if compareProject.ProjectName != "" {
-			pns = append(pns, compareProject.ProjectName)
-		}
-		if len(compareProject.MPN) > 0 {
-			pns = append(pns, compareProject.MPN...)
-		}
-		ifind := 0
-		for _, v := range pns {
-			if info.ProjectName == v {
-				ifind = 1
-				break
-			} else {
-				//if strings.Contains(info.ProjectName, v) || strings.Contains(v, info.ProjectName) ||
-				retv = CheckContain(info.ProjectName, v, info.Buyer)
-				//if CheckContain(info.ProjectName, v, info.Buyer)
-				if retv == 3 {
-					ifind = 1
-					break
-				} else if retv == 1 {
-					ifind = 2
-				} else if ifind == 0 {
-					ifind = 3
-				}
-			}
-		}
-		switch ifind {
-		case 0:
-			compareStr = "D"
-		case 1:
-			compareStr = "A"
-			score += 4
-			if len([]rune(info.ProjectName)) > 19 {
-				score += 2
-			}
-		case 2:
-			compareStr = "B"
-			score += 2
-			if len([]rune(info.ProjectName)) > 19 {
-				score += 1
-			}
-		case 3:
-			compareStr = "C"
-		}
-	} else {
-		compareStr = "D"
-	}
-	PNStr := compareStr
-	for _, pc := range []string{info.ProjectCode, info.PTC} {
-		if pc != "" {
-			pcs := []string{}
-			if compareProject.ProjectCode != "" {
-				pcs = append(pcs, compareProject.ProjectCode)
-			}
-			if len(compareProject.MPC) > 0 {
-				pcs = append(pcs, compareProject.MPC...)
-			}
-			ifind := 0
-			for _, v := range pcs {
-				if pc == v {
-					ifind = 1
-					break
-				} else {
-					// math.Abs(float64(len([]rune(pc))-len([]rune(v)))) < 6
-					//if !_numreg1.MatchString(pc) && !_zimureg1.MatchString(pc) && !_numreg1.MatchString(v) && !_zimureg1.MatchString(v)
-					if (PNStr == "A" || PNStr == "B" || (!_nzreg.MatchString(pc) && !_nzreg.MatchString(v))) && (strings.Contains(pc, v) || strings.Contains(v, pc)) {
-						t1 := pc
-						t2 := v
-						if len(v) > len(pc) {
-							t1 = v
-							t2 = pc
-						}
-						t3 := strings.Replace(t1, t2, "", -1)
-						t3 = _datereg.ReplaceAllString(t3, "")
-						if t3 == "" {
-							ifind = 1
-							break
-						} else if _hanreg.MatchString(t3) || _numreg1.MatchString(t3) {
-							ifind = 2
-						} else {
-							ifind = 3
-						}
-					} else if ifind == 0 {
-						ifind = 3
-					}
-				}
-			}
-			switch ifind {
-			case 0:
-				compareStr += "D"
-			case 1:
-				compareStr += "A"
-				score += 4
-				if len([]rune(pc)) > 19 {
-					score += 2
-				}
-			case 2:
-				if compareStr == "C" && retv == 2 {
-					compareStr += "C"
-				} else {
-					compareStr += "B"
-					score += 2
-					if len([]rune(pc)) > 19 {
-						score += 1
-					}
-				}
-			case 3:
-				compareStr += "C"
-			}
-
-		} else {
-			compareStr += "D"
-		}
-	}
-	return
-}
-
-var _datereg = regexp.MustCompile("20[0-2][0-9][年-][0-9]{1,2}[月-][0-9]{1,2}[日-]([0-9]{1,2}时[0-9]{0,2})?")
-var _numreg1 = regexp.MustCompile("^[0-9-]{1,8}$")
-var _zimureg1 = regexp.MustCompile("^[a-zA-Z-]{1,7}$")
-var _nzreg = regexp.MustCompile("^[0-9a-zA-Z-]+$")
-var _hanreg = regexp.MustCompile(`^[\p{Han}:：【】\\[\\]（）()－-、]+$`)
-
-var replaceStr = regexp.MustCompile("(工程|采购|项目|[?！、【】（）]|栏标价|中标候选人|招标代理)")
-var pStr = regexp.MustCompile("(勘察|监理|施工|设计|验收|标段|分包|子包|[0-9A-Z]包|[一二三四五六七八九十0-9]批)")
-
-func CheckContain(b1, b2, infoBuyer string) (res int) {
-	b1 = replaceStr.ReplaceAllString(b1, "")
-	b2 = replaceStr.ReplaceAllString(b2, "")
-	b1 = strings.Replace(b1, infoBuyer, "", 1)
-	b2 = strings.Replace(b2, infoBuyer, "", 1)
-
-	if b1 == b2 {
-		res = 3
-		return
-	}
-	bs1 := []rune(b1)
-	bs2 := []rune(b2)
-	tmp := ""
-	for i := 0; i < len(bs1); i++ {
-		for j := 0; j < len(bs2); j++ {
-			if bs1[i] == bs2[j] {
-				tmp += string(bs1[i])
-			} else if tmp != "" {
-				b1 = strings.Replace(b1, tmp, "", -1)
-				b2 = strings.Replace(b2, tmp, "", -1)
-				tmp = ""
-			}
-		}
-	}
-	if tmp != "" {
-		b1 = strings.Replace(b1, tmp, "", -1)
-		b2 = strings.Replace(b2, tmp, "", -1)
-	}
-	if b1 == b2 {
-		res = 3
-	} else if (b1 == "" && !pStr.MatchString(b2)) || (b2 == "" && !pStr.MatchString(b1)) {
-		res = 1
-	} else {
-		res = 2
-	}
-	return
-}

fullproject/src_dev1/config.json

@@ -1,34 +0,0 @@
-{
-    "mongodbServers": "192.168.3.207:27082",
-    "mongodbPoolSize": 10,
-    "mongodbName": "cesuo",
-    "jkmail": {
-        "to": "zhangjinkun@topnet.net.cn",
-        "api": "http://10.171.112.160:19281/_send/_mail"
-    },
-    "thread": 1,
-    "extractColl": "key1_biddingall",
-    "projectColl": "projectset_0720",
-    "lenprojectname": 18,
-    "redisPoolSize": 20,
-    "redisaddrs": "ids=192.168.3.207:1378,keys=192.168.3.207:1378,info=192.168.3.207:1378",
-    "clearedis": {
-        "open": true,
-        "clearcron": "0 10 15 ? * 4",
-        "projectcycle": 180
-    },
-    "megerfields": {
-        "projectlen": 5,
-        "projectcodelen": 8
-    },
-    "taskstock": {
-        "open": true,
-		"startTime":1451610310,
-        "startdate": "2015-11-01",
-        "endate": "2019-06-30"
-    },
-    "udpport": ":1482",
-    "nextNode": [
-
-    ]
-}

fullproject/src_dev1/fulldata.go

@@ -1,215 +0,0 @@
-package main
-
-import (
-	"log"
-	"qfw/util"
-	//	"qfw/util/mongodb"
-
-	//	"qfw/util/redis"
-	"sync"
-	"time"
-
-	//	"gopkg.in/mgo.v2/bson"
-)
-
-var FullCount = 0
-
-func RunFullData() {
-	startTime, END := int64(0), int64(0)
-	sts, bres := MQFW.Find(extractColl, `{}`, "publishtime", `{"publishtime":1}`, true, 1, 1)
-	if bres && sts != nil && len(*sts) == 1 {
-		startTime = util.Int64All((*sts)[0]["publishtime"])
-		startTime -= 1
-		sts, bres = MQFW.Find(extractColl, `{}`, "-publishtime", `{"publishtime":1}`, true, 1, 1)
-		if bres && sts != nil && len(*sts) == 1 {
-			END = util.Int64All((*sts)[0]["publishtime"])
-		}
-		log.Println(startTime, END)
-	} else {
-		return
-	}
-	defer util.Catch()
-	var wg = sync.WaitGroup{}
-	//2012-01-01  到 2015-01-01  1420041600
-
-	findPoolSize := 4
-	pool := make(chan *task, findPoolSize)
-
-	endChan := make(chan bool, 1)
-	_ = time.Now().Unix()
-
-	//	sess := MQFW.GetMgoConn()
-	//	var result []map[string]interface{}
-	//	sess.DB(MQFW.DbName).C(extractColl).Find(map[string]interface{}{}).Sort("publishtime").All(&result)
-	//	log.Println("查询结果：", len(result))
-	//	MQFW.DestoryMongoConn(sess)
-	//	pool <- &task{result}
-	//endChan <- true
-
-	before15year := int64(1420041600) //15年之前3天查询一次
-	day := 0
-	go func() {
-		for {
-			if startTime >= END {
-				log.Println("任务结束")
-				endChan <- true
-				break
-			}
-			addDay := 1
-			if startTime < before15year {
-				addDay = 3
-			}
-			//endTime := int64(1561828196)
-			endTime := startTime + int64(20*86400)
-			day += addDay
-			log.Println("day====", day, startTime, endTime)
-			q := map[string]interface{}{
-				"publishtime": map[string]interface{}{
-					"$gt":  startTime,
-					"$lte": endTime,
-				},
-			}
-			//			q = bson.M{"_id": bson.M{"$in": []interface{}{
-			//				util.StringTOBsonId("5a29933f40d2d9bbe87ba510"),
-			//				util.StringTOBsonId("59cb110740d2d9bbe8a5ea89"),
-			//				util.StringTOBsonId("59dec3c640d2d9bbe8fc2067"),
-			//			}}}
-
-			//			q = bson.M{"_id": bson.M{"$in": []interface{}{
-			//				util.StringTOBsonId("58ea834ee1382322d055aba2"),
-			//				util.StringTOBsonId("5762767261a0721f1504317e"),
-			//				util.StringTOBsonId("5909acaee138233f2da53ebc"),
-			//				util.StringTOBsonId("57764ddaedbcdc49e6003b62"),
-			//				util.StringTOBsonId("590a9b2ee138233f2da964b7"),
-			//				util.StringTOBsonId("58dab09ae138233607531939"),
-			//				util.StringTOBsonId("57909adcedbcdc35c8005ab5"),
-			//				util.StringTOBsonId("57861257edbcdc1cea01478c"),
-			//				util.StringTOBsonId("57e0ac3861a0721f15324175"),
-			//				util.StringTOBsonId("58da95b8e138233607524f76"),
-			//				util.StringTOBsonId("590a88b3e138233f2da9111b"),
-			//			}}}
-			//			q = bson.M{"_id": bson.M{"$in": []interface{}{
-			//				util.StringTOBsonId("59cf6c7940d2d9bbe8c62d42"),
-			//				util.StringTOBsonId("59dedd1b40d2d9bbe8fe1382"),
-			//				util.StringTOBsonId("59dedcc140d2d9bbe8fe0f7f"),
-			//				util.StringTOBsonId("59dedd1b40d2d9bbe8fe1386"),
-			//				util.StringTOBsonId("59dedd1b40d2d9bbe8fe138d"),
-			//				util.StringTOBsonId("59dedcc140d2d9bbe8fe0f63"),
-			//				util.StringTOBsonId("59e9584340d2d9bbe84bfefd"),
-			//				util.StringTOBsonId("59e9584340d2d9bbe84bff01"),
-			//				util.StringTOBsonId("59e9584340d2d9bbe84bff08"),
-			//				util.StringTOBsonId("59e9584340d2d9bbe84bff0c"),
-			//				util.StringTOBsonId("59e9676340d2d9bbe84d4ba1"),
-			//				util.StringTOBsonId("59e9795540d2d9bbe84eacf2"),
-			//				util.StringTOBsonId("59e979af40d2d9bbe84eb0f0"),
-			//				util.StringTOBsonId("59e978fb40d2d9bbe84ea8b6"),
-			//				util.StringTOBsonId("59e9ad6740d2d9bbe851fee7"),
-			//				util.StringTOBsonId("59e9ad6740d2d9bbe851fef6"),
-			//				util.StringTOBsonId("59e9ae7640d2d9bbe8521746"),
-			//				util.StringTOBsonId("59e9aed040d2d9bbe8521fbb"),
-			//				util.StringTOBsonId("59e9aed040d2d9bbe8521feb"),
-			//				util.StringTOBsonId("59e9b88f40d2d9bbe852f222"),
-			//				util.StringTOBsonId("59efde8640d2d9bbe87677e0"),
-			//				util.StringTOBsonId("59efde8640d2d9bbe87677dc"),
-			//				util.StringTOBsonId("59f0107b40d2d9bbe87a8935"),
-			//				util.StringTOBsonId("5a026a1e40d2d9bbe8ffbbbc"),
-			//				util.StringTOBsonId("5a0269c440d2d9bbe8ffb586"),
-			//				util.StringTOBsonId("5a02840c40d2d9bbe8019c75"),
-			//				util.StringTOBsonId("5a02840c40d2d9bbe8019c80"),
-			//				util.StringTOBsonId("5a02be7640d2d9bbe8057516"),
-			//			}}}
-
-			//			startTime = 1561828197
-			//数据正序处理
-			sess := MQFW.GetMgoConn()
-			if sess == nil {
-				time.Sleep(10 * time.Second)
-				continue
-			}
-			var result []map[string]interface{}
-			sess.DB(MQFW.DbName).C(extractColl).Find(q).Sort("publishtime").All(&result)
-			startTime = endTime
-			log.Println("查询结果：", len(result))
-			if len(result) == 0 {
-				continue
-			}
-			MQFW.DestoryMongoConn(sess)
-			pool <- &task{result}
-			wg.Add(1)
-			startTime = endTime
-			if day > 0 && day%(1*findPoolSize) == 0 {
-				wg.Wait()
-				//				MQFW.Destory()
-				//				MQFW = mongodb.MongodbSim{
-				//					MongodbAddr: Sysconfig["mongodbServers"].(string),
-				//					Size:        2 * findPoolSize,
-				//					DbName:      Sysconfig["mongodbName"].(string),
-				//				}
-				//				MQFW.InitPool()
-			}
-		}
-	}()
-
-	for {
-		select {
-		case t := <-pool:
-			t.query()
-			t.result = nil
-			t = nil
-			wg.Done()
-		case <-endChan:
-			return
-		}
-	}
-
-}
-
-type task struct {
-	result []map[string]interface{}
-}
-
-func (t *task) query() {
-	index := 0
-	wg := &sync.WaitGroup{}
-
-	for _, tmp := range t.result {
-		if index%2000 == 0 {
-			log.Println(index, tmp["_id"])
-		}
-		index++
-		if util.IntAll(tmp["repeat"]) == 1 {
-			continue
-		}
-		pt := util.Int64All(tmp["publishtime"])
-		if pt > currentMegerTime {
-			currentMegerTime = pt
-		}
-		currentMegerCount++
-		if currentMegerCount > 600000 {
-			log.Println("执行清理", currentMegerTime)
-			clearPKey()
-			currentMegerCount = 0
-		}
-		wg.Add(1)
-		MultiThread <- true
-		go func(tmp map[string]interface{}) {
-			defer func() {
-				<-MultiThread
-				wg.Done()
-			}()
-			info := PreThisInfo(tmp)
-			if info != nil && (info.LenPC > 3 || info.LenPN > 3 || info.LenPTC > 3) {
-				startProjectMerge(info, tmp)
-				//thisid := util.BsonIdToSId(tmp["_id"])
-				//redis.Put(INFOID, thisid, 1, INFOTIMEOUT)
-				currentMegerTime = info.Publishtime
-			}
-		}(tmp)
-		//time.Sleep(10 * time.Microsecond)
-	}
-	wg.Wait()
-	FullCount += index
-	t.result = nil
-	log.Println("currentFull", FullCount)
-
-}

fullproject/src_dev1/infotool.go

@@ -1,436 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	//	"fmt"
-	du "jy/util"
-	qu "qfw/util"
-	"regexp"
-	"strings"
-	"sync"
-	"time"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-//抽取信息映射实体类
-type Info struct {
-	Id            string                 `json:"_id"`
-	Href          string                 `json:"href"`
-	Publishtime   int64                  `json:"publishtime"`
-	Title         string                 `json:"title"`
-	TopType       string                 `json:"toptype"`
-	SubType       string                 `json:"subtype"`
-	ProjectName   string                 `json:"projectname"`
-	ProjectCode   string                 `json:"projectcode"`
-	Buyer         string                 `json:"buyer"`
-	Buyerperson   string                 `json:"buyerperson"`
-	Buyertel      string                 `json:"buyertel"`
-	Agency        string                 `json:"agency"`
-	Area          string                 `json:"area"`
-	City          string                 `json:"city"`
-	HasPackage    bool                   `json:"haspackage"`
-	Package       map[string]interface{} `json:"package"`
-	PNum          string                 `json:"pnum"`
-	Topscopeclass []string               `json:"topscopeclass"`
-	Subscopeclass []string               `json:"subscopeclass"`
-	Winners       []string
-	dealtype      int
-
-	Buyerclass  string `json:"buyerclass"`
-	Bidopentime int64  `json:"bidopentime"`
-	District    string `json:"district"`
-	Winnerorder []string
-
-	PTC    string
-	pnbval int
-	LenPC  int
-	LenPN  int
-	LenPTC int
-}
-
-var (
-	PNKeyMap, PCKeyMap, PBKeyMap = sync.Map{}, sync.Map{}, sync.Map{}
-	pnreg                        = regexp.MustCompile("^(及编号[：:])|(项目|采购|招标|中标|成交|结果|[_]|公告)$")
-	titleGetPn                   = regexp.MustCompile("^([\\[【(（]?.?(资格预审|中标|招标|延期|成交|结果|合同|失败|询价|关于对?)(公告)?[\\]】)）]?([:：]|关于对?)?)?(.{4,70}?(采购|工程)?(项目)?)([(（【]?(第?[一二三四五六七八九1-9再]次|重新|重招|公开|[预拟]).{0,3}?[)）】]?)?(招标|采购|采购计划|发包|结果|变更|更正|成交|网上(竞价)?|电子化|电子反拍|询比?价|比价|竞争性(谈判|磋商)|流标|废标|邀请|合同|验收|违规|资格|预审|中标(结果)?|延期|澄清|暂停|补遗|终止|文件|标前|征求|报建|征集|论证|谈判|拟实施|中止|需求|比选|评标(过程)?及?|磋商|未入围|进口|投标|答疑|抽签|异常|质疑|答复|回复|应答|遴选|最高|拦标|推迟|开标|取消|延迟|撤销|控制价|场外|作废|候选人|采用|实施|预|不良记录|竞买|反拍|修正|调整|简称|小型)?(公告|记录|公示|预告|通知[函书]?|意见[函书]?|[函书])?([(（【].*?[)）】])?$")
-	titleGetPc                   = regexp.MustCompile("^([-0-9a-zA-Z第号采招政询电审竞#]{8,}[-0-9a-zA-Z#]+)")
-	titleGetPc1                  = regexp.MustCompile("[\\[【(（](.{0,6}(编号|编码|项号|包号|代码|标段?号)[:：为])?([-0-9a-zA-Z第号采招政询电审竞#]{5,}([\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+[\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+)?)[\\]】)）]")
-	titleGetPc2                  = regexp.MustCompile("([-0-9a-zA-Z第号采政招询电审竞#]{8,}[-0-9a-zA-Z#]+)(.{0,5}公告)?$")
-	numreg                       = regexp.MustCompile("^[0-9]$")
-	numreg2                      = regexp.MustCompile("^[0-9]+$")
-	numCheckPc                   = regexp.MustCompile("^[0-9-]{1,10}$")
-	TitleReg                     = regexp.MustCompile("([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ、\\-~至]+(子|合同|分|施工|监理|标)?[包标段][号段]?[、]?)+|((子|合同|分|施工|监理|标)?[包标段][号段]?[一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ、\\-~至]+[、]?)+|(子|合同|分|施工|监理|标)?[包标段][号段]?[a-zA-Z0-9]+[\\-~－至、](子|合同|分|施工|监理|标)?[包标段][号段]?[a-zA-Z0-9]+")
-)
-
-type ProjectInfo struct {
-	Id            string                 `json:"id"`
-	Publistime    []int64                `json:"publistime"` //多条信息的发布时间、跨度
-	InfoType      [][]string             `json:"infotype"`   //多条信息内的 toptype、subtype
-	Ids           []string               `json:"ids"`
-	Topscopeclass []string               `json:"topscopeclass"`
-	Subscopeclass []string               `json:"subscopeclass"`
-	Winners       []string               `json:"winners"`
-	ProjectName   string                 `json:"projectname"`
-	ProjectCode   string                 `json:"projectcode"` //项目代码唯一（纯数字的权重低）
-	Buyer         string                 `json:"buyer"`       //采购单位唯一
-	MPN           []string               `json:"mpn"`         //合并后多余的项目名称
-	MPC           []string               `json:"mpc"`         //合并后多余的项目编号
-	Buyerperson   string                 `json:"buyerperson"`
-	Buyertel      string                 `json:"buyertel"`
-	Agency        string                 `json:"agency"`     //代理机构唯一
-	Area          string                 `json:"area"`       //地区唯一
-	City          string                 `json:"city"`       //地市
-	District      string                 `json:"district"`   //区县
-	HasPackage    bool                   `json:"haspackage"` //是否有分包
-	Package       map[string]interface{} `json:"package"`    //分包的对比对象
-
-	Buyerclass  string   `json:"buyerclass"`  //采购单位分类
-	Bidopentime int64    `json:"bidopentime"` //开标时间
-	Winnerorder []string //中标候选人
-	score       int
-	comStr      string
-}
-
-type KeyMap struct {
-	Lock sync.Mutex
-	Map  map[string]*Key
-}
-
-type Key struct {
-	Arr  []string
-	Lock sync.Mutex
-}
-type IdAndLock struct {
-	Id   string
-	Lock sync.Mutex
-}
-
-func NewKeyMap() *KeyMap {
-	return &KeyMap{
-		Map:  map[string]*Key{},
-		Lock: sync.Mutex{},
-	}
-}
-
-var size, idsMapSize = 30, 100
-var AllPNMap = make([]*KeyMap, size)  //存储 项目名称,值为id数组
-var AllPCMap = make([]*KeyMap, size)  //存储 项目编号,值为id数组
-var AllPTCMap = make([]*KeyMap, size) //存储 项目编号,值为id数组
-var AllPBMap = make([]*KeyMap, size)  //存储 采购单位,值为id数组
-
-type ID struct {
-	Id       string
-	Lock     sync.Mutex
-	lastTime int64
-	pos      int
-}
-
-//所有项目id对象，加锁，删除等用
-var AllIdsMap = make([]map[string]*ID, idsMapSize)
-var AllIdsMap2 = map[string]*ID{}
-var AllIdsMapLock = sync.Mutex{}
-
-//预定义字符串 [ABCD][ABCD][ABCD]  项目名称/编号/标题编号
-var compareNoPass = map[string]bool{}
-var compareAB = map[string]bool{}
-var compareAB2D = map[string]bool{}
-var compareABD = map[string]bool{}
-var compareAB2CD = map[string]bool{}
-var compareABCD = map[string]bool{}
-
-func init() {
-	for i := 0; i < size; i++ {
-		AllPNMap[i] = NewKeyMap()
-		AllPCMap[i] = NewKeyMap()
-		AllPTCMap[i] = NewKeyMap()
-		AllPBMap[i] = NewKeyMap()
-	}
-
-	for i := 0; i < idsMapSize; i++ {
-		AllIdsMap[i] = map[string]*ID{}
-	}
-
-	//---不能通过
-	vm := []string{"C", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				compareNoPass[key] = true
-				//fmt.Println(key)
-			}
-		}
-	}
-	//fmt.Println("-------------------")
-
-	//三个元素一致 [AB][AB][AB]，分值最高
-	vm = []string{"A", "B"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				compareAB[key] = true
-				//fmt.Println(key)
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB))
-	//---至少两个一致，其他可能不存在
-	//[AB][AB][ABD]
-	//[AB][ABD][AB]
-	vm = []string{"A", "B"}
-	vm2 := []string{"A", "B", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 3; k++ {
-				key := vm[i] + vm[j] + vm2[k]
-				if !compareAB[key] {
-					compareAB2D[key] = true
-					//fmt.Println(key)
-
-				}
-			}
-		}
-	}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 3; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm2[j] + vm[k]
-				if !compareAB[key] {
-					compareAB2D[key] = true
-					//fmt.Println(key)
-
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB2D))
-	//---至少一个一致，其他可能不存在
-	//[ABD][ABD][ABD] //已经删除DDD
-	vm = []string{"A", "B", "D"}
-	for i := 0; i < 3; i++ {
-		for j := 0; j < 3; j++ {
-			for k := 0; k < 3; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] {
-					compareABD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareABD))
-
-	//[AB][ABCD][AB]
-	//[AB][AB][ABCD]
-	vm = []string{"A", "B"}
-	vm2 = []string{"A", "B", "C", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 4; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm2[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-					compareAB2CD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 4; k++ {
-				key := vm[i] + vm[j] + vm2[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-					compareAB2CD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB2CD))
-	//[ABECD][ABECD][ABECD]  //已经删除[CD][CD][CD]   //这个要重点讨论
-	vm = []string{"A", "B", "C", "D"}
-	for i := 0; i < 4; i++ {
-		for j := 0; j < 4; j++ {
-			for k := 0; k < 4; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareABD[key] && !compareNoPass[key] && !compareAB2CD[key] {
-					compareABCD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareABCD))
-
-}
-
-//预处理信息
-func PreThisInfo(tmp map[string]interface{}) *Info {
-	bys, _ := json.Marshal(tmp)
-	var thisinfo *Info
-	json.Unmarshal(bys, &thisinfo)
-	if thisinfo == nil {
-		return nil
-	}
-	if len(thisinfo.Topscopeclass) == 0 {
-		thisinfo.Topscopeclass = []string{}
-	}
-	if len(thisinfo.Subscopeclass) == 0 {
-		thisinfo.Subscopeclass = []string{}
-	}
-	//去重
-	thisinfo.Subscopeclass = RemoveDup(thisinfo.Subscopeclass)
-
-	if len(thisinfo.Package) > 0 { //信息是否分包
-		thisinfo.HasPackage = true
-	} else if thisinfo.TopType == "结果" && TitleReg.MatchString(thisinfo.Title) {
-		//当信息类型是结果时，并且标题中包含分包字样，找到包号，用以后面比较打分
-		res := TitleReg.FindAllStringSubmatch(thisinfo.Title, -1)
-		pnum := du.PackageNumberConvert(res[0][0])
-		//du.Debug(pnum, res)
-		thisinfo.PNum = pnum
-	}
-	//	if checkInfoAlter(tmp) {
-	//		thisinfo.SubType = "变更"
-	//	}
-	//计算中标人
-	winner, _ := tmp["winner"].(string)
-	m1 := map[string]bool{}
-	winners := []string{}
-	if winner != "" {
-		m1[winner] = true
-		winners = append(winners, winner)
-	}
-	if thisinfo.HasPackage {
-		packageM, _ := tmp["package"].(bson.M)
-		for _, p := range packageM {
-			pm, _ := p.(map[string]interface{})
-			pw, _ := pm["winner"].(string)
-			if pw != "" {
-				m1[pw] = true
-				winners = append(winners, pw)
-			}
-		}
-	}
-	thisinfo.Winners = winners
-	m1 = nil
-	//中标候选人
-	winnerorder := []string{}
-	if winorders, ok := tmp["winnerorder"].([]interface{}); ok {
-		for _, wins := range winorders {
-			if win, ok := wins.(map[string]interface{}); ok {
-				entname := qu.ObjToString(win["entname"])
-				if entname != "" && len([]rune(entname)) > 6 {
-					winnerorder = append(winnerorder, entname)
-				}
-			}
-		}
-	}
-	thisinfo.Winnerorder = winnerorder
-	res := titleGetPc.FindStringSubmatch(thisinfo.Title)
-	if len(res) > 1 && len(res[1]) > 8 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
-		thisinfo.PTC = res[1]
-		thisinfo.pnbval++
-	} else {
-		res = titleGetPc1.FindStringSubmatch(thisinfo.Title)
-		if len(res) > 3 && len(res[3]) > 8 && thisinfo.ProjectCode != res[3] && !numCheckPc.MatchString(res[3]) && !_zimureg1.MatchString(res[3]) {
-			thisinfo.PTC = res[3]
-			thisinfo.pnbval++
-		} else {
-			res = titleGetPc2.FindStringSubmatch(thisinfo.Title)
-			if len(res) > 1 && len(res[1]) > 8 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
-				thisinfo.PTC = res[1]
-				thisinfo.pnbval++
-			}
-		}
-	}
-
-	if thisinfo.ProjectName != "" {
-		thisinfo.pnbval++
-	}
-
-	if thisinfo.ProjectCode != "" && !_zimureg1.MatchString(thisinfo.ProjectCode) {
-		thisinfo.pnbval++
-	} else {
-		thisinfo.ProjectCode = ""
-	}
-
-	if thisinfo.Buyer != "" {
-		thisinfo.pnbval++
-	}
-
-	thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
-	thisinfo.LenPTC = len([]rune(thisinfo.PTC))
-	thisinfo.LenPN = len([]rune(thisinfo.ProjectName))
-	return thisinfo
-}
-
-//移除数组中重复的元素
-func RemoveDup(arr []string) (newarr []string) {
-	m1 := map[string]bool{}
-	newarr = []string{}
-	for _, k := range arr {
-		if !m1[k] {
-			m1[k] = true
-			newarr = append(newarr, k)
-		}
-	}
-	return
-}
-
-//阻塞同名的pb、pc、pv并发
-func lockPNCBMap(thisinfo *Info) {
-	for { //等待其他任务完成
-		ok := true
-		if thisinfo.LenPN > 0 {
-			if _, b := PNKeyMap.Load(thisinfo.ProjectName); b {
-				ok = false
-			}
-		}
-		if thisinfo.LenPC > 0 {
-			if _, b := PCKeyMap.Load(thisinfo.ProjectCode); b {
-				ok = false
-			}
-		}
-		if thisinfo.LenPTC > 0 {
-			if _, b := PCKeyMap.Load(thisinfo.PTC); b {
-				ok = false
-			}
-		}
-		if len(thisinfo.Buyer) > 0 {
-			if _, b := PBKeyMap.Load(thisinfo.Buyer); b {
-				ok = false
-			}
-		}
-		if ok {
-			break
-		} else {
-			time.Sleep(30 * time.Millisecond)
-		}
-	}
-}
-
-//放行
-func unlockPNCBMap(thisinfo *Info) {
-	PNKeyMap.Delete(thisinfo.ProjectName)
-	PCKeyMap.Delete(thisinfo.ProjectCode)
-	PCKeyMap.Delete(thisinfo.PTC)
-	PBKeyMap.Delete(thisinfo.Buyer)
-}
-
-//判断信息是否是变更
-func checkInfoAlter(tmp map[string]interface{} /*新信息*/) bool {
-	toptype := qu.ObjToString(tmp["toptype"])
-	subtype := qu.ObjToString(tmp["subtype"])
-	title := qu.ObjToString(tmp["title"])
-	if subtype == "变更" || strings.Index(title, "变更公告") > -1 || strings.Index(title, "更正公告") > -1 {
-		//当信息类型是变更或标题中含变更时
-		if toptype == "招标" {
-			//招标的变更公告，不作处理
-		} else if toptype == "结果" {
-			subtype = "变更"
-		}
-	}
-	return subtype == "变更"
-}

fullproject/src_dev1/main.go

@@ -1,74 +0,0 @@
-package main
-
-import (
-	"qfw/util"
-	"qfw/util/mongodb"
-	"qfw/util/redis"
-	"sync"
-	"time"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-const (
-	REDISIDS    = "ids"
-	REDISKEYS   = "keys"
-	INFOID      = "info"
-	INFOTIMEOUT = 86400 * 30
-)
-
-var (
-	Sysconfig                map[string]interface{}
-	MQFW                     mongodb.MongodbSim
-	extractColl, projectColl string
-	lenprojectname           int
-	MultiThread              chan bool
-	IdLock                   = &sync.Mutex{}
-	PncbMayLock              = &sync.Mutex{}
-
-	//三组lock,对应的（PNKey）key为项目名称,值对应的是此项目名称对应的项目id数组
-	PNKey, PCKey, PBKey = NewKeyMap(), NewKeyMap(), NewKeyMap()
-
-	currentMegerTime  int64 //合并项目的时间位置，用来清理几个月之前的项目
-	currentMegerCount int   //合并项目的计数，用来定时清理
-)
-
-func init() {
-
-	util.ReadConfig(&Sysconfig)
-	MultiThread = make(chan bool, util.IntAllDef(Sysconfig["thread"], 200))
-	lenprojectname = util.IntAllDef(Sysconfig["lenprojectname"], 20) - 1
-
-	redis.InitRedisBySize(Sysconfig["redisaddrs"].(string), util.IntAllDef(Sysconfig["redisPoolSize"], 100), 30, 300)
-	MQFW = mongodb.MongodbSim{
-		MongodbAddr: Sysconfig["mongodbServers"].(string),
-		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
-		DbName:      Sysconfig["mongodbName"].(string),
-	}
-	MQFW.InitPool()
-	extractColl = Sysconfig["extractColl"].(string)
-	projectColl = Sysconfig["projectColl"].(string)
-}
-
-func main() {
-	RunFullData()
-	time.Sleep(99999 * time.Hour)
-}
-
-func NewPushInfo(tmp map[string]interface{}) bson.M {
-	return bson.M{
-		"comeintime":  tmp["comeintime"],
-		"publishtime": tmp["publishtime"],
-		"title":       tmp["title"],
-		"toptype":     tmp["toptype"],
-		"subtype":     tmp["subtype"],
-		"infoformat":  tmp["infoformat"],
-		"infoid":      util.BsonIdToSId(tmp["_id"]),
-		"area":        tmp["area"],
-		"city":        tmp["city"],
-		"projectname": tmp["projectname"],
-		"projectcode": tmp["projectcode"],
-		"buyer":       tmp["buyer"],
-		"href":        tmp["href"],
-	}
-}

fullproject/src_dev1/merge.go

@@ -1,719 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"log"
-	"math"
-	qu "qfw/util"
-	"qfw/util/redis"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-func startProjectMerge(info *Info, tmp map[string]interface{}) {
-	//只有或没有采购单位的无法合并
-	if info.pnbval < 2 && info.Buyer != "" {
-		//extInfoTag("invalid", qu.BsonIdToSId(tmp["_id"]))
-		log.Println("invalid", info.pnbval, info.Buyer, info.Id)
-	} else {
-		bpn, bpc, bptc, bpb, pids := getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer)
-		repeatId := map[string]bool{}
-		idArr := []string{} //项目id
-		IDArr := []*ID{}    //项目信息
-		//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock
-		for _, m := range pids {
-			defer m.Lock.Unlock()
-			for _, id := range m.Arr {
-				if !repeatId[id] {
-					repeatId[id] = true
-					_, _ = strconv.ParseInt(id[0:8], 16, 64)
-					AllIdsMapLock.Lock()
-					//Id := AllIdsMap[int(t)%idsMapSize][id]
-					Id := AllIdsMap2[id]
-					AllIdsMapLock.Unlock()
-					Id.Lock.Lock()
-					defer Id.Lock.Unlock()
-					idArr = append(idArr, id)
-					IDArr = append(IDArr, Id)
-				}
-			}
-		}
-
-		bFindProject := false
-		findPid := ""
-		//获取完id，进行计算
-		if len(idArr) > 0 && len(idArr) == len(IDArr) {
-			res := redis.Mget(REDISIDS, idArr)
-			//定义两组
-			comRes1 := []*ProjectInfo{}
-			comRes2 := []*ProjectInfo{}
-			comRes3 := []*ProjectInfo{}
-			if len(res) == len(idArr) {
-				for _, v := range res {
-					var cp ProjectInfo
-					comStr := ""
-					var compareProject *ProjectInfo
-					err := json.Unmarshal(v.([]byte), &cp)
-					compareProject = &cp
-					diffTime := math.Abs(float64(info.Publishtime - compareProject.Publistime[len(compareProject.Publistime)-1]))
-					if err != nil {
-						log.Println("从redis取id信息出错！！！", err)
-					} else if diffTime < 360*86400 {
-						//"A 相等 	B 被包含 	C 不相等	 	D不存在  E被包含
-						compareStr, score := comparePNCB(info, compareProject)
-						//if info.ProjectCode == "ZWJLZXQ-2016-0508" {
-						//}
-						resVal := Select(compareStr, info, compareProject)
-						//---------------------------------------
-						if resVal > 0 {
-							compareBuyer := ""
-							if info.Buyer != "" {
-								if info.Buyer == compareProject.Buyer {
-									compareBuyer += "A"
-								} else if compareProject.Buyer != "" {
-									if strings.Contains(info.Buyer, compareProject.Buyer) || strings.Contains(compareProject.Buyer, info.Buyer) {
-										compareBuyer += "B"
-									} else {
-										compareBuyer += "C"
-									}
-								} else {
-									compareBuyer += "D"
-								}
-							} else {
-								compareBuyer += "D"
-							}
-
-							//---------------------------------------
-							compareCity := ""
-							if info.Area != "全国" && info.Area != "" && info.Area == compareProject.Area {
-								compareCity += "A"
-								score += 2
-							} else if info.Area == "全国" || compareProject.Area == "全国" {
-								compareCity += "B"
-								score += 1
-							} else {
-								compareCity += "C"
-							}
-
-							if info.City != "" && info.City == compareProject.City {
-								compareCity += "A"
-								score += 2
-							} else {
-								if info.Area == "全国" || compareProject.Area == "全国" {
-									compareCity += "B"
-								} else {
-									compareCity += "C"
-								}
-							}
-							compareTime := ""
-							//diffTime := math.Abs(float64(info.Publishtime - compareProject.Publistime[len(compareProject.Publistime)-1]))
-							if diffTime < 45*86400 {
-								compareTime += "A"
-								score += 2
-							} else if diffTime < 90*86400 {
-								compareTime += "B"
-								score += 1
-							} else {
-								compareTime += "D"
-							}
-							compareAgency := ""
-							if info.Agency != "" {
-								if info.Agency == compareProject.Agency {
-									compareAgency += "A"
-									score += 2
-								} else if compareProject.Agency != "" {
-									if strings.Contains(info.Agency, compareProject.Agency) || strings.Contains(compareProject.Agency, info.Agency) {
-										compareAgency += "B"
-										score += 1
-									} else {
-										compareAgency += "C"
-									}
-								} else {
-									compareAgency += "D"
-								}
-							} else {
-								compareAgency += "D"
-							}
-
-							//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构
-
-							comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency
-							compareProject.comStr = comStr
-							//							log.Println(resVal, info.ProjectName, info.ProjectCode, info.Buyer, compareProject.Id, compareProject.ProjectName, compareProject.ProjectCode, compareProject.Buyer, compareProject.MPC, compareStr)
-							//							log.Println(compareBuyer, compareCity, compareAgency, compareTime)
-							switch resVal {
-							case 3:
-								if compareBuyer == "A" || compareBuyer == "B" {
-									comRes1 = append(comRes1, compareProject)
-								} else if compareBuyer == "D" {
-									if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-										comRes1 = append(comRes1, compareProject)
-
-									} else {
-										comRes2 = append(comRes2, compareProject)
-
-									}
-								} else {
-									if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-										comRes2 = append(comRes2, compareProject)
-
-									} else if compareCity[1:1] != "C" && compareAgency != "C" && compareTime != "D" {
-										comRes3 = append(comRes3, compareProject)
-									}
-								}
-							case 1, 2:
-
-								if compareBuyer == "A" {
-									comRes1 = append(comRes1, compareProject)
-								} else if compareBuyer == "B" {
-									comRes2 = append(comRes2, compareProject)
-
-								} else if compareBuyer == "D" {
-									if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-										comRes2 = append(comRes2, compareProject)
-									} else {
-										comRes3 = append(comRes3, compareProject)
-									}
-								} else {
-									if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-										comRes3 = append(comRes3, compareProject)
-									}
-								}
-							}
-						}
-					}
-				}
-				//--------------------------------对比完成-----------------------
-				for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} {
-					if len(resN) > 0 {
-						if len(resN) > 1 {
-							sort.Slice(comRes1, func(i, j int) bool {
-								return comRes1[i].score > comRes1[j].score
-							})
-						}
-
-						bFindProject = true
-						findPid = resN[0].Id
-						for k2, bv := range []int{bpn, bpc, bptc, bpb} {
-							if bv > -1 {
-								pids[bv].Arr = append(pids[bv].Arr, findPid)
-								if k2 == 0 {
-									if resN[0].ProjectName == "" {
-										resN[0].ProjectName = info.ProjectName
-									} else {
-										if resN[0].MPN == nil {
-											resN[0].MPN = []string{info.ProjectName}
-										} else {
-											resN[0].MPN = append(resN[0].MPN, info.ProjectName)
-										}
-									}
-
-								} else if k2 < 3 {
-									if resN[0].ProjectCode == "" {
-										resN[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
-									} else {
-										if resN[0].MPC == nil {
-											resN[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
-										} else {
-											resN[0].MPC = append(resN[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
-										}
-									}
-
-								} else {
-									if resN[0].Buyer == "" {
-										resN[0].Buyer = info.Buyer
-									}
-								}
-							}
-						}
-						UpdateProject(tmp, info, resN[0], kv+1, resN[0].comStr)
-						//更新AllIdsMao中的时间
-						redis.Put(REDISIDS, findPid, resN[0], 0)
-						break
-					}
-				}
-			} else {
-				log.Println("redis记录缺失！！！")
-			}
-		}
-
-		if !bFindProject {
-			//没有找到
-			id := NewProject(tmp, info)
-			_, _ = strconv.ParseInt(id[0:8], 16, 64)
-			AllIdsMapLock.Lock()
-			//AllIdsMap[int(t)%idsMapSize][id] = &ID{Id: id, lastTime: info.Publishtime}
-			AllIdsMap2[id] = &ID{Id: id, lastTime: info.Publishtime}
-			AllIdsMapLock.Unlock()
-			for _, m := range pids {
-				m.Arr = append(m.Arr, id)
-			}
-		}
-	}
-}
-
-func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string) {
-	//log.Println(thisinfo.ProjectName, pInfo.ProjectName, comStr)
-	//	updateid := pInfo.Id
-	if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
-		return //updateid
-	}
-	set := map[string]interface{}{}
-	res, bres := MQFW.FindById(projectColl, pInfo.Id, `{"list":0}`)
-	EqInfoUpdate(thisinfo, pInfo)
-	if bres && res != nil && *res != nil {
-		set["topscopeclass"] = pInfo.Topscopeclass
-		set["subscopeclass"] = pInfo.Subscopeclass
-		s_subscopeclass := strings.Join(pInfo.Subscopeclass, ",")
-		if len(s_subscopeclass) > 0 {
-			s_subscopeclass = "," + s_subscopeclass + ","
-		}
-		set["s_subscopeclass"] = s_subscopeclass
-		s_winner := strings.Join(pInfo.Winners, ",")
-		if len(s_winner) > 0 {
-			s_winner = "," + s_winner + ","
-		}
-		set["s_winner"] = s_winner
-		if pInfo.Buyerperson != "" && pInfo.Buyertel != "" {
-			set["buyerperson"] = pInfo.Buyerperson
-			set["buyertel"] = pInfo.Buyertel
-		}
-		if pInfo.Buyerclass != "" {
-			set["buyerclass"] = pInfo.Buyerclass
-		}
-		if pInfo.District != "" {
-			set["district"] = pInfo.District
-		}
-		if pInfo.Bidopentime > 0 {
-			set["bidopentime"] = pInfo.Bidopentime
-		}
-		if len(pInfo.Winnerorder) > 0 {
-			set["winnerorder"] = pInfo.Winnerorder
-		}
-		if thisinfo.HasPackage {
-			set["multipackage"] = 1
-		} else {
-			set["multipackage"] = 0
-		}
-
-		if pInfo.ProjectName != "" {
-			set["projectname"] = pInfo.ProjectName
-		}
-
-		if pInfo.ProjectCode != "" {
-			set["projectcode"] = pInfo.ProjectCode
-		}
-
-		if pInfo.Buyer != "" {
-			set["buyer"] = pInfo.Buyer
-		}
-
-		//预算、中标价
-		if qu.Float64All(tmp["bidamount"]) > 0 {
-			if qu.Float64All((*res)["bidamount"]) == 0 {
-				set["bidamount"] = tmp["bidamount"]
-			}
-		}
-
-		if qu.Float64All(tmp["budget"]) > 0 {
-			if qu.Float64All((*res)["budget"]) == 0 {
-				set["budget"] = tmp["budget"]
-			}
-		}
-
-		set["mpn"] = pInfo.MPN
-		set["mpc"] = pInfo.MPC
-		set["area"] = pInfo.Area
-		set["city"] = pInfo.City
-
-		//e := InitEL(util.ObjToString((*res)["extractpos"]))
-		if thisinfo.dealtype == 1 {
-			var sonpackage map[string]interface{}
-			for _, obj := range tmp["package"].(map[string]interface{}) {
-				sonpackage, _ = obj.(map[string]interface{})
-			}
-			for _, v2 := range []string{"budget", "budget_w", "winner", "winner_w", "bidstatus", "bidstatus_w"} {
-				if sonpackage[v2] != nil {
-					tmp[v2] = sonpackage[v2]
-				}
-			}
-		}
-		//e.fieldpriority(&tmp, res, &set)
-		//set["extractpos"] = e.GetVal()
-		if thisinfo.HasPackage { //多包处理
-			p1, _ := (*res)["package"].(map[string]interface{})
-			p2, _ := tmp["package"].(map[string]interface{})
-			if p2 != nil {
-				if p1 != nil {
-					for pk2, pv2 := range p2 {
-						if p1[pk2] != nil { //合并
-							item1, _ := p1[pk2].(map[string]interface{})
-							item2, _ := pv2.(map[string]interface{})
-							if item1 != nil && item2 != nil { //原始项
-								for ik1, iv1 := range item2 {
-									if item1[ik1] == nil {
-										item1[ik1] = iv1
-									}
-								}
-							}
-						} else {
-							p1[pk2] = pv2
-						}
-					}
-				} else {
-					p1 = p2
-				}
-			}
-			set["package"] = p1
-		}
-		//中标候选人合并
-
-		update := map[string]interface{}{}
-		if len(set) > 0 {
-			update["$set"] = set
-		}
-		//保留原数据吧
-		push := NewPushInfo(tmp)
-		push["compareStr"] = comStr
-		update["$push"] = map[string]interface{}{
-			"list": push,
-		}
-		if len(update) > 0 {
-			MQFW.Update(projectColl, map[string]interface{}{
-				"_id": qu.StringTOBsonId(pInfo.Id),
-			}, &update, false, false)
-		}
-	}
-	//再往redis中放 index
-	//往队列中增加时间 -------------->start
-}
-
-func EqInfoUpdate(thisinfo *Info, pInfo *ProjectInfo) {
-	var tk int
-	bf1 := false
-	for _k, tv := range pInfo.Publistime {
-		tk = _k
-		if tv > thisinfo.Publishtime {
-			bf1 = true
-			break
-		}
-	}
-	if bf1 {
-		pInfo.Publistime = append(append(pInfo.Publistime[:tk], thisinfo.Publishtime), pInfo.Publistime[tk:]...)
-		//pInfo.InfoType = append(append(pInfo.InfoType[:tk], []string{thisinfo.TopType, thisinfo.SubType}), pInfo.InfoType[tk:]...)
-		//pInfo.Ids = append(append(pInfo.Ids[:tk], thisinfo.Id), pInfo.Ids[tk:]...)
-	} else {
-		pInfo.Publistime = append(pInfo.Publistime, thisinfo.Publishtime)
-		//pInfo.InfoType = append(pInfo.InfoType, []string{thisinfo.TopType, thisinfo.SubType})
-	}
-	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
-	//增加发布时间结束----------------->end
-
-	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
-		pInfo.Buyer = thisinfo.Buyer
-	}
-	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
-		pInfo.Agency = thisinfo.Agency
-	}
-	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
-		pInfo.ProjectCode = thisinfo.ProjectCode
-	}
-
-	if pInfo.Area == "全国" && thisinfo.Area != "全国" {
-		pInfo.Area = thisinfo.Area
-		pInfo.City = thisinfo.City
-	}
-	if thisinfo.Buyerperson != "" && thisinfo.Buyertel != "" && len([]rune(thisinfo.Buyertel)) > 6 {
-		pInfo.Buyerperson = thisinfo.Buyerperson
-		pInfo.Buyertel = thisinfo.Buyertel
-	}
-	if thisinfo.Buyerclass != "" {
-		pInfo.Buyerclass = thisinfo.Buyerclass
-	}
-	if thisinfo.District != "" {
-		pInfo.District = thisinfo.District
-	}
-	if thisinfo.Bidopentime > 0 {
-		pInfo.Bidopentime = thisinfo.Bidopentime
-	}
-	if len(thisinfo.Topscopeclass) > 0 {
-		sort.Strings(pInfo.Topscopeclass)
-		for _, k := range thisinfo.Topscopeclass {
-			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
-				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
-				sort.Strings(pInfo.Topscopeclass)
-			}
-		}
-	}
-
-	if len(thisinfo.Subscopeclass) > 0 {
-		sort.Strings(pInfo.Subscopeclass)
-		for _, k := range thisinfo.Subscopeclass {
-			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
-				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
-				sort.Strings(pInfo.Subscopeclass)
-			}
-		}
-	}
-	//winner
-	if len(thisinfo.Winners) > 0 {
-		sort.Strings(pInfo.Winners)
-		for _, k := range thisinfo.Winners {
-			if BinarySearch(pInfo.Winners, k) == -1 {
-				pInfo.Winners = append(pInfo.Winners, k)
-				sort.Strings(pInfo.Winners)
-			}
-		}
-	}
-	//winnerorder
-	if len(thisinfo.Winnerorder) > 0 {
-		sort.Strings(pInfo.Winnerorder)
-		for _, k := range thisinfo.Winnerorder {
-			if BinarySearch(pInfo.Winnerorder, k) == -1 {
-				pInfo.Winnerorder = append(pInfo.Winnerorder, k)
-				sort.Strings(pInfo.Winnerorder)
-			}
-		}
-	}
-}
-
-//二分字符串查找
-func BinarySearch(s []string, k string) int {
-	sort.Strings(s)
-	lo, hi := 0, len(s)-1
-	for lo <= hi {
-		m := (lo + hi) >> 1
-		if s[m] < k {
-			lo = m + 1
-		} else if s[m] > k {
-			hi = m - 1
-		} else {
-			return m
-		}
-	}
-	return -1
-}
-
-var FIELDS = []string{"area", "city", "district", "projectname", "projectcode", "buyer", "winner", "budget", "bidamount", "bidstatus", "agency", "projectscope"}
-
-func NewProject(tmp map[string]interface{}, thisinfo *Info) string {
-
-	set := map[string]interface{}{}
-	for _, f := range FIELDS {
-		if tmp[f] != nil {
-			set[f] = tmp[f]
-		}
-	}
-	set["s_projectname"] = thisinfo.ProjectName
-	set["createtime"] = time.Now().Unix()
-	set["sourceinfoid"] = qu.BsonIdToSId(tmp["_id"])
-	set["sourceinfourl"] = tmp["href"]
-	set["topscopeclass"] = thisinfo.Topscopeclass
-	set["subscopeclass"] = thisinfo.Subscopeclass
-	if thisinfo.Buyerperson != "" {
-		set["buyerperson"] = thisinfo.Buyerperson
-	}
-	if thisinfo.Buyertel != "" {
-		set["buyertel"] = thisinfo.Buyertel
-	}
-	if thisinfo.Buyerclass != "" {
-		set["buyertel"] = thisinfo.Buyerclass
-	}
-	if thisinfo.Bidopentime > 0 {
-		set["bidopentime"] = thisinfo.Bidopentime
-	}
-	if len(thisinfo.Winnerorder) > 0 {
-		set["winnerorder"] = thisinfo.Winnerorder
-	}
-	s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
-	set["s_subscopeclass"] = s_subscopeclass
-	s_winner := strings.Join(thisinfo.Winners, ",")
-	set["s_winner"] = s_winner
-	if tmp["package"] != nil {
-		set["package"] = tmp["package"] //没定义优先级
-	}
-	push := NewPushInfo(tmp)
-	set["list"] = []bson.M{
-		push,
-	}
-
-	id := MQFW.Save(projectColl, set)
-	p1 := NewPinfo(id, thisinfo)
-	redis.PutCKV(REDISIDS, id, p1)
-
-	return id
-}
-
-//生成存放在redis数组中的对象
-func NewPinfo(id string, thisinfo *Info) ProjectInfo {
-	p1 := ProjectInfo{
-		Publistime:    []int64{thisinfo.Publishtime},
-		InfoType:      [][]string{[]string{thisinfo.TopType, thisinfo.SubType}},
-		Id:            id,
-		Ids:           []string{thisinfo.Id},
-		Topscopeclass: thisinfo.Topscopeclass,
-		Subscopeclass: thisinfo.Subscopeclass,
-		Winners:       thisinfo.Winners,
-		ProjectName:   thisinfo.ProjectName,
-		ProjectCode:   thisinfo.ProjectCode,
-		Buyer:         thisinfo.Buyer,
-		Agency:        thisinfo.Agency,
-		Area:          thisinfo.Area,
-		City:          thisinfo.City,
-		District:      thisinfo.District,
-		MPN:           []string{},
-		MPC:           []string{},
-		HasPackage:    thisinfo.HasPackage,
-		Package:       map[string]interface{}{},
-		Buyerclass:    thisinfo.Buyerclass,
-		Bidopentime:   thisinfo.Bidopentime,
-		Winnerorder:   thisinfo.Winnerorder,
-	}
-	if thisinfo.LenPTC > 5 {
-		p1.MPC = append(p1.MPC, thisinfo.PTC)
-	}
-	for k4, _ := range thisinfo.Package {
-		p1.Package[k4] = ""
-	}
-	return p1
-}
-
-type IdsMapAndIndex struct {
-	Key   *Key
-	Index int
-}
-
-var sm = sync.Map{}
-
-var ALock = sync.Mutex{}
-
-var mapPn = map[string]*Key{}
-var mapPc = map[string]*Key{}
-var mapPb = map[string]*Key{}
-
-//var mapPnLock, mapPcLock, mapPbLock = sync.Mutex{}, sync.Mutex{}, sync.Mutex{}
-
-//从对应map中获取对比的项目id
-func getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key) {
-	ALock.Lock()
-	defer ALock.Unlock()
-	res = []*Key{}
-	bpn, bpc, bptc, bpb = -1, -1, -1, -1
-	if pn != "" {
-		//mapPnLock.Lock()
-		ids := mapPn[pn]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPn[pn] = ids
-			bpn = 0
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-		//mapPnLock.Unlock()
-
-		//		index := len(pn) % size
-		//		m := AllPNMap[index]
-		//		m.Lock.Lock()
-		//		ids := m.Map[pn]
-		//		//log.Println("PN", ids, pn, index)
-		//		if ids == nil {
-		//			ids = &Key{Arr: []string{}}
-		//			m.Map[pn] = ids
-		//			bpn = 0
-		//		}
-		//		ids.Lock.Lock()
-		//		res = append(res, ids)
-		//		m.Lock.Unlock()
-	}
-
-	if pc != "" {
-		//mapPcLock.Lock()
-		ids := mapPc[pc]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPc[pc] = ids
-			bpc = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-		//mapPcLock.Unlock()
-		//		index := len(pc) % size
-		//		m := AllPCMap[index]
-		//		m.Lock.Lock()
-		//		ids := m.Map[pc]
-		//		//log.Println("PC", ids, pc, index)
-		//		if ids == nil {
-		//			ids = &Key{Arr: []string{}}
-		//			m.Map[pc] = ids
-		//			bpc = len(res)
-		//		}
-		//		ids.Lock.Lock()
-		//		res = append(res, ids)
-		//		m.Lock.Unlock()
-	}
-
-	if ptc != "" {
-		ids := mapPc[ptc]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPc[ptc] = ids
-			bptc = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-		//		index := len(ptc) % size
-		//		m := AllPTCMap[index]
-		//		m.Lock.Lock()
-		//		ids := m.Map[ptc]
-		//		//log.Println("PTC", ids, ptc, index)
-		//		if ids == nil {
-		//			ids = &Key{Arr: []string{}}
-		//			m.Map[ptc] = ids
-		//			bptc = len(res)
-		//		}
-		//		ids.Lock.Lock()
-		//		res = append(res, ids)
-		//		m.Lock.Unlock()
-	}
-
-	if pb != "" {
-		ids := mapPb[pb]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPb[pb] = ids
-			bpb = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-		//		index := len(pb) % size
-		//		m := AllPBMap[index]
-		//		m.Lock.Lock()
-		//		ids := m.Map[pb]
-		//		//log.Println("PB", ids, pb, index)
-		//		if ids == nil {
-		//			ids = &Key{Arr: []string{}}
-		//			m.Map[pb] = ids
-		//			bpb = len(res)
-		//		}
-		//		ids.Lock.Lock()
-		//		res = append(res, ids)
-		//		m.Lock.Unlock()
-	}
-
-	return
-}
-
-//抽取信息打标记
-func extInfoTag(sflag, id string) {
-	MQFW.UpdateById(extractColl, id,
-		map[string]interface{}{
-			"$set": map[string]interface{}{
-				"meger_sflag": sflag,
-			},
-		})
-}

fullproject/src_dev1/merge_select.go

@@ -1,321 +0,0 @@
-package main
-
-import (
-	"math"
-	"regexp"
-)
-
-func Select(compareStr string, info *Info, compareInfo *ProjectInfo) (res int) {
-	//没有可对比的项目名称、或项目编号
-	if compareNoPass[compareStr] {
-
-	} else {
-		switch compareStr {
-		case "AAA":
-			res = 3
-		case "AAB":
-			res = 3
-		case "ABA":
-			res = 3
-		case "ABB":
-			if info.LenPTC > 6 || info.LenPC > 6 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BAA":
-			if info.LenPN > 10 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BAB":
-			if info.LenPN > 10 || info.LenPTC > 6 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BBA":
-			if info.LenPN > 10 || info.LenPC > 6 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BBB":
-			v := 0
-			if info.LenPN > 10 {
-				v++
-			}
-			if info.LenPC > 6 {
-				v++
-			}
-			if info.LenPTC > 6 {
-				v++
-			}
-			if v > 1 {
-				res = 3
-			} else if v == 1 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "AAD":
-			res = 3
-		case "ABD":
-			if info.LenPC > 12 || info.LenPN > 16 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BAD":
-			if info.LenPC > 12 || info.LenPN > 16 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BBD":
-			if info.LenPC > 12 && info.LenPN > 16 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "ADA":
-			res = 3
-		case "ADB":
-			if info.LenPTC > 12 || info.LenPN > 16 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BDA":
-			if info.LenPTC > 12 || info.LenPN > 16 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BDB":
-			if info.LenPTC > 12 && info.LenPN > 16 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "ADD":
-			//			if info.LenPN > 18 {
-			//				res = 3
-			//			} else
-			if info.LenPN > 14 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BDD":
-			if info.LenPN > 18 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DAA":
-			if info.LenPTC > 12 || info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DAB":
-			if info.LenPTC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DAD":
-			if info.LenPTC > 14 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DBA":
-			if info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DBB":
-			if info.LenPTC > 12 && info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DBD":
-			if info.LenPC > 8 {
-				res = 1
-			}
-		case "DDA":
-			if info.LenPTC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DDB":
-			if info.LenPC > 8 {
-				res = 1
-			}
-		case "ACA":
-			if info.LenPN > 12 || info.LenPTC > 9 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "ACB":
-			if info.LenPN > 14 && info.LenPTC > 10 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BCA":
-			if info.LenPN > 12 && info.LenPTC > 9 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BCB":
-			if info.LenPN > 12 && info.LenPTC > 9 {
-				res = 2
-			} else if info.LenPN > 16 || info.LenPTC > 12 {
-				res = 1
-			}
-		case "AAC":
-			if info.LenPN > 12 || info.LenPC > 9 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "ABC":
-			if info.LenPN > 14 || info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BAC":
-			if info.LenPN > 14 || info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BBC":
-			//			if info.LenPN > 12 && info.LenPTC > 9 {
-			//				res = 1
-			//			} else {
-			//				res = 1
-			//			}
-			res = 1
-		case "ACC":
-			//			if info.LenPC > 4 && len(compareInfo.ProjectCode) > 4 && CheckHanAndNum(info.ProjectCode) && CheckHanAndNum(compareInfo.ProjectCode) {
-			//				//未考虑MPC
-			//			} else if info.LenPN > 16 {
-			//				res = 1
-			//			}
-			//
-		case "ACD":
-			//项目编号不一致
-			if math.Abs(float64(info.LenPC-len([]rune(compareInfo.ProjectCode)))) > 3 && info.LenPN > 12 {
-				res = 1
-			}
-		case "ADC":
-			if info.LenPN > 16 {
-				res = 1
-			}
-		case "BCC":
-			//项目编号不一致
-		case "BCD":
-			//项目编号不一致
-			if info.LenPC > 4 && len(compareInfo.ProjectCode) > 4 && (!_numreg1.MatchString(info.ProjectCode) && !_numreg1.MatchString(compareInfo.ProjectCode)) {
-				//未考虑MPC
-			} else if info.LenPN > 18 && info.LenPC-len([]rune(compareInfo.ProjectCode)) != 0 {
-				res = 1
-			}
-		case "BDC":
-			if info.LenPN > 18 && info.LenPTC-len([]rune(compareInfo.ProjectCode)) != 0 {
-				res = 1
-			}
-		case "CAA":
-			if info.LenPC > 10 || info.LenPTC > 10 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "CAB":
-			if info.LenPC > 10 && info.LenPTC > 10 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "CAC":
-			if info.LenPC > 14 && math.Abs(float64(info.LenPN-len([]rune(compareInfo.ProjectName)))) > 4 {
-				res = 1
-			}
-		case "CAD":
-			if info.LenPC > 14 && math.Abs(float64(info.LenPN-len([]rune(compareInfo.ProjectName)))) > 4 {
-				res = 1
-			}
-		case "CBA":
-			if info.LenPC > 10 && info.LenPTC > 10 {
-				res = 1
-			}
-		case "CBB":
-			if info.LenPC > 10 && info.LenPTC > 10 {
-				res = 1
-			}
-		case "CBC":
-			if info.LenPC > 14 || (info.LenPC > 9 && !numCheckPc.MatchString(info.ProjectCode)) {
-				res = 1
-			}
-		case "CBD":
-			if info.LenPC > 14 || (info.LenPC > 10 && !numCheckPc.MatchString(info.ProjectCode)) {
-				res = 1
-			}
-		case "CCA":
-			if (info.LenPTC > 12 && !numCheckPc.MatchString(info.PTC)) || CheckHanAndNum(info.PTC) {
-				res = 1
-			}
-		case "CCB":
-			//
-		case "CDA":
-			if info.LenPTC > 10 || (info.LenPTC > 8 && !numCheckPc.MatchString(info.PTC)) || CheckHanAndNum(info.PTC) {
-				res = 1
-			}
-		case "CDB":
-			if info.LenPTC > 15 || (info.LenPTC > 12 && !numCheckPc.MatchString(info.PTC)) {
-				res = 1
-			}
-		case "DAC":
-			if info.LenPC > 12 || (info.LenPC > 10 && !numCheckPc.MatchString(info.ProjectCode)) || CheckHanAndNum(info.ProjectCode) {
-				res = 1
-			}
-		case "DBC":
-			if info.LenPC > 15 || (info.LenPC > 12 && !numCheckPc.MatchString(info.ProjectCode)) {
-				res = 1
-			}
-		case "DCA":
-			if info.LenPTC > 14 || (info.LenPTC > 10 && !numCheckPc.MatchString(info.PTC)) || CheckHanAndNum(info.PTC) {
-				res = 1
-			}
-		case "DCB":
-			//
-			if !CheckHanAndNum(info.PTC) && !numCheckPc.MatchString(info.PTC) && info.LenPTC > 10 {
-				res = 1
-			}
-		}
-
-	}
-	return
-}
-
-var nreg1 = regexp.MustCompile("[0-9]{2,}")
-var zreg1 = regexp.MustCompile("[a-zA-Z]{1,}")
-var hreg1 = regexp.MustCompile(`[\p{Han}]+`)
-
-func CheckHanAndNum(str string) (b bool) {
-	return nreg1.MatchString(str) && hreg1.MatchString(str)
-}
-func CheckZimuAndNum(str string) (b bool) {
-	return zreg1.MatchString(str) && nreg1.MatchString(str)
-}

fullproject/src_dev2/config.json

@@ -1,16 +0,0 @@
-{
-    "thread": 1,
-    "mongodbServers": "192.168.3.207:27082",
-    "mongodbPoolSize": 10,
-    "mongodbName": "cesuo",
-    "extractColl": "key1_biddingall",
-    "projectColl": "projectset_0809",
-    "redisaddrs": "ids=192.168.3.207:1378,keys=192.168.3.207:1378,info=192.168.3.207:1378",
-    "redisPoolSize": 20,
-    "jkmail": {
-        "to": "zhangjinkun@topnet.net.cn",
-        "api": "http://10.171.112.160:19281/_send/_mail"
-    },
-    "udpport": ":1482",
-    "nextNode": []
-}

fullproject/src_dev2/init.go

@@ -1,291 +0,0 @@
-package main
-
-import (
-	"log"
-	mu "mfw/util"
-	"qfw/util"
-	"qfw/util/mongodb"
-	"qfw/util/redis"
-	"regexp"
-	"sync"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-const (
-	ProjectCache = "info" //存放每条项目信息，key为项目ID
-)
-
-var (
-	Sysconfig                map[string]interface{} //读取配置文件
-	MongoTool                mongodb.MongodbSim     //mongodb连接
-	ExtractColl, ProjectColl string                 //抽取表、项目表
-	CurrentMegerTime         int64                  //当前合并到的信息的时间
-	CurrentMegerCount        int                    //当前合并计数
-	MultiThread              = make(chan bool, 5)   //项目合并线程
-	AllIdsMap2               = map[string]*ID{}
-	AllIdsMapLock            = sync.Mutex{}
-)
-
-var (
-	_datereg   = regexp.MustCompile("20[0-2][0-9][年-][0-9]{1,2}[月-][0-9]{1,2}[日-]([0-9]{1,2}时[0-9]{0,2})?")
-	_numreg1   = regexp.MustCompile("^[0-9-]{1,8}$")
-	_zimureg1  = regexp.MustCompile("^[a-zA-Z-]{1,7}$")
-	_nzreg     = regexp.MustCompile("^[0-9a-zA-Z-]+$")
-	_hanreg    = regexp.MustCompile(`^[\p{Han}:：【】\\[\\]（）()－-、]+$`)
-	replaceStr = regexp.MustCompile("(工程|采购|项目|[?！、【】（）—()－-]|栏标价|中标候选人|招标代理)")
-	pStr       = regexp.MustCompile("(勘察|监理|施工|设计|验收|标段|分包|子包|[0-9A-Z]包|[一二三四五六七八九十0-9]批)")
-	nreg1      = regexp.MustCompile("[0-9]{2,}")
-	zreg1      = regexp.MustCompile("[a-zA-Z]{1,}")
-	hreg1      = regexp.MustCompile(`[\p{Han}]+`)
-	numCheckPc = regexp.MustCompile("^[0-9-]{1,10}$")
-
-	//存放项目名称
-	mapPn = map[string]*Key{}
-	//存放项目编号
-	mapPc = map[string]*Key{}
-	//存放采购单位
-	mapPb         = map[string]*Key{}
-	compareNoPass = map[string]bool{}
-	compareAB     = map[string]bool{}
-	compareAB2D   = map[string]bool{}
-	compareABD    = map[string]bool{}
-	compareAB2CD  = map[string]bool{}
-	compareABCD   = map[string]bool{}
-)
-
-func init() {
-	util.ReadConfig(&Sysconfig)
-	MultiThread = make(chan bool, util.IntAllDef(Sysconfig["thread"], 5))
-	redis.InitRedisBySize(Sysconfig["redisaddrs"].(string), util.IntAllDef(Sysconfig["redisPoolSize"], 60), 10, 300)
-	MongoTool = mongodb.MongodbSim{
-		MongodbAddr: Sysconfig["mongodbServers"].(string),
-		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
-		DbName:      Sysconfig["mongodbName"].(string),
-	}
-	MongoTool.InitPool()
-	ExtractColl = Sysconfig["extractColl"].(string)
-	ProjectColl = Sysconfig["projectColl"].(string)
-
-	udpport, _ := Sysconfig["udpport"].(string)
-	udpclient = mu.UdpClient{Local: udpport, BufSize: 1024}
-	udpclient.Listen(processUdpMsg)
-	log.Println("Udp服务监听", udpport)
-
-	//---不能通过
-	vm := []string{"C", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				compareNoPass[key] = true
-				//fmt.Println(key)
-			}
-		}
-	}
-	//fmt.Println("-------------------")
-
-	//三个元素一致 [AB][AB][AB]，分值最高
-	vm = []string{"A", "B"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				compareAB[key] = true
-				//fmt.Println(key)
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB))
-	//---至少两个一致，其他可能不存在
-	//[AB][AB][ABD]
-	//[AB][ABD][AB]
-	vm = []string{"A", "B"}
-	vm2 := []string{"A", "B", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 3; k++ {
-				key := vm[i] + vm[j] + vm2[k]
-				if !compareAB[key] {
-					compareAB2D[key] = true
-					//fmt.Println(key)
-
-				}
-			}
-		}
-	}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 3; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm2[j] + vm[k]
-				if !compareAB[key] {
-					compareAB2D[key] = true
-					//fmt.Println(key)
-
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB2D))
-	//---至少一个一致，其他可能不存在
-	//[ABD][ABD][ABD] //已经删除DDD
-	vm = []string{"A", "B", "D"}
-	for i := 0; i < 3; i++ {
-		for j := 0; j < 3; j++ {
-			for k := 0; k < 3; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] {
-					compareABD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareABD))
-
-	//[AB][ABCD][AB]
-	//[AB][AB][ABCD]
-	vm = []string{"A", "B"}
-	vm2 = []string{"A", "B", "C", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 4; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm2[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-					compareAB2CD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 4; k++ {
-				key := vm[i] + vm[j] + vm2[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-					compareAB2CD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB2CD))
-	//[ABECD][ABECD][ABECD]  //已经删除[CD][CD][CD]   //这个要重点讨论
-	vm = []string{"A", "B", "C", "D"}
-	for i := 0; i < 4; i++ {
-		for j := 0; j < 4; j++ {
-			for k := 0; k < 4; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareABD[key] && !compareNoPass[key] && !compareAB2CD[key] {
-					compareABCD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-}
-
-func CheckHanAndNum(str string) (b bool) {
-	return nreg1.MatchString(str) && hreg1.MatchString(str)
-}
-func CheckZimuAndNum(str string) (b bool) {
-	return zreg1.MatchString(str) && nreg1.MatchString(str)
-}
-
-type KeyMap struct {
-	Lock sync.Mutex
-	Map  map[string]*Key
-}
-
-type ID struct {
-	Id       string
-	Lock     sync.Mutex
-	lastTime int64
-	pos      int
-}
-type Key struct {
-	Arr  []string
-	Lock sync.Mutex
-}
-type IdAndLock struct {
-	Id   string
-	Lock sync.Mutex
-}
-
-func NewKeyMap() *KeyMap {
-	return &KeyMap{
-		Map:  map[string]*Key{},
-		Lock: sync.Mutex{},
-	}
-}
-
-//招标信息实体类
-type Info struct {
-	Id            string                 `json:"_id"`
-	Href          string                 `json:"href"` //源地址
-	Publishtime   int64                  `json:"publishtime"`
-	Title         string                 `json:"title"`
-	TopType       string                 `json:"toptype"`
-	SubType       string                 `json:"subtype"`
-	ProjectName   string                 `json:"projectname"`
-	ProjectCode   string                 `json:"projectcode"`
-	Buyer         string                 `json:"buyer"`
-	Buyerperson   string                 `json:"buyerperson"`
-	Buyertel      string                 `json:"buyertel"`
-	Agency        string                 `json:"agency"`
-	Area          string                 `json:"area"`
-	City          string                 `json:"city"`
-	District      string                 `json:"district"`
-	HasPackage    bool                   `json:"haspackage"`
-	Package       map[string]interface{} `json:"package"`
-	PNum          string                 `json:"pnum"`
-	Topscopeclass []string               `json:"topscopeclass"`
-	Subscopeclass []string               `json:"subscopeclass"`
-	Buyerclass    string                 `json:"buyerclass"`
-	Bidopentime   int64                  `json:"bidopentime"`
-	budget        float64                `json:"budget"`
-	bidamount     float64                `json:"bidamount"`
-	Winners       []string
-	dealtype      int
-
-	Winnerorder []string
-
-	PTC    string //从标题中抽的项目编号
-	pnbval int    //项目名称、编号、采购单位存在的个数
-	LenPC  int    //项目编号长度
-	LenPN  int    //项目名称长度
-	LenPTC int    //标题抽的项目编号长度
-}
-
-//项目实体类
-type ProjectInfo struct {
-	Id            bson.ObjectId          `bson:"_id"`
-	FirstTime     int64                  `json:"firsttime"` //项目的最早时间
-	LastTime      int64                  `json:"lasttime"`  //项目的最后时间
-	Ids           []string               `json:"ids"`
-	Topscopeclass []string               `json:"topscopeclass"`
-	Subscopeclass []string               `json:"subscopeclass"` //子行业分类
-	Winners       []string               `json:"winners"`       //中标人
-	ProjectName   string                 `json:"projectname"`   //项目名称
-	ProjectCode   string                 `json:"projectcode"`   //项目代码唯一（纯数字的权重低）
-	Buyer         string                 `json:"buyer"`         //采购单位唯一
-	MPN           []string               `json:"mpn"`           //合并后多余的项目名称
-	MPC           []string               `json:"mpc"`           //合并后多余的项目编号
-	Buyerperson   string                 `json:"buyerperson"`   //采购联系人
-	Buyertel      string                 `json:"buyertel"`      //采购联系人电话
-	Agency        string                 `json:"agency"`        //代理机构
-	Area          string                 `json:"area"`          //地区
-	City          string                 `json:"city"`          //地市
-	District      string                 `json:"district"`      //区县
-	HasPackage    bool                   `json:"haspackage"`    //是否有分包
-	Package       map[string]interface{} `json:"package"`       //分包的对比对象
-	Buyerclass    string                 `json:"buyerclass"`    //采购单位分类
-	Bidopentime   int64                  `json:"bidopentime"`   //开标时间
-	Zbtime        int64                  `json:"zbtime"`        //招标时间
-	Jgtime        int64                  `json:"jgtime"`        //结果中标时间
-	Bidamount     float64                `json:"bidamount"`     //中标金额
-	Budget        float64                `json:"budget"`        //预算
-	Winnerorder   []string               `json:"winnerorder"`   //中标候选人
-	score         int
-	comStr        string
-}

fullproject/src_dev2/load_data.go

@@ -1,65 +0,0 @@
-package main
-
-import (
-	"time"
-	//"encoding/json"
-	"log"
-)
-
-//初始加载数据，默认加载最近6个月的数据
-
-func loadData(projectColl string, month int, bCacheRedis bool) {
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
-
-	q := map[string]interface{}{}
-	it := sess.DB(MongoTool.DbName).C(projectColl).Find(&q).Iter()
-	AllIdsMapLock.Lock()
-	tmp := &ProjectInfo{}
-	n := 0
-	for it.Next(tmp) {
-		n++
-		if n%1000 == 0 {
-			log.Println("current", n, "\n", tmp.Id, tmp)
-			time.Sleep(2 * time.Second)
-		}
-		for _, v := range append([]string{tmp.ProjectName}, tmp.MPN...) {
-			if v != "" {
-				k := mapPn[v]
-				if k == nil {
-					k = &Key{Arr: []string{}}
-					mapPn[v] = k
-				}
-				k.Arr = append(k.Arr, tmp.Id.Hex())
-			}
-		}
-		for _, v := range append([]string{tmp.ProjectCode}, tmp.MPC...) {
-			if v != "" {
-				k := mapPc[v]
-				if k == nil {
-					k = &Key{Arr: []string{}}
-					mapPc[v] = k
-				}
-				k.Arr = append(k.Arr, tmp.Id.Hex())
-			}
-		}
-		if tmp.Buyer != "" {
-			k := mapPb[tmp.Buyer]
-			if k == nil {
-				k = &Key{Arr: []string{}}
-				mapPb[tmp.Buyer] = k
-			}
-			k.Arr = append(k.Arr, tmp.Id.Hex())
-		}
-
-		AllIdsMap2[tmp.Id.Hex()] = &ID{Id: tmp.Id.Hex(), lastTime: tmp.LastTime}
-
-		if bCacheRedis {
-			//存入redis
-
-		}
-
-	}
-	AllIdsMapLock.Unlock()
-	log.Println("load over")
-}

fullproject/src_dev2/main.go

@@ -1,61 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"log"
-	mu "mfw/util"
-	"net"
-	"time"
-)
-
-var (
-	udpclient    mu.UdpClient //udp对象
-	SingleThread = make(chan bool, 1)
-)
-
-func main() {
-
-	//udp跑增量  id段   zl
-	//udp跑全量			ql
-	//udp跑历史数据  信息id1,id2/或id段  ls
-	//udp强制合并  信息id1,id2,id3 [项目id] 不存在时新建  qzhb
-	//udp强制拆分  项目id,信息id1,id2          qzcf
-	//udp重新合并  信息id1,id2,id3             cxhb
-	//loadData("project_0809", 1, true)
-	time.Sleep(99999 * time.Hour)
-}
-
-func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
-	switch act {
-	case mu.OP_TYPE_DATA: //上个节点的数据
-		var mapInfo map[string]interface{}
-		err := json.Unmarshal(data, &mapInfo)
-		log.Println("err:", err, "mapInfo:", mapInfo)
-		if err != nil {
-			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
-		} else if mapInfo != nil {
-			key, _ := mapInfo["key"].(string)
-			if key == "" {
-				key = "udpok"
-			}
-			go udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
-			SingleThread <- true
-			tasktype, _ := mapInfo["stype"].(string)
-			log.Println("tasktype:", tasktype)
-			switch tasktype {
-			case "ql":
-				go func() {
-					defer func() {
-						<-SingleThread
-					}()
-					taskQl(mapInfo)
-				}()
-			}
-		}
-	case mu.OP_NOOP: //下个节点回应
-		ok := string(data)
-		if ok != "" {
-			log.Println("ok:", ok)
-		}
-	}
-}

fullproject/src_dev2/merge.go

@@ -1,377 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"log"
-	"math"
-	qu "qfw/util"
-	"qfw/util/redis"
-	"sort"
-	//"strconv"
-	"strings"
-	"sync"
-)
-
-//单线程控制查找
-var findLock = sync.Mutex{}
-
-//从对应map中获取对比的项目id
-func getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
-	findLock.Lock()
-	defer findLock.Unlock()
-	res = []*Key{}
-	//是否查找到，并标识位置。-1代表未查找到。
-	bpn, bpc, bptc, bpb = -1, -1, -1, -1
-	if pn != "" {
-		ids := mapPn[pn]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPn[pn] = ids
-			bpn = 0
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-
-	if pc != "" {
-		ids := mapPc[pc]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPc[pc] = ids
-			bpc = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-
-	if ptc != "" {
-		ids := mapPc[ptc]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPc[ptc] = ids
-			bptc = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-
-	if pb != "" {
-		ids := mapPb[pb]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPb[pb] = ids
-			bpb = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-	repeatId := map[string]bool{}
-	idArr = []string{} //项目id
-	IDArr = []*ID{}    //项目信息
-	for _, m := range res {
-		for _, id := range m.Arr {
-			if !repeatId[id] {
-				repeatId[id] = true
-				//_, _ = strconv.ParseInt(id[0:8], 16, 64)
-				AllIdsMapLock.Lock()
-				Id := AllIdsMap2[id]
-				AllIdsMapLock.Unlock()
-				Id.Lock.Lock()
-				idArr = append(idArr, id)
-				IDArr = append(IDArr, Id)
-			}
-		}
-	}
-	return
-}
-
-//项目合并入口
-func startProjectMerge(info *Info, tmp map[string]interface{}) {
-	//只有或没有采购单位的无法合并
-	//bpn, bpc, bptc, bpb 是否查找到，并标识位置。-1代表未查找到。
-	//pids 是项目id数组集合
-	//IDArr，是单个项目ID对象集合
-	bpn, bpc, bptc, bpb, pids, idArr, IDArr := getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer)
-	//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock
-
-	for _, m := range pids {
-		defer m.Lock.Unlock()
-	}
-	for _, id := range IDArr {
-		defer id.Lock.Unlock()
-	}
-
-	bFindProject := false
-	findPid := ""
-	//获取完id，进行计算
-	if len(idArr) > 0 && len(idArr) == len(IDArr) {
-		res := redis.Mget(ProjectCache, idArr)
-		//定义两组
-		comRes1 := []*ProjectInfo{} //优先级最高的对比结果数组
-		comRes2 := []*ProjectInfo{} //优化级其次
-		comRes3 := []*ProjectInfo{}
-		if len(res) == len(idArr) {
-			for _, v := range res {
-				var cp ProjectInfo
-				comStr := ""
-				var compareProject *ProjectInfo
-				err := json.Unmarshal(v.([]byte), &cp)
-				compareProject = &cp
-				//问题出地LastTime!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-				diffTime := math.Abs(float64(info.Publishtime - compareProject.LastTime))
-				if err != nil {
-					log.Println("从redis取id信息出错！！！", err)
-				} else if diffTime < 360*86400 {
-					//"A 相等 	B 被包含 	C 不相等	 	D不存在  E被包含
-					compareStr, score := comparePNCB(info, compareProject)
-					//if info.ProjectCode == "ZWJLZXQ-2016-0508" {
-					//log.Println(compareStr)
-					//}
-					resVal := Select(compareStr, info, compareProject)
-					//---------------------------------------
-					if resVal > 0 {
-						compareBuyer := "D"
-						if info.Buyer != "" {
-							if info.Buyer == compareProject.Buyer {
-								compareBuyer = "A"
-								score += 3
-							} else if compareProject.Buyer != "" {
-								if strings.Contains(info.Buyer, compareProject.Buyer) || strings.Contains(compareProject.Buyer, info.Buyer) {
-									compareBuyer = "B"
-									score += 1
-								} else {
-									compareBuyer = "C"
-								}
-							}
-						}
-
-						//---------------------------------------
-
-						compareCity := ""
-						if info.Area != "全国" && info.Area != "" && info.Area == compareProject.Area {
-							compareCity += "A"
-							score += 2
-						} else if info.Area == "全国" || compareProject.Area == "全国" {
-							compareCity += "B"
-							score += 1
-						} else {
-							compareCity += "C"
-						}
-						if compareCity != "C" {
-							if info.City != "" && info.City == compareProject.City {
-								compareCity += "A"
-								score += 2
-							} else {
-								if info.Area == "全国" || compareProject.Area == "全国" {
-									compareCity += "B"
-								} else if info.City == compareCity {
-									compareCity += "B"
-								} else {
-									compareCity += "C"
-								}
-							}
-						} else {
-							compareCity += "C"
-						}
-
-						compareTime := "D"
-						//diffTime := math.Abs(float64(info.Publishtime - compareProject.Publistime[len(compareProject.Publistime)-1]))
-						if diffTime < 45*86400 {
-							compareTime = "A"
-							score += 2
-						} else if diffTime < 90*86400 {
-							compareTime = "B"
-							score += 1
-						}
-						score2 := 0
-						compareAgency := "D"
-						if info.Agency != "" {
-							if info.Agency == compareProject.Agency {
-								compareAgency = "A"
-								score += 2
-								score2 += 1
-							} else if compareProject.Agency != "" {
-								if strings.Contains(info.Agency, compareProject.Agency) || strings.Contains(compareProject.Agency, info.Agency) {
-									compareAgency = "B"
-									score += 1
-									score2 += 1
-								} else {
-									compareAgency = "C"
-								}
-							}
-						}
-						compareBudget := "C"
-						if info.budget > 0 && (info.budget == compareProject.Budget || (compareProject.Bidamount > 0 && info.budget > compareProject.Bidamount && (info.budget-compareProject.Bidamount) < (0.1*info.budget))) {
-							compareBudget = "A"
-							score += 1
-							score2 += 1
-						}
-						compareBidmount := "C"
-						if info.bidamount > 0 && (info.bidamount == compareProject.Bidamount || (compareProject.Budget > 0 && compareProject.Budget > info.bidamount && (compareProject.Budget-info.bidamount) < 0.1*compareProject.Budget)) {
-							compareBidmount = "A"
-							score += 1
-							score2 += 1
-						}
-
-						//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构
-
-						comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
-						compareProject.comStr = comStr
-						eqV := 0
-						switch resVal {
-						case 3:
-							if compareBuyer < "C" {
-								eqV = 1
-							} else if compareBuyer == "D" {
-								if "AA" == compareCity && compareAgency != "C" && compareTime != "D" && score2 > 0 {
-									eqV = 2
-								} else if compareCity[1:1] != "C" && score2 > 0 && compareTime == "A" {
-									eqV = 3
-								} else if compareTime != "D" && compareAgency != "C" && score2 > 1 {
-									eqV = 3
-								}
-							} else {
-								if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-									eqV = 2
-								} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 1 {
-									eqV = 3
-								}
-							}
-
-						case 2:
-							if compareBuyer < "C" {
-								if compareTime != "D" && "AA" == compareCity && score2 > 0 {
-									eqV = 2
-								} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
-									eqV = 3
-								}
-							} else if compareBuyer == "D" {
-								if "AA" == compareCity && compareTime != "D" && score2 > 1 {
-									eqV = 3
-								} else if score2 > 2 && compareTime == "A" {
-									eqV = 3
-								}
-							} else {
-								if "AA" == compareCity && (compareAgency == "A" || score2 > 1) && compareTime == "A" {
-									eqV = 3
-								}
-							}
-						case 1:
-							if compareBuyer < "C" {
-								if compareTime != "D" && "AA" == compareCity && score2 > 1 {
-									eqV = 2
-								} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 1 {
-									eqV = 3
-								}
-							} else if compareBuyer == "D" {
-								if "AA" == compareCity && compareTime != "D" && score2 > 2 {
-									eqV = 3
-								} else if compareCity[1:1] != "C" && score2 > 2 && compareTime == "A" {
-									eqV = 3
-								}
-							} else {
-								if "AA" == compareCity && score2 > 2 && compareTime == "A" {
-									eqV = 3
-								}
-							}
-						}
-						if eqV == 1 {
-							comRes1 = append(comRes1, compareProject)
-						} else if eqV == 2 {
-							comRes2 = append(comRes2, compareProject)
-						} else if eqV == 3 {
-							comRes3 = append(comRes3, compareProject)
-						} else {
-							log.Println("+++++++++++", resVal, comStr, info, compareProject)
-						}
-					}
-				}
-			}
-			//--------------------------------对比完成-----------------------
-			//更新数组、更新项目
-			for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} {
-				if len(resN) > 0 {
-					if len(resN) > 1 {
-						sort.Slice(resN, func(i, j int) bool {
-							return resN[i].score > resN[j].score
-						})
-					}
-
-					bFindProject = true
-					findPid = resN[0].Id.Hex()
-					for k2, bv := range []int{bpn, bpc, bptc, bpb} {
-						if bv > -1 {
-							pids[bv].Arr = append(pids[bv].Arr, findPid)
-							if k2 == 0 {
-								if resN[0].ProjectName == "" {
-									resN[0].ProjectName = info.ProjectName
-								} else {
-									if resN[0].MPN == nil {
-										resN[0].MPN = []string{info.ProjectName}
-									} else {
-										resN[0].MPN = append(resN[0].MPN, info.ProjectName)
-									}
-								}
-
-							} else if k2 < 3 {
-								if resN[0].ProjectCode == "" {
-									resN[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
-								} else {
-									if resN[0].MPC == nil {
-										resN[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
-									} else {
-										resN[0].MPC = append(resN[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
-									}
-								}
-
-							} else {
-								if resN[0].Buyer == "" {
-									resN[0].Buyer = info.Buyer
-								}
-							}
-						}
-					}
-					UpdateProject(tmp, info, resN[0], kv+1, resN[0].comStr)
-					//更新AllIdsMao中的时间
-					redis.Put(ProjectCache, findPid, resN[0], 0)
-					break
-				}
-			}
-		} else {
-			log.Println("redis记录缺失！！！")
-		}
-	}
-
-	if !bFindProject {
-		//没有找到
-		id := NewProject(tmp, info)
-		//_, _ = strconv.ParseInt(id[0:8], 16, 64)
-		AllIdsMapLock.Lock()
-		//AllIdsMap[int(t)%idsMapSize][id] = &ID{Id: id, lastTime: info.Publishtime}
-		AllIdsMap2[id] = &ID{Id: id, lastTime: info.Publishtime}
-		AllIdsMapLock.Unlock()
-		for _, m := range pids {
-			m.Arr = append(m.Arr, id)
-		}
-	}
-
-}
-
-//二分字符串查找
-func BinarySearch(s []string, k string) int {
-	sort.Strings(s)
-	lo, hi := 0, len(s)-1
-	for lo <= hi {
-		m := (lo + hi) >> 1
-		if s[m] < k {
-			lo = m + 1
-		} else if s[m] > k {
-			hi = m - 1
-		} else {
-			return m
-		}
-	}
-	return -1
-}
-
-var FIELDS = []string{"area", "city", "district", "projectname", "projectcode", "buyer", "winner", "budget", "bidamount", "bidstatus", "agency", "projectscope"}

fullproject/src_dev2/merge_comparepncb.go

@@ -1,160 +0,0 @@
-package main
-
-import (
-	"strings"
-)
-
-func comparePNCB(info *Info, compareProject *ProjectInfo) (compareStr string, score int) {
-	if info.ProjectName != "" {
-		pns := []string{}
-		if compareProject.ProjectName != "" {
-			pns = append(pns, compareProject.ProjectName)
-		}
-		if len(compareProject.MPN) > 0 {
-			pns = append(pns, compareProject.MPN...)
-		}
-		ifind := 0
-		for _, v := range pns {
-			if info.ProjectName == v {
-				ifind = 1
-				break
-			} else {
-				//if strings.Contains(info.ProjectName, v) || strings.Contains(v, info.ProjectName) ||
-				retv := CheckContain(info.ProjectName, v, info.Buyer)
-				if retv == 1 {
-					ifind = 1
-					break
-				} else if retv == 2 {
-					ifind = 2
-				} else if ifind == 0 {
-					ifind = 3
-				}
-			}
-		}
-		switch ifind {
-		case 0:
-			compareStr = "D"
-		case 1:
-			compareStr = "A"
-			score += 4
-			if len([]rune(info.ProjectName)) > 18 {
-				score += 2
-			}
-		case 2:
-			compareStr = "B"
-			score += 2
-			if len([]rune(info.ProjectName)) > 18 {
-				score += 1
-			}
-		case 3:
-			compareStr = "C"
-		}
-	} else {
-		compareStr = "D"
-	}
-
-	/*
-			项目编号 - －()（） 要注意
-			init_text = ["号","(重)","(第二次)","（重）"]
-	all_clean_mark = ["[","(","【","（","〖","]",")","】","）","〗","-","〔","〕","《","［","］","｛","｝","{","—"," ","－","﹝","﹞","–"]
-	*/
-	for _, pc := range []string{info.ProjectCode, info.PTC} {
-		if pc != "" {
-			pcs := []string{}
-			if compareProject.ProjectCode != "" {
-				pcs = append(pcs, compareProject.ProjectCode)
-			}
-			if len(compareProject.MPC) > 0 {
-				pcs = append(pcs, compareProject.MPC...)
-			}
-			ifind := 0
-			for _, v := range pcs {
-				if pc == v {
-					ifind = 1
-					break
-				} else {
-					// math.Abs(float64(len([]rune(pc))-len([]rune(v)))) < 6
-					//if !_numreg1.MatchString(pc) && !_zimureg1.MatchString(pc) && !_numreg1.MatchString(v) && !_zimureg1.MatchString(v)
-					if strings.Contains(pc, v) || strings.Contains(v, pc) {
-						t1 := pc
-						t2 := v
-						if len(v) > len(pc) {
-							t1 = v
-							t2 = pc
-						}
-						t3 := strings.Replace(t1, t2, "", -1)
-						t3 = _datereg.ReplaceAllString(t3, "")
-						if t3 == "" {
-							ifind = 1
-							break
-						} else {
-							ifind = 2
-						}
-					} else if ifind == 0 {
-						ifind = 3
-					}
-				}
-			}
-			switch ifind {
-			case 0:
-				compareStr += "D"
-			case 1:
-				compareStr += "A"
-				score += 4
-				if len([]rune(pc)) > 18 {
-					score += 2
-				}
-			case 2:
-				compareStr += "B"
-				score += 2
-				if len([]rune(pc)) > 18 {
-					score += 1
-				}
-			case 3:
-				compareStr += "C"
-			}
-
-		} else {
-			compareStr += "D"
-		}
-	}
-	return
-}
-
-func CheckContain(b1, b2, infoBuyer string) (res int) {
-	b1 = replaceStr.ReplaceAllString(b1, "")
-	b2 = replaceStr.ReplaceAllString(b2, "")
-	b1 = strings.Replace(b1, infoBuyer, "", 1)
-	b2 = strings.Replace(b2, infoBuyer, "", 1)
-
-	if b1 == b2 {
-		res = 1 //相等
-		return
-	}
-	bs1 := []rune(b1)
-	bs2 := []rune(b2)
-	tmp := ""
-	for i := 0; i < len(bs1); i++ {
-		for j := 0; j < len(bs2); j++ {
-			if bs1[i] == bs2[j] {
-				tmp += string(bs1[i])
-			} else if tmp != "" {
-				b1 = strings.Replace(b1, tmp, "", -1)
-				b2 = strings.Replace(b2, tmp, "", -1)
-				tmp = ""
-			}
-		}
-	}
-	if tmp != "" {
-		b1 = strings.Replace(b1, tmp, "", -1)
-		b2 = strings.Replace(b2, tmp, "", -1)
-	}
-	if b1 == b2 {
-		res = 1 //相等
-	} else if b1 == "" || b2 == "" {
-		res = 2 //包含
-	} else {
-		res = 3 //不相等
-	}
-	return
-}

fullproject/src_dev2/merge_select.go

@@ -1,276 +0,0 @@
-package main
-
-func Select(compareStr string, info *Info, compareInfo *ProjectInfo) (res int) {
-	//没有可对比的项目名称、或项目编号
-	if compareNoPass[compareStr] {
-
-	} else {
-		switch compareStr {
-		case "AAA":
-			res = 3
-		case "AAB":
-			res = 3
-		case "ABA":
-			res = 3
-		case "ABB":
-			if info.LenPTC > 6 || info.LenPC > 6 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BAA":
-			if info.LenPN > 10 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BAB":
-			if info.LenPN > 10 || info.LenPTC > 6 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BBA":
-			if info.LenPN > 10 || info.LenPC > 6 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BBB":
-			v := 0
-			if info.LenPN > 12 {
-				v++
-			}
-			if info.LenPC > 8 {
-				v++
-			}
-			if info.LenPTC > 8 {
-				v++
-			}
-			if v > 1 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "AAD":
-			res = 3
-		case "ABD":
-			//			if info.LenPC > 12 || info.LenPN > 16 {
-			//				res = 3
-			//			} else {
-			//				res = 2
-			//			}
-			res = 2
-		case "BAD":
-			//			if info.LenPC > 12 || info.LenPN > 16 {
-			//				res = 3
-			//			} else {
-			//				res = 2
-			//			}
-			res = 2
-		case "BBD":
-			if info.LenPC > 12 && info.LenPN > 16 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "ADA":
-			res = 3
-		case "ADB":
-			//			if info.LenPTC > 12 || info.LenPN > 16 {
-			//				res = 3
-			//			} else {
-			//				res = 2
-			//			}
-			res = 2
-		case "BDA":
-			//			if info.LenPTC > 12 || info.LenPN > 16 {
-			//				res = 3
-			//			} else {
-			//				res = 2
-			//			}
-			res = 2
-		case "BDB":
-			if info.LenPTC > 12 && info.LenPN > 16 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "ADD":
-			if info.LenPN > 14 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BDD":
-			//			if info.LenPN > 18 {
-			//				res = 2
-			//			} else {
-			//				res = 1
-			//			}
-			res = 1
-		case "DAA":
-			if info.LenPTC > 12 || info.LenPC > 12 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "DAB":
-			if info.LenPTC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DAD":
-			if info.LenPTC > 14 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DBA":
-			if info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DBB":
-			if info.LenPTC > 12 && info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DBD":
-			if info.LenPC > 8 {
-				res = 1
-			}
-		case "DDA":
-			if info.LenPTC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DDB":
-			if info.LenPC > 8 {
-				res = 1
-			}
-		case "ACA":
-			if info.LenPN > 12 || info.LenPTC > 9 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "ACB":
-			if info.LenPN > 14 && info.LenPTC > 10 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BCA":
-			if info.LenPN > 12 && info.LenPTC > 9 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BCB":
-			if info.LenPN > 12 && info.LenPTC > 9 {
-				res = 2
-			} else if info.LenPN > 16 || info.LenPTC > 12 {
-				res = 1
-			}
-		case "AAC":
-			if info.LenPN > 12 || info.LenPC > 9 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "ABC":
-			if info.LenPN > 14 || info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BAC":
-			if info.LenPN > 14 || info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BBC":
-			//			if info.LenPN > 12 && info.LenPTC > 9 {
-			//				res = 1
-			//			} else {
-			//				res = 1
-			//			}
-			res = 1
-		case "ACC":
-			//			if info.LenPC > 4 && len(compareInfo.ProjectCode) > 4 && CheckHanAndNum(info.ProjectCode) && CheckHanAndNum(compareInfo.ProjectCode) {
-			//				//未考虑MPC
-			//			} else if info.LenPN > 16 {
-			//				res = 1
-			//			}
-			//
-			res = 1
-		case "ACD":
-			//项目编号不一致
-			res = 1
-		case "ADC":
-			res = 1
-		case "BCC":
-			//项目编号不一致
-			res = 1
-		case "BCD":
-			//项目编号不一致
-			if info.LenPC > 4 && len(compareInfo.ProjectCode) > 4 && (!_numreg1.MatchString(info.ProjectCode) && !_numreg1.MatchString(compareInfo.ProjectCode)) {
-				//未考虑MPC
-			} else if info.LenPN > 18 && info.LenPC-len([]rune(compareInfo.ProjectCode)) != 0 {
-				res = 1
-			}
-		case "BDC":
-			if info.LenPN > 18 && info.LenPTC-len([]rune(compareInfo.ProjectCode)) != 0 {
-				res = 1
-			}
-		case "CAA":
-			if info.LenPC > 10 || info.LenPTC > 10 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "CAB":
-			if info.LenPC > 10 && info.LenPTC > 10 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "CAC":
-			res = 1
-		case "CAD":
-			res = 1
-		case "CBA":
-			res = 1
-		case "CBB":
-			res = 1
-		case "CBC":
-			res = 1
-		case "CBD":
-			res = 1
-		case "CCA":
-			res = 1
-		case "CCB":
-			//
-		case "CDA":
-			res = 1
-		case "CDB":
-			res = 1
-		case "DAC":
-			res = 1
-		case "DBC":
-			res = 1
-		case "DCA":
-			res = 1
-		case "DCB":
-			res = 1
-		}
-
-	}
-	return
-}

fullproject/src_dev2/new_project.go

@@ -1,116 +0,0 @@
-package main
-
-import (
-	qu "qfw/util"
-	"qfw/util/redis"
-	"strings"
-	"time"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-func NewProject(tmp map[string]interface{}, thisinfo *Info) string {
-
-	set := map[string]interface{}{}
-	for _, f := range FIELDS {
-		if tmp[f] != nil {
-			set[f] = tmp[f]
-		}
-	}
-	set["s_projectname"] = thisinfo.ProjectName
-	set["createtime"] = time.Now().Unix()
-	set["sourceinfoid"] = qu.BsonIdToSId(tmp["_id"])
-	set["sourceinfourl"] = tmp["href"]
-	set["topscopeclass"] = thisinfo.Topscopeclass
-	set["subscopeclass"] = thisinfo.Subscopeclass
-	if thisinfo.Buyerperson != "" {
-		set["buyerperson"] = thisinfo.Buyerperson
-	}
-	if thisinfo.Buyertel != "" {
-		set["buyertel"] = thisinfo.Buyertel
-	}
-	if thisinfo.Buyerclass != "" {
-		set["buyertel"] = thisinfo.Buyerclass
-	}
-	if thisinfo.Bidopentime > 0 {
-		set["bidopentime"] = thisinfo.Bidopentime
-	}
-	if len(thisinfo.Winnerorder) > 0 {
-		set["winnerorder"] = thisinfo.Winnerorder
-	}
-	s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
-	set["s_subscopeclass"] = s_subscopeclass
-	s_winner := strings.Join(thisinfo.Winners, ",")
-	set["s_winner"] = s_winner
-	if tmp["package"] != nil {
-		set["package"] = tmp["package"] //没定义优先级
-	}
-	push := NewPushInfo(tmp)
-	set["list"] = []bson.M{
-		push,
-	}
-
-	id := MongoTool.Save(ProjectColl, set)
-	p1 := NewPinfo(id, thisinfo)
-	redis.PutCKV(ProjectCache, id, p1)
-
-	return id
-}
-
-func NewPushInfo(tmp map[string]interface{}) bson.M {
-	res := bson.M{
-		"comeintime":  tmp["comeintime"],
-		"publishtime": tmp["publishtime"],
-		"title":       tmp["title"],
-		"toptype":     tmp["toptype"],
-		"subtype":     tmp["subtype"],
-		"infoformat":  tmp["infoformat"],
-		"infoid":      qu.BsonIdToSId(tmp["_id"]),
-		"area":        tmp["area"],
-		"city":        tmp["city"],
-		"projectname": tmp["projectname"],
-		"projectcode": tmp["projectcode"],
-		"buyer":       tmp["buyer"],
-		"href":        tmp["href"],
-	}
-	for _, k := range []string{"winner", "budget", "bidamount"} {
-		if tmp[k] != nil {
-			res[k] = tmp[k]
-		}
-	}
-	return res
-}
-
-//生成存放在redis数组中的对象
-func NewPinfo(id string, thisinfo *Info) ProjectInfo {
-	p1 := ProjectInfo{
-		Id:            bson.ObjectIdHex(id),
-		Ids:           []string{thisinfo.Id},
-		Topscopeclass: thisinfo.Topscopeclass,
-		Subscopeclass: thisinfo.Subscopeclass,
-		Winners:       thisinfo.Winners,
-		ProjectName:   thisinfo.ProjectName,
-		ProjectCode:   thisinfo.ProjectCode,
-		Buyer:         thisinfo.Buyer,
-		Agency:        thisinfo.Agency,
-		Area:          thisinfo.Area,
-		City:          thisinfo.City,
-		District:      thisinfo.District,
-		MPN:           []string{},
-		MPC:           []string{},
-		HasPackage:    thisinfo.HasPackage,
-		Package:       map[string]interface{}{},
-		Buyerclass:    thisinfo.Buyerclass,
-		Bidopentime:   thisinfo.Bidopentime,
-		Winnerorder:   thisinfo.Winnerorder,
-		FirstTime:     thisinfo.Publishtime,
-		LastTime:      thisinfo.Publishtime,
-	}
-	if thisinfo.LenPTC > 5 {
-		p1.MPC = append(p1.MPC, thisinfo.PTC)
-	}
-	for k4, _ := range thisinfo.Package {
-		p1.Package[k4] = ""
-	}
-	return p1
-}

fullproject/src_dev2/task.go

@@ -1,200 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"log"
-	"qfw/util"
-	"regexp"
-	//"strings"
-	"time"
-)
-
-const (
-	InitMinTime = int64(1325347200) //最小时间位置2012
-)
-
-//全量合并
-func taskQl(udpInfo map[string]interface{}) {
-	defer util.Catch()
-	//1、检查pubilshtime索引
-	db, _ := udpInfo["db"].(string)
-	if db == "" {
-		db = MongoTool.DbName
-	}
-	coll, _ := udpInfo["coll"].(string)
-	if coll == "" {
-		coll = ExtractColl
-	}
-	sess := MongoTool.GetMgoConn()
-	bcon := false
-	if sess.DB(db).C(coll).EnsureIndexKey("publishtime_1", "publishtime_-1") == nil {
-		bcon = true
-	} else {
-		log.Println("publishtime_1索引不存在")
-	}
-	MongoTool.DestoryMongoConn(sess)
-	thread := util.IntAllDef(udpInfo["thread"], 1)
-	if bcon {
-		//获取起始时间
-		startTime, END := int64(0), int64(0)
-
-		sts, bres := MongoTool.Find(ExtractColl, `{}`, "publishtime", `{"publishtime":1}`, true, 0, 1)
-		if bres && sts != nil && len(*sts) == 1 {
-			startTime = util.Int64All((*sts)[0]["publishtime"])
-			sts, bres = MongoTool.Find(ExtractColl, `{}`, "-publishtime", `{"publishtime":1}`, true, 0, 1)
-			if bres && sts != nil && len(*sts) == 1 {
-				END = util.Int64All((*sts)[0]["publishtime"])
-			}
-			log.Println("查询到的起始时间", startTime, END)
-		} else {
-			return
-		}
-		startTime -= 1
-		sum := 0
-		if startTime < InitMinTime {
-			q := map[string]interface{}{
-				"publishtime": map[string]interface{}{
-					"$gt":  startTime,
-					"$lte": InitMinTime,
-				},
-			}
-			sum = Mql(q, thread, db, coll, sum)
-			startTime = InitMinTime
-		}
-		for {
-			if startTime >= END {
-				break
-			}
-			et := startTime + 50*86400
-			if et >= END {
-				et = END
-			}
-			q := map[string]interface{}{
-				"publishtime": map[string]interface{}{
-					"$gt":  startTime,
-					"$lte": et,
-				},
-			}
-			sum = Mql(q, thread, db, coll, sum)
-			startTime = et
-			time.Sleep(1 * time.Second)
-		}
-	}
-	log.Println("task over!!!")
-}
-
-func Mql(q map[string]interface{}, thread int, db, coll string, sum int) int {
-	defer util.Catch()
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
-	query := sess.DB(db).C(coll).Find(q).Sort("publishtime").Iter()
-	pool := make(chan bool, thread)
-	count := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
-		info := ParseInfo(tmp)
-		if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
-			pool <- true
-			go func(info *Info, tmp map[string]interface{}) {
-				defer func() {
-					<-pool
-				}()
-				startProjectMerge(info, tmp)
-			}(info, tmp)
-		} else {
-			//log.Println("info err:", tmp["_id"], tmp["title"], tmp["buyer"])
-		}
-		if sum%1000 == 0 {
-			log.Println("current", sum)
-		}
-		sum++
-		tmp = make(map[string]interface{})
-	}
-	//阻塞
-	for n := 0; n < thread; n++ {
-		pool <- true
-	}
-	//完成
-	log.Println("sontask over:", count, sum, q)
-	return sum
-}
-
-var (
-	titleGetPc  = regexp.MustCompile("^([-0-9a-zA-Z第号采招政询电审竞#]{8,}[-0-9a-zA-Z#]+)")
-	titleGetPc1 = regexp.MustCompile("[\\[【(（](.{0,6}(编号|编码|项号|包号|代码|标段?号)[:：为])?([-0-9a-zA-Z第号采招政询电审竞#]{5,}([\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+[\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+)?)[\\]】)）]")
-	titleGetPc2 = regexp.MustCompile("([-0-9a-zA-Z第号采政招询电审竞#]{8,}[-0-9a-zA-Z#]+)(.{0,5}公告)?$")
-	pcReplace   = regexp.MustCompile("([\\[【(（〖〔《｛﹝{](重|第?[二三四再]次.{0,4})[\\]】)）〗〕》｝﹞}])$|[\\[\\]【】()（）〖〗〔〕《》｛｝﹝﹞－{}– 　]+|(号|重|第?[二三四五再]次(招标)?)$")
-)
-
-func ParseInfo(tmp map[string]interface{}) (info *Info) {
-	bys, _ := json.Marshal(tmp)
-	var thisinfo *Info
-	json.Unmarshal(bys, &thisinfo)
-	if thisinfo == nil {
-		return nil
-	}
-	if len(thisinfo.Topscopeclass) == 0 {
-		thisinfo.Topscopeclass = []string{}
-	}
-	if len(thisinfo.Subscopeclass) == 0 {
-		thisinfo.Subscopeclass = []string{}
-	}
-
-	res := titleGetPc.FindStringSubmatch(thisinfo.Title)
-	if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
-		thisinfo.PTC = res[1]
-		thisinfo.pnbval++
-	} else {
-		res = titleGetPc1.FindStringSubmatch(thisinfo.Title)
-		if len(res) > 3 && len(res[3]) > 6 && thisinfo.ProjectCode != res[3] && !numCheckPc.MatchString(res[3]) && !_zimureg1.MatchString(res[3]) {
-			thisinfo.PTC = res[3]
-			thisinfo.pnbval++
-		} else {
-			res = titleGetPc2.FindStringSubmatch(thisinfo.Title)
-			if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
-				thisinfo.PTC = res[1]
-				thisinfo.pnbval++
-			}
-		}
-	}
-
-	if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
-		//		thisinfo.ProjectName = strings.Replace(thisinfo.ProjectName, "（", "(", -1)
-		//		thisinfo.ProjectName = strings.Replace(thisinfo.ProjectName, "）", ")", -1)
-		//		thisinfo.ProjectName = strings.Replace(thisinfo.ProjectName, "－", "", -1)
-		thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
-		if thisinfo.ProjectName != "" {
-			thisinfo.pnbval++
-		}
-	}
-
-	if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
-		if thisinfo.ProjectCode != "" {
-			thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
-			//			thisinfo.ProjectCode = strings.Replace(thisinfo.ProjectCode, "（", "(", -1)
-			//			thisinfo.ProjectCode = strings.Replace(thisinfo.ProjectCode, "）", ")", -1)
-			//			thisinfo.ProjectCode = strings.Replace(thisinfo.ProjectCode, "－", "", -1)
-		} else {
-			thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
-			//			thisinfo.PTC = strings.Replace(thisinfo.PTC, "（", "(", -1)
-			//			thisinfo.PTC = strings.Replace(thisinfo.PTC, "）", ")", -1)
-			//			thisinfo.PTC = strings.Replace(thisinfo.PTC, "－", "", -1)
-		}
-		if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
-			thisinfo.pnbval++
-		}
-	}
-	if thisinfo.ProjectCode == thisinfo.PTC {
-		thisinfo.PTC = ""
-	}
-
-	if thisinfo.Buyer != "" && len([]rune(thisinfo.Buyer)) > 2 {
-		thisinfo.pnbval++
-	} else {
-		thisinfo.Buyer = ""
-	}
-
-	thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
-	thisinfo.LenPTC = len([]rune(thisinfo.PTC))
-	thisinfo.LenPN = len([]rune(thisinfo.ProjectName))
-	return thisinfo
-}

fullproject/src_dev2/update_project.go

@@ -1,221 +0,0 @@
-package main
-
-import (
-	qu "qfw/util"
-	"sort"
-	"strings"
-)
-
-func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string) {
-	//log.Println(thisinfo.ProjectName, pInfo.ProjectName, comStr)
-	//	updateid := pInfo.Id
-	if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
-		return //updateid
-	}
-	set := map[string]interface{}{}
-	res, bres := MongoTool.FindById(ProjectColl, pInfo.Id.Hex(), `{"list":0}`)
-	EqInfoUpdate(thisinfo, pInfo)
-	if bres && res != nil && *res != nil {
-		set["topscopeclass"] = pInfo.Topscopeclass
-		set["subscopeclass"] = pInfo.Subscopeclass
-		s_subscopeclass := strings.Join(pInfo.Subscopeclass, ",")
-		if len(s_subscopeclass) > 0 {
-			s_subscopeclass = "," + s_subscopeclass + ","
-		}
-		set["s_subscopeclass"] = s_subscopeclass
-		s_winner := strings.Join(pInfo.Winners, ",")
-		if len(s_winner) > 0 {
-			s_winner = "," + s_winner + ","
-		}
-		set["s_winner"] = s_winner
-		if pInfo.Buyerperson != "" && pInfo.Buyertel != "" {
-			set["buyerperson"] = pInfo.Buyerperson
-			set["buyertel"] = pInfo.Buyertel
-		}
-		if pInfo.Buyerclass != "" {
-			set["buyerclass"] = pInfo.Buyerclass
-		}
-		if pInfo.District != "" {
-			set["district"] = pInfo.District
-		}
-		if pInfo.Bidopentime > 0 {
-			set["bidopentime"] = pInfo.Bidopentime
-		}
-		if len(pInfo.Winnerorder) > 0 {
-			set["winnerorder"] = pInfo.Winnerorder
-		}
-		if thisinfo.HasPackage {
-			set["multipackage"] = 1
-		} else {
-			set["multipackage"] = 0
-		}
-
-		if pInfo.ProjectName != "" {
-			set["projectname"] = pInfo.ProjectName
-		}
-
-		if pInfo.ProjectCode != "" {
-			set["projectcode"] = pInfo.ProjectCode
-		}
-
-		if pInfo.Buyer != "" {
-			set["buyer"] = pInfo.Buyer
-		}
-
-		//预算、中标价
-		if qu.Float64All(tmp["bidamount"]) > 0 {
-			if qu.Float64All((*res)["bidamount"]) == 0 {
-				set["bidamount"] = tmp["bidamount"]
-			}
-		}
-
-		if qu.Float64All(tmp["budget"]) > 0 {
-			if qu.Float64All((*res)["budget"]) == 0 {
-				set["budget"] = tmp["budget"]
-			}
-		}
-
-		set["mpn"] = pInfo.MPN
-		set["mpc"] = pInfo.MPC
-		set["area"] = pInfo.Area
-		set["city"] = pInfo.City
-
-		//e := InitEL(util.ObjToString((*res)["extractpos"]))
-		if thisinfo.dealtype == 1 {
-			var sonpackage map[string]interface{}
-			for _, obj := range tmp["package"].(map[string]interface{}) {
-				sonpackage, _ = obj.(map[string]interface{})
-			}
-			for _, v2 := range []string{"budget", "budget_w", "winner", "winner_w", "bidstatus", "bidstatus_w"} {
-				if sonpackage[v2] != nil {
-					tmp[v2] = sonpackage[v2]
-				}
-			}
-		}
-		//e.fieldpriority(&tmp, res, &set)
-		//set["extractpos"] = e.GetVal()
-		if thisinfo.HasPackage { //多包处理
-			p1, _ := (*res)["package"].(map[string]interface{})
-			p2, _ := tmp["package"].(map[string]interface{})
-			if p2 != nil {
-				if p1 != nil {
-					for pk2, pv2 := range p2 {
-						if p1[pk2] != nil { //合并
-							item1, _ := p1[pk2].(map[string]interface{})
-							item2, _ := pv2.(map[string]interface{})
-							if item1 != nil && item2 != nil { //原始项
-								for ik1, iv1 := range item2 {
-									if item1[ik1] == nil {
-										item1[ik1] = iv1
-									}
-								}
-							}
-						} else {
-							p1[pk2] = pv2
-						}
-					}
-				} else {
-					p1 = p2
-				}
-			}
-			set["package"] = p1
-		}
-		//中标候选人合并
-
-		update := map[string]interface{}{}
-		if len(set) > 0 {
-			update["$set"] = set
-		}
-		//保留原数据吧
-		push := NewPushInfo(tmp)
-		push["compareStr"] = comStr
-		update["$push"] = map[string]interface{}{
-			"list": push,
-		}
-		if len(update) > 0 {
-			MongoTool.Update(ProjectColl, map[string]interface{}{
-				"_id": qu.StringTOBsonId(pInfo.Id.Hex()),
-			}, &update, false, false)
-		}
-	}
-	//再往redis中放 index
-	//往队列中增加时间 -------------->start
-}
-
-func EqInfoUpdate(thisinfo *Info, pInfo *ProjectInfo) {
-	if thisinfo.Publishtime > pInfo.LastTime {
-		pInfo.LastTime = thisinfo.Publishtime
-	}
-	if pInfo.FirstTime == 0 || (thisinfo.Publishtime < pInfo.FirstTime && thisinfo.Publishtime > 0) {
-		pInfo.FirstTime = thisinfo.Publishtime
-	}
-	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
-	//增加发布时间结束----------------->end
-
-	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
-		pInfo.Buyer = thisinfo.Buyer
-	}
-	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
-		pInfo.Agency = thisinfo.Agency
-	}
-	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
-		pInfo.ProjectCode = thisinfo.ProjectCode
-	}
-
-	if pInfo.Area == "全国" && thisinfo.Area != "全国" {
-		pInfo.Area = thisinfo.Area
-		pInfo.City = thisinfo.City
-	}
-	if thisinfo.Buyerperson != "" && thisinfo.Buyertel != "" && len([]rune(thisinfo.Buyertel)) > 6 {
-		pInfo.Buyerperson = thisinfo.Buyerperson
-		pInfo.Buyertel = thisinfo.Buyertel
-	}
-	if thisinfo.Buyerclass != "" {
-		pInfo.Buyerclass = thisinfo.Buyerclass
-	}
-	if thisinfo.District != "" {
-		pInfo.District = thisinfo.District
-	}
-	if thisinfo.Bidopentime > 0 {
-		pInfo.Bidopentime = thisinfo.Bidopentime
-	}
-	if len(thisinfo.Topscopeclass) > 0 {
-		sort.Strings(pInfo.Topscopeclass)
-		for _, k := range thisinfo.Topscopeclass {
-			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
-				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
-				sort.Strings(pInfo.Topscopeclass)
-			}
-		}
-	}
-
-	if len(thisinfo.Subscopeclass) > 0 {
-		sort.Strings(pInfo.Subscopeclass)
-		for _, k := range thisinfo.Subscopeclass {
-			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
-				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
-				sort.Strings(pInfo.Subscopeclass)
-			}
-		}
-	}
-	//winner
-	if len(thisinfo.Winners) > 0 {
-		sort.Strings(pInfo.Winners)
-		for _, k := range thisinfo.Winners {
-			if BinarySearch(pInfo.Winners, k) == -1 {
-				pInfo.Winners = append(pInfo.Winners, k)
-				sort.Strings(pInfo.Winners)
-			}
-		}
-	}
-	//winnerorder
-	if len(thisinfo.Winnerorder) > 0 {
-		sort.Strings(pInfo.Winnerorder)
-		for _, k := range thisinfo.Winnerorder {
-			if BinarySearch(pInfo.Winnerorder, k) == -1 {
-				pInfo.Winnerorder = append(pInfo.Winnerorder, k)
-				sort.Strings(pInfo.Winnerorder)
-			}
-		}
-	}
-}

fullproject/src_dev3/clearmem.go

@@ -1,87 +0,0 @@
-package main
-
-import (
-	"log"
-
-	"github.com/robfig/cron"
-)
-
-//定时清理内存
-//当前位置时间，小于此时间超过6个月，则清理
-var currentTime = int64(0)
-var validTime = int64(6 * 30 * 86400)
-
-func clearMem() {
-	c := cron.New()
-	c.AddFunc("50 0/1 * * * *", func() {
-		findLock.Lock()
-		defer findLock.Unlock()
-		wg.Wait()
-		//遍历id
-		AllIdsMapLock.Lock()
-		defer AllIdsMapLock.Unlock()
-		clearNum := 0
-		for k, v := range AllIdsMap2 {
-			if currentTime-v.lastTime > validTime {
-				clearNum++
-				//删除id的map
-				delete(AllIdsMap2, k)
-				//删除pb
-				if v.P.Buyer != "" {
-					ids := mapPb[v.P.Buyer]
-					if ids != nil {
-						ids.Lock.Lock()
-						ids.Arr = deleteSlice(ids.Arr, k)
-						if len(ids.Arr) == 0 {
-							delete(mapPb, v.P.Buyer)
-						}
-						ids.Lock.Unlock()
-					}
-				}
-				//删除mapPn
-				for _, vn := range append([]string{v.P.ProjectName}, v.P.MPN...) {
-					if vn != "" {
-						ids := mapPn[vn]
-						if ids != nil {
-							ids.Lock.Lock()
-							ids.Arr = deleteSlice(ids.Arr, k)
-							if len(ids.Arr) == 0 {
-								delete(mapPn, vn)
-							}
-							ids.Lock.Unlock()
-						}
-					}
-				}
-				//删除mapPc
-				for _, vn := range append([]string{v.P.ProjectCode}, v.P.MPC...) {
-					if vn != "" {
-						ids := mapPc[vn]
-						if ids != nil {
-							ids.Lock.Lock()
-							ids.Arr = deleteSlice(ids.Arr, k)
-							if len(ids.Arr) == 0 {
-								delete(mapPc, vn)
-							}
-							ids.Lock.Unlock()
-						}
-					}
-				}
-				v = nil
-			}
-		}
-		log.Println("清除完成:", clearNum, len(AllIdsMap2))
-	})
-	c.Start()
-	defer c.Stop()
-	select {}
-}
-
-func deleteSlice(arr []string, v string) []string {
-	j := 0
-	for _, v1 := range arr {
-		if v1 != v {
-			arr[j] = v1
-		}
-	}
-	return arr[:j]
-}

fullproject/src_dev3/config.json

@@ -1,16 +0,0 @@
-{
-    "thread": 1,
-    "mongodbServers": "192.168.3.207:27082",
-    "mongodbPoolSize": 10,
-    "mongodbName": "cesuo",
-    "extractColl": "key1_biddingall",
-    "projectColl": "projectset_0809",
-    "redisaddrs": "ids=192.168.3.207:1378,keys=192.168.3.207:1378,info=192.168.3.207:1378",
-    "redisPoolSize": 20,
-    "jkmail": {
-        "to": "zhangjinkun@topnet.net.cn",
-        "api": "http://10.171.112.160:19281/_send/_mail"
-    },
-    "udpport": ":1482",
-    "nextNode": []
-}

fullproject/src_dev3/init.go

@@ -1,290 +0,0 @@
-package main
-
-import (
-	"log"
-	mu "mfw/util"
-	"qfw/util"
-	"qfw/util/mongodb"
-	"regexp"
-	"sync"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-const (
-	ProjectCache = "info" //存放每条项目信息，key为项目ID
-)
-
-var (
-	Sysconfig                map[string]interface{} //读取配置文件
-	MongoTool                mongodb.MongodbSim     //mongodb连接
-	ExtractColl, ProjectColl string                 //抽取表、项目表
-	CurrentMegerTime         int64                  //当前合并到的信息的时间
-	CurrentMegerCount        int                    //当前合并计数
-	MultiThread              = make(chan bool, 5)   //项目合并线程
-	AllIdsMap2               = map[string]*ID{}
-	AllIdsMapLock            = sync.Mutex{}
-)
-
-var (
-	_datereg   = regexp.MustCompile("20[0-2][0-9][年-][0-9]{1,2}[月-][0-9]{1,2}[日-]([0-9]{1,2}时[0-9]{0,2})?")
-	_numreg1   = regexp.MustCompile("^[0-9-]{1,8}$")
-	_zimureg1  = regexp.MustCompile("^[a-zA-Z-]{1,7}$")
-	_nzreg     = regexp.MustCompile("^[0-9a-zA-Z-]+$")
-	_hanreg    = regexp.MustCompile(`^[\p{Han}:：【】\\[\\]（）()－-、]+$`)
-	replaceStr = regexp.MustCompile("(工程|采购|项目|[?！、【】（）—()－-]|栏标价|中标候选人|招标代理)")
-	pStr       = regexp.MustCompile("(勘察|监理|施工|设计|验收|标段|分包|子包|[0-9A-Z]包|[一二三四五六七八九十0-9]批)")
-	nreg1      = regexp.MustCompile("[0-9]{2,}")
-	zreg1      = regexp.MustCompile("[a-zA-Z]{1,}")
-	hreg1      = regexp.MustCompile(`[\p{Han}]+`)
-	numCheckPc = regexp.MustCompile("^[0-9-]{1,10}$")
-
-	//存放项目名称
-	mapPn = map[string]*Key{}
-	//存放项目编号
-	mapPc = map[string]*Key{}
-	//存放采购单位
-	mapPb         = map[string]*Key{}
-	compareNoPass = map[string]bool{}
-	compareAB     = map[string]bool{}
-	compareAB2D   = map[string]bool{}
-	compareABD    = map[string]bool{}
-	compareAB2CD  = map[string]bool{}
-	compareABCD   = map[string]bool{}
-)
-
-func init() {
-	util.ReadConfig(&Sysconfig)
-	MultiThread = make(chan bool, util.IntAllDef(Sysconfig["thread"], 5))
-	MongoTool = mongodb.MongodbSim{
-		MongodbAddr: Sysconfig["mongodbServers"].(string),
-		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
-		DbName:      Sysconfig["mongodbName"].(string),
-	}
-	MongoTool.InitPool()
-	ExtractColl = Sysconfig["extractColl"].(string)
-	ProjectColl = Sysconfig["projectColl"].(string)
-
-	udpport, _ := Sysconfig["udpport"].(string)
-	udpclient = mu.UdpClient{Local: udpport, BufSize: 1024}
-	udpclient.Listen(processUdpMsg)
-	log.Println("Udp服务监听", udpport)
-
-	//---不能通过
-	vm := []string{"C", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				compareNoPass[key] = true
-				//fmt.Println(key)
-			}
-		}
-	}
-	//fmt.Println("-------------------")
-
-	//三个元素一致 [AB][AB][AB]，分值最高
-	vm = []string{"A", "B"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				compareAB[key] = true
-				//fmt.Println(key)
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB))
-	//---至少两个一致，其他可能不存在
-	//[AB][AB][ABD]
-	//[AB][ABD][AB]
-	vm = []string{"A", "B"}
-	vm2 := []string{"A", "B", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 3; k++ {
-				key := vm[i] + vm[j] + vm2[k]
-				if !compareAB[key] {
-					compareAB2D[key] = true
-					//fmt.Println(key)
-
-				}
-			}
-		}
-	}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 3; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm2[j] + vm[k]
-				if !compareAB[key] {
-					compareAB2D[key] = true
-					//fmt.Println(key)
-
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB2D))
-	//---至少一个一致，其他可能不存在
-	//[ABD][ABD][ABD] //已经删除DDD
-	vm = []string{"A", "B", "D"}
-	for i := 0; i < 3; i++ {
-		for j := 0; j < 3; j++ {
-			for k := 0; k < 3; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] {
-					compareABD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareABD))
-
-	//[AB][ABCD][AB]
-	//[AB][AB][ABCD]
-	vm = []string{"A", "B"}
-	vm2 = []string{"A", "B", "C", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 4; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm2[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-					compareAB2CD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 4; k++ {
-				key := vm[i] + vm[j] + vm2[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-					compareAB2CD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB2CD))
-	//[ABECD][ABECD][ABECD]  //已经删除[CD][CD][CD]   //这个要重点讨论
-	vm = []string{"A", "B", "C", "D"}
-	for i := 0; i < 4; i++ {
-		for j := 0; j < 4; j++ {
-			for k := 0; k < 4; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareABD[key] && !compareNoPass[key] && !compareAB2CD[key] {
-					compareABCD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-}
-
-func CheckHanAndNum(str string) (b bool) {
-	return nreg1.MatchString(str) && hreg1.MatchString(str)
-}
-func CheckZimuAndNum(str string) (b bool) {
-	return zreg1.MatchString(str) && nreg1.MatchString(str)
-}
-
-type KeyMap struct {
-	Lock sync.Mutex
-	Map  map[string]*Key
-}
-
-type ID struct {
-	Id       string
-	Lock     sync.Mutex
-	lastTime int64
-	pos      int
-	P        *ProjectInfo
-}
-type Key struct {
-	Arr  []string
-	Lock sync.Mutex
-}
-type IdAndLock struct {
-	Id   string
-	Lock sync.Mutex
-}
-
-func NewKeyMap() *KeyMap {
-	return &KeyMap{
-		Map:  map[string]*Key{},
-		Lock: sync.Mutex{},
-	}
-}
-
-//招标信息实体类
-type Info struct {
-	Id            string                 `json:"_id"`
-	Href          string                 `json:"href"` //源地址
-	Publishtime   int64                  `json:"publishtime"`
-	Title         string                 `json:"title"`
-	TopType       string                 `json:"toptype"`
-	SubType       string                 `json:"subtype"`
-	ProjectName   string                 `json:"projectname"`
-	ProjectCode   string                 `json:"projectcode"`
-	Buyer         string                 `json:"buyer"`
-	Buyerperson   string                 `json:"buyerperson"`
-	Buyertel      string                 `json:"buyertel"`
-	Agency        string                 `json:"agency"`
-	Area          string                 `json:"area"`
-	City          string                 `json:"city"`
-	District      string                 `json:"district"`
-	HasPackage    bool                   `json:"haspackage"`
-	Package       map[string]interface{} `json:"package"`
-	PNum          string                 `json:"pnum"`
-	Topscopeclass []string               `json:"topscopeclass"`
-	Subscopeclass []string               `json:"subscopeclass"`
-	Buyerclass    string                 `json:"buyerclass"`
-	Bidopentime   int64                  `json:"bidopentime"`
-	Budget        float64                `json:"budget"`
-	Bidamount     float64                `json:"bidamount"`
-	Winners       []string
-	dealtype      int
-
-	Winnerorder []string
-
-	PTC    string //从标题中抽的项目编号
-	pnbval int    //项目名称、编号、采购单位存在的个数
-	LenPC  int    //项目编号长度
-	LenPN  int    //项目名称长度
-	LenPTC int    //标题抽的项目编号长度
-}
-
-//项目实体类
-type ProjectInfo struct {
-	Id            bson.ObjectId          `bson:"_id"`
-	FirstTime     int64                  `json:"firsttime"` //项目的最早时间
-	LastTime      int64                  `json:"lasttime"`  //项目的最后时间
-	Ids           []string               `json:"ids"`
-	Topscopeclass []string               `json:"topscopeclass"`
-	Subscopeclass []string               `json:"subscopeclass"` //子行业分类
-	Winners       []string               `json:"winners"`       //中标人
-	ProjectName   string                 `json:"projectname"`   //项目名称
-	ProjectCode   string                 `json:"projectcode"`   //项目代码唯一（纯数字的权重低）
-	Buyer         string                 `json:"buyer"`         //采购单位唯一
-	MPN           []string               `json:"mpn"`           //合并后多余的项目名称
-	MPC           []string               `json:"mpc"`           //合并后多余的项目编号
-	Buyerperson   string                 `json:"buyerperson"`   //采购联系人
-	Buyertel      string                 `json:"buyertel"`      //采购联系人电话
-	Agency        string                 `json:"agency"`        //代理机构
-	Area          string                 `json:"area"`          //地区
-	City          string                 `json:"city"`          //地市
-	District      string                 `json:"district"`      //区县
-	HasPackage    bool                   `json:"haspackage"`    //是否有分包
-	Package       map[string]interface{} `json:"package"`       //分包的对比对象
-	Buyerclass    string                 `json:"buyerclass"`    //采购单位分类
-	Bidopentime   int64                  `json:"bidopentime"`   //开标时间
-	Zbtime        int64                  `json:"zbtime"`        //招标时间
-	Jgtime        int64                  `json:"jgtime"`        //结果中标时间
-	Bidamount     float64                `json:"bidamount"`     //中标金额
-	Budget        float64                `json:"budget"`        //预算
-	Winnerorder   []string               `json:"winnerorder"`   //中标候选人
-	score         int
-	comStr        string
-}

fullproject/src_dev3/load_data.go

@@ -1,65 +0,0 @@
-package main
-
-import (
-	"time"
-	//"encoding/json"
-	"log"
-)
-
-//初始加载数据，默认加载最近6个月的数据
-
-func loadData(projectColl string, month int, bCacheRedis bool) {
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
-
-	q := map[string]interface{}{}
-	it := sess.DB(MongoTool.DbName).C(projectColl).Find(&q).Iter()
-	AllIdsMapLock.Lock()
-	tmp := &ProjectInfo{}
-	n := 0
-	for it.Next(tmp) {
-		n++
-		if n%1000 == 0 {
-			log.Println("current", n, "\n", tmp.Id, tmp)
-			time.Sleep(2 * time.Second)
-		}
-		for _, v := range append([]string{tmp.ProjectName}, tmp.MPN...) {
-			if v != "" {
-				k := mapPn[v]
-				if k == nil {
-					k = &Key{Arr: []string{}}
-					mapPn[v] = k
-				}
-				k.Arr = append(k.Arr, tmp.Id.Hex())
-			}
-		}
-		for _, v := range append([]string{tmp.ProjectCode}, tmp.MPC...) {
-			if v != "" {
-				k := mapPc[v]
-				if k == nil {
-					k = &Key{Arr: []string{}}
-					mapPc[v] = k
-				}
-				k.Arr = append(k.Arr, tmp.Id.Hex())
-			}
-		}
-		if tmp.Buyer != "" {
-			k := mapPb[tmp.Buyer]
-			if k == nil {
-				k = &Key{Arr: []string{}}
-				mapPb[tmp.Buyer] = k
-			}
-			k.Arr = append(k.Arr, tmp.Id.Hex())
-		}
-
-		AllIdsMap2[tmp.Id.Hex()] = &ID{Id: tmp.Id.Hex(), lastTime: tmp.LastTime, P: tmp}
-
-		if bCacheRedis {
-			//存入redis
-
-		}
-
-	}
-	AllIdsMapLock.Unlock()
-	log.Println("load over")
-}

fullproject/src_dev3/main.go

@@ -1,61 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"log"
-	mu "mfw/util"
-	"net"
-	"time"
-)
-
-var (
-	udpclient    mu.UdpClient //udp对象
-	SingleThread = make(chan bool, 1)
-)
-
-func main() {
-
-	//udp跑增量  id段   zl
-	//udp跑全量			ql
-	//udp跑历史数据  信息id1,id2/或id段  ls
-	//udp强制合并  信息id1,id2,id3 [项目id] 不存在时新建  qzhb
-	//udp强制拆分  项目id,信息id1,id2          qzcf
-	//udp重新合并  信息id1,id2,id3             cxhb
-	//loadData("project_0809", 1, true)
-	time.Sleep(99999 * time.Hour)
-}
-
-func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
-	switch act {
-	case mu.OP_TYPE_DATA: //上个节点的数据
-		var mapInfo map[string]interface{}
-		err := json.Unmarshal(data, &mapInfo)
-		log.Println("err:", err, "mapInfo:", mapInfo)
-		if err != nil {
-			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
-		} else if mapInfo != nil {
-			key, _ := mapInfo["key"].(string)
-			if key == "" {
-				key = "udpok"
-			}
-			go udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
-			SingleThread <- true
-			tasktype, _ := mapInfo["stype"].(string)
-			log.Println("tasktype:", tasktype)
-			switch tasktype {
-			case "ql":
-				go func() {
-					defer func() {
-						<-SingleThread
-					}()
-					taskQl(mapInfo)
-				}()
-			}
-		}
-	case mu.OP_NOOP: //下个节点回应
-		ok := string(data)
-		if ok != "" {
-			log.Println("ok:", ok)
-		}
-	}
-}

fullproject/src_dev3/merge.go

@@ -1,486 +0,0 @@
-package main
-
-import (
-	"math"
-	qu "qfw/util"
-	"sort"
-	//"strconv"
-	"strings"
-	"sync"
-)
-
-//单线程控制查找
-var findLock = sync.Mutex{}
-
-//从对应map中获取对比的项目id
-func getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
-	findLock.Lock()
-	defer findLock.Unlock()
-	wg.Add(1)
-	res = []*Key{}
-	//是否查找到，并标识位置。-1代表未查找到。
-	bpn, bpc, bptc, bpb = -1, -1, -1, -1
-	if pn != "" {
-		ids := mapPn[pn]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPn[pn] = ids
-			bpn = 0
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-
-	if pc != "" {
-		ids := mapPc[pc]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPc[pc] = ids
-			bpc = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-
-	if ptc != "" {
-		ids := mapPc[ptc]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPc[ptc] = ids
-			bptc = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-
-	if pb != "" {
-		ids := mapPb[pb]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPb[pb] = ids
-			bpb = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-	repeatId := map[string]bool{}
-	idArr = []string{} //项目id
-	IDArr = []*ID{}    //项目信息
-	for _, m := range res {
-		for _, id := range m.Arr {
-			if !repeatId[id] {
-				repeatId[id] = true
-				//_, _ = strconv.ParseInt(id[0:8], 16, 64)
-				AllIdsMapLock.Lock()
-				Id := AllIdsMap2[id]
-				AllIdsMapLock.Unlock()
-				if Id != nil {
-					Id.Lock.Lock()
-					idArr = append(idArr, id)
-					IDArr = append(IDArr, Id)
-				}
-			}
-		}
-	}
-	return
-}
-
-//项目合并入口
-func startProjectMerge(info *Info, tmp map[string]interface{}) {
-	//只有或没有采购单位的无法合并
-	//bpn, bpc, bptc, bpb 是否查找到，并标识位置。-1代表未查找到。
-	//pids 是项目id数组集合
-	//IDArr，是单个项目ID对象集合
-	defer wg.Done()
-	bpn, bpc, bptc, bpb, pids, _, IDArr := getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer)
-	//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock
-
-	for _, m := range pids {
-		defer m.Lock.Unlock()
-	}
-	for _, id := range IDArr {
-		defer id.Lock.Unlock()
-	}
-
-	bFindProject := false
-	findPid := ""
-	//获取完id，进行计算
-	//定义两组
-	comRes1 := []*ProjectInfo{} //优先级最高的对比结果数组
-	comRes2 := []*ProjectInfo{} //优化级其次
-	comRes3 := []*ProjectInfo{}
-
-	for _, v := range IDArr {
-		comStr := ""
-		compareProject := v.P
-		//问题出地LastTime!!!!!
-		diffTime := math.Abs(float64(info.Publishtime - compareProject.LastTime))
-		if diffTime < 360*86400 {
-			//"A 相等 	B 被包含 	C 不相等	 	D不存在  E被包含
-			compareStr, score := comparePNCB(info, compareProject)
-
-			resVal := Select(compareStr, info, compareProject)
-			//---------------------------------------
-
-			if resVal > 0 {
-
-				compareBuyer := "D"
-				if info.Buyer != "" {
-					if info.Buyer == compareProject.Buyer {
-						compareBuyer = "A"
-						score += 3
-					} else if compareProject.Buyer != "" {
-						if strings.Contains(info.Buyer, compareProject.Buyer) || strings.Contains(compareProject.Buyer, info.Buyer) {
-							compareBuyer = "B"
-							score += 1
-						} else {
-							compareBuyer = "C"
-						}
-					}
-				}
-
-				//---------------------------------------
-
-				compareCity := ""
-				if info.Area != "全国" && info.Area != "" && info.Area == compareProject.Area {
-					compareCity += "A"
-					score += 2
-				} else if info.Area == "全国" || compareProject.Area == "全国" {
-					compareCity += "B"
-					score += 1
-				} else {
-					compareCity += "C"
-				}
-				if compareCity != "C" {
-					if info.City != "" && info.City == compareProject.City {
-						compareCity += "A"
-						score += 2
-					} else {
-						if info.Area == "全国" || compareProject.Area == "全国" {
-							compareCity += "B"
-						} else if info.City == compareCity {
-							compareCity += "B"
-						} else {
-							compareCity += "C"
-						}
-					}
-				} else {
-					compareCity += "C"
-				}
-				score2 := 0
-				if compareCity == "AA" {
-					if info.District != "" && info.District == compareProject.District {
-						score2 = 1
-					}
-				}
-
-				compareTime := "D"
-				//diffTime := math.Abs(float64(info.Publishtime - compareProject.Publistime[len(compareProject.Publistime)-1]))
-				if diffTime < 45*86400 {
-					compareTime = "A"
-					score += 2
-				} else if diffTime < 90*86400 {
-					compareTime = "B"
-					score += 1
-				}
-
-				compareAgency := "D"
-				if info.Agency != "" {
-					if info.Agency == compareProject.Agency {
-						compareAgency = "A"
-						score += 2
-						score2 += 1
-					} else if compareProject.Agency != "" {
-						if strings.Contains(info.Agency, compareProject.Agency) || strings.Contains(compareProject.Agency, info.Agency) {
-							compareAgency = "B"
-							score += 1
-							score2 += 1
-						} else {
-							compareAgency = "C"
-						}
-					}
-				}
-				compareBudget := "C"
-				if info.Budget > 0 && (info.Budget == compareProject.Budget || (compareProject.Bidamount > 0 && info.Budget > compareProject.Bidamount && (info.Budget-compareProject.Bidamount) < (0.1*info.Budget))) {
-					compareBudget = "A"
-					score += 1
-					score2 += 1
-				}
-				compareBidmount := "C"
-				if info.Bidamount > 0 && (info.Bidamount == compareProject.Bidamount || (compareProject.Budget > 0 && compareProject.Budget > info.Bidamount && (compareProject.Budget-info.Bidamount) < 0.1*compareProject.Budget)) {
-					compareBidmount = "A"
-					score += 1
-					score2 += 1
-				}
-
-				//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构
-
-				comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
-				compareProject.comStr = comStr
-				eqV := 0
-				switch resVal {
-				case 3:
-					if compareBuyer < "C" {
-						eqV = 1
-					} else if compareBuyer == "D" {
-						if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && score2 > 0 && compareTime == "A" {
-							eqV = 3
-						} else if compareTime != "D" && compareAgency != "C" && score2 > 1 {
-							eqV = 3
-						}
-					} else {
-						if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 1 {
-							eqV = 3
-						}
-					}
-
-				case 2:
-					if compareBuyer < "C" {
-						if compareTime != "D" && "AA" == compareCity && compareAgency != "C" {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
-							eqV = 3
-						}
-					} else if compareBuyer == "D" {
-						if "AA" == compareCity && compareTime != "D" && score2 > 1 {
-							eqV = 3
-						} else if score2 > 2 && compareTime == "A" {
-							eqV = 3
-						}
-					} else {
-						if "AA" == compareCity && (compareAgency == "A" || score2 > 1) && compareTime == "A" {
-							eqV = 3
-						}
-					}
-				case 1:
-					if compareBuyer < "C" {
-						if compareTime != "D" && "AA" == compareCity && score2 > 0 && compareAgency != "C" {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 1 {
-							eqV = 3
-						}
-					} else if compareBuyer == "D" {
-						if "AA" == compareCity && compareTime != "D" && score2 > 1 && compareAgency != "C" {
-							eqV = 3
-						} else if compareCity[1:1] != "C" && score2 > 1 && compareTime == "A" && compareAgency != "C" {
-							eqV = 3
-						}
-					} else {
-						if "AA" == compareCity && score2 > 2 && compareTime == "A" {
-							eqV = 3
-						}
-					}
-				}
-				if eqV == 1 {
-					comRes1 = append(comRes1, compareProject)
-				} else if eqV == 2 {
-					comRes2 = append(comRes2, compareProject)
-				} else if eqV == 3 {
-					comRes3 = append(comRes3, compareProject)
-				} else {
-					//log.Println("+++++++++++", resVal, comStr, info, compareProject)
-				}
-
-				//				compareBuyer := "D"
-				//				if info.Buyer != "" {
-				//					if info.Buyer == compareProject.Buyer {
-				//						compareBuyer += "A"
-				//					} else if compareProject.Buyer != "" {
-				//						if strings.Contains(info.Buyer, compareProject.Buyer) || strings.Contains(compareProject.Buyer, info.Buyer) {
-				//							compareBuyer += "B"
-				//						} else {
-				//							compareBuyer += "C"
-				//						}
-				//					}
-				//				}
-
-				//				//---------------------------------------
-				//				compareCity := ""
-				//				if info.Area != "全国" && info.Area != "" && info.Area == compareProject.Area {
-				//					compareCity += "A"
-				//					score += 2
-				//				} else if info.Area == "全国" || compareProject.Area == "全国" {
-				//					compareCity += "B"
-				//					score += 1
-				//				} else {
-				//					compareCity += "C"
-				//				}
-				//				if compareCity != "C" {
-				//					if info.City != "" && info.City == compareProject.City {
-				//						compareCity += "A"
-				//						score += 2
-				//					} else {
-				//						if info.Area == "全国" || compareProject.Area == "全国" {
-				//							compareCity += "B"
-				//						} else if info.City == compareCity {
-				//							compareCity += "B"
-				//						} else {
-				//							compareCity += "C"
-				//						}
-				//					}
-				//				} else {
-				//					compareCity += "C"
-				//				}
-
-				//				compareTime := "D"
-				//				//diffTime := math.Abs(float64(info.Publishtime - compareProject.Publistime[len(compareProject.Publistime)-1]))
-				//				if diffTime < 45*86400 {
-				//					compareTime += "A"
-				//					score += 2
-				//				} else if diffTime < 90*86400 {
-				//					compareTime += "B"
-				//					score += 1
-				//				}
-				//				compareAgency := "D"
-				//				if info.Agency != "" {
-				//					if info.Agency == compareProject.Agency {
-				//						compareAgency += "A"
-				//						score += 2
-				//					} else if compareProject.Agency != "" {
-				//						if strings.Contains(info.Agency, compareProject.Agency) || strings.Contains(compareProject.Agency, info.Agency) {
-				//							compareAgency += "B"
-				//							score += 1
-				//						} else {
-				//							compareAgency += "C"
-				//						}
-				//					}
-				//				}
-				//				compareBudget := "C"
-				//				if info.budget > 0 && float64(info.budget) == compareProject.Budget {
-				//					compareBudget = "A"
-				//				}
-				//				compareBidmount := "C"
-				//				if info.bidamount > 0 && float64(info.bidamount) == compareProject.Bidamount {
-				//					compareBidmount = "A"
-				//				}
-
-				//				//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构
-
-				//				comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
-				//				compareProject.comStr = comStr
-				//				switch resVal {
-				//				case 3:
-				//					if compareBuyer == "A" || compareBuyer == "B" {
-				//						comRes1 = append(comRes1, compareProject)
-				//					} else if compareBuyer == "D" {
-				//						if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-				//							comRes1 = append(comRes1, compareProject)
-				//						} else if compareCity[1:1] != "C" {
-				//							comRes2 = append(comRes2, compareProject)
-				//						}
-				//					} else {
-				//						if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-				//							comRes2 = append(comRes2, compareProject)
-				//						} else if compareCity[1:1] != "C" && compareAgency != "C" && compareTime != "D" {
-				//							comRes3 = append(comRes3, compareProject)
-				//						} else if compareCity[1:1] != "C" && (compareBudget == "A" || compareBidmount == "A") {
-				//							comRes3 = append(comRes3, compareProject)
-				//						}
-				//					}
-				//				case 1, 2:
-				//					if compareBuyer == "A" && compareTime != "D" {
-				//						comRes2 = append(comRes2, compareProject)
-				//					} else if compareBuyer == "B" && compareTime != "D" && compareAgency != "C" {
-				//						comRes3 = append(comRes3, compareProject)
-				//					} else if compareBuyer == "D" {
-				//						if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-				//							comRes2 = append(comRes2, compareProject)
-				//						} else {
-				//							comRes3 = append(comRes3, compareProject)
-				//						}
-				//					} else {
-				//						if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
-				//							comRes3 = append(comRes3, compareProject)
-				//						} else if "AA" == compareCity && (compareBudget == "A" || compareBidmount == "A") {
-				//							comRes3 = append(comRes3, compareProject)
-				//						}
-				//					}
-				//				}
-			}
-		}
-	}
-	//--------------------------------对比完成-----------------------
-	//更新数组、更新项目
-	for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} {
-		if len(resN) > 0 {
-			if len(resN) > 1 {
-				sort.Slice(resN, func(i, j int) bool {
-					return resN[i].score > resN[j].score
-				})
-			}
-
-			bFindProject = true
-			findPid = resN[0].Id.Hex()
-			for k2, bv := range []int{bpn, bpc, bptc, bpb} {
-				if bv > -1 {
-					pids[bv].Arr = append(pids[bv].Arr, findPid)
-					if k2 == 0 {
-						if resN[0].ProjectName == "" {
-							resN[0].ProjectName = info.ProjectName
-						} else {
-							if resN[0].MPN == nil {
-								resN[0].MPN = []string{info.ProjectName}
-							} else {
-								resN[0].MPN = append(resN[0].MPN, info.ProjectName)
-							}
-						}
-
-					} else if k2 < 3 {
-						if resN[0].ProjectCode == "" {
-							resN[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
-						} else {
-							if resN[0].MPC == nil {
-								resN[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
-							} else {
-								resN[0].MPC = append(resN[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
-							}
-						}
-
-					} else {
-						if resN[0].Buyer == "" {
-							resN[0].Buyer = info.Buyer
-						}
-					}
-				}
-			}
-			UpdateProject(tmp, info, resN[0], kv+1, resN[0].comStr)
-			//更新AllIdsMao中的时间
-			break
-		}
-	}
-
-	if !bFindProject {
-		//没有找到
-		id, p := NewProject(tmp, info)
-		//_, _ = strconv.ParseInt(id[0:8], 16, 64)
-		AllIdsMapLock.Lock()
-		//AllIdsMap[int(t)%idsMapSize][id] = &ID{Id: id, lastTime: info.Publishtime}
-		AllIdsMap2[id] = &ID{Id: id, lastTime: info.Publishtime, P: p}
-		AllIdsMapLock.Unlock()
-		for _, m := range pids {
-			m.Arr = append(m.Arr, id)
-		}
-	}
-
-}
-
-//二分字符串查找
-func BinarySearch(s []string, k string) int {
-	sort.Strings(s)
-	lo, hi := 0, len(s)-1
-	for lo <= hi {
-		m := (lo + hi) >> 1
-		if s[m] < k {
-			lo = m + 1
-		} else if s[m] > k {
-			hi = m - 1
-		} else {
-			return m
-		}
-	}
-	return -1
-}

fullproject/src_dev3/merge_comparepncb.go

@@ -1,160 +0,0 @@
-package main
-
-import (
-	"strings"
-)
-
-func comparePNCB(info *Info, compareProject *ProjectInfo) (compareStr string, score int) {
-	if info.ProjectName != "" {
-		pns := []string{}
-		if compareProject.ProjectName != "" {
-			pns = append(pns, compareProject.ProjectName)
-		}
-		if len(compareProject.MPN) > 0 {
-			pns = append(pns, compareProject.MPN...)
-		}
-		ifind := 0
-		for _, v := range pns {
-			if info.ProjectName == v {
-				ifind = 1
-				break
-			} else {
-				//if strings.Contains(info.ProjectName, v) || strings.Contains(v, info.ProjectName) ||
-				retv := CheckContain(info.ProjectName, v, info.Buyer)
-				if retv == 1 {
-					ifind = 1
-					break
-				} else if retv == 2 {
-					ifind = 2
-				} else if ifind == 0 {
-					ifind = 3
-				}
-			}
-		}
-		switch ifind {
-		case 0:
-			compareStr = "D"
-		case 1:
-			compareStr = "A"
-			score += 4
-			if len([]rune(info.ProjectName)) > 18 {
-				score += 2
-			}
-		case 2:
-			compareStr = "B"
-			score += 2
-			if len([]rune(info.ProjectName)) > 18 {
-				score += 1
-			}
-		case 3:
-			compareStr = "C"
-		}
-	} else {
-		compareStr = "D"
-	}
-
-	/*
-			项目编号 - －()（） 要注意
-			init_text = ["号","(重)","(第二次)","（重）"]
-	all_clean_mark = ["[","(","【","（","〖","]",")","】","）","〗","-","〔","〕","《","［","］","｛","｝","{","—"," ","－","﹝","﹞","–"]
-	*/
-	for _, pc := range []string{info.ProjectCode, info.PTC} {
-		if pc != "" {
-			pcs := []string{}
-			if compareProject.ProjectCode != "" {
-				pcs = append(pcs, compareProject.ProjectCode)
-			}
-			if len(compareProject.MPC) > 0 {
-				pcs = append(pcs, compareProject.MPC...)
-			}
-			ifind := 0
-			for _, v := range pcs {
-				if pc == v {
-					ifind = 1
-					break
-				} else {
-					// math.Abs(float64(len([]rune(pc))-len([]rune(v)))) < 6
-					//if !_numreg1.MatchString(pc) && !_zimureg1.MatchString(pc) && !_numreg1.MatchString(v) && !_zimureg1.MatchString(v)
-					if strings.Contains(pc, v) || strings.Contains(v, pc) {
-						t1 := pc
-						t2 := v
-						if len(v) > len(pc) {
-							t1 = v
-							t2 = pc
-						}
-						t3 := strings.Replace(t1, t2, "", -1)
-						t3 = _datereg.ReplaceAllString(t3, "")
-						if t3 == "" {
-							ifind = 1
-							break
-						} else {
-							ifind = 2
-						}
-					} else if ifind == 0 {
-						ifind = 3
-					}
-				}
-			}
-			switch ifind {
-			case 0:
-				compareStr += "D"
-			case 1:
-				compareStr += "A"
-				score += 4
-				if len([]rune(pc)) > 18 {
-					score += 2
-				}
-			case 2:
-				compareStr += "B"
-				score += 2
-				if len([]rune(pc)) > 18 {
-					score += 1
-				}
-			case 3:
-				compareStr += "C"
-			}
-
-		} else {
-			compareStr += "D"
-		}
-	}
-	return
-}
-
-func CheckContain(b1, b2, infoBuyer string) (res int) {
-	b1 = replaceStr.ReplaceAllString(b1, "")
-	b2 = replaceStr.ReplaceAllString(b2, "")
-	b1 = strings.Replace(b1, infoBuyer, "", 1)
-	b2 = strings.Replace(b2, infoBuyer, "", 1)
-
-	if b1 == b2 {
-		res = 1 //相等
-		return
-	}
-	bs1 := []rune(b1)
-	bs2 := []rune(b2)
-	tmp := ""
-	for i := 0; i < len(bs1); i++ {
-		for j := 0; j < len(bs2); j++ {
-			if bs1[i] == bs2[j] {
-				tmp += string(bs1[i])
-			} else if tmp != "" {
-				b1 = strings.Replace(b1, tmp, "", -1)
-				b2 = strings.Replace(b2, tmp, "", -1)
-				tmp = ""
-			}
-		}
-	}
-	if tmp != "" {
-		b1 = strings.Replace(b1, tmp, "", -1)
-		b2 = strings.Replace(b2, tmp, "", -1)
-	}
-	if b1 == b2 {
-		res = 1 //相等
-	} else if b1 == "" || b2 == "" {
-		res = 2 //包含
-	} else {
-		res = 3 //不相等
-	}
-	return
-}

fullproject/src_dev3/merge_select.go

@@ -1,276 +0,0 @@
-package main
-
-func Select(compareStr string, info *Info, compareInfo *ProjectInfo) (res int) {
-	//没有可对比的项目名称、或项目编号
-	if compareNoPass[compareStr] {
-
-	} else {
-		switch compareStr {
-		case "AAA":
-			res = 3
-		case "AAB":
-			res = 3
-		case "ABA":
-			res = 3
-		case "ABB":
-			if info.LenPTC > 6 || info.LenPC > 6 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BAA":
-			if info.LenPN > 10 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BAB":
-			if info.LenPN > 10 || info.LenPTC > 6 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BBA":
-			if info.LenPN > 10 || info.LenPC > 6 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "BBB":
-			v := 0
-			if info.LenPN > 12 {
-				v++
-			}
-			if info.LenPC > 8 {
-				v++
-			}
-			if info.LenPTC > 8 {
-				v++
-			}
-			if v > 1 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "AAD":
-			res = 3
-		case "ABD":
-			//			if info.LenPC > 12 || info.LenPN > 16 {
-			//				res = 3
-			//			} else {
-			//				res = 2
-			//			}
-			res = 2
-		case "BAD":
-			//			if info.LenPC > 12 || info.LenPN > 16 {
-			//				res = 3
-			//			} else {
-			//				res = 2
-			//			}
-			res = 2
-		case "BBD":
-			if info.LenPC > 12 && info.LenPN > 16 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "ADA":
-			res = 3
-		case "ADB":
-			//			if info.LenPTC > 12 || info.LenPN > 16 {
-			//				res = 3
-			//			} else {
-			//				res = 2
-			//			}
-			res = 2
-		case "BDA":
-			//			if info.LenPTC > 12 || info.LenPN > 16 {
-			//				res = 3
-			//			} else {
-			//				res = 2
-			//			}
-			res = 2
-		case "BDB":
-			if info.LenPTC > 12 && info.LenPN > 16 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "ADD":
-			if info.LenPN > 14 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BDD":
-			//			if info.LenPN > 18 {
-			//				res = 2
-			//			} else {
-			//				res = 1
-			//			}
-			res = 1
-		case "DAA":
-			if info.LenPTC > 12 || info.LenPC > 12 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "DAB":
-			if info.LenPTC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DAD":
-			if info.LenPTC > 14 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DBA":
-			if info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DBB":
-			if info.LenPTC > 12 && info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DBD":
-			if info.LenPC > 8 {
-				res = 1
-			}
-		case "DDA":
-			if info.LenPTC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "DDB":
-			if info.LenPC > 8 {
-				res = 1
-			}
-		case "ACA":
-			if info.LenPN > 12 || info.LenPTC > 9 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "ACB":
-			if info.LenPN > 14 && info.LenPTC > 10 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BCA":
-			if info.LenPN > 12 && info.LenPTC > 9 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BCB":
-			if info.LenPN > 12 && info.LenPTC > 9 {
-				res = 2
-			} else if info.LenPN > 16 || info.LenPTC > 12 {
-				res = 1
-			}
-		case "AAC":
-			if info.LenPN > 12 || info.LenPC > 9 {
-				res = 3
-			} else {
-				res = 2
-			}
-		case "ABC":
-			if info.LenPN > 14 || info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BAC":
-			if info.LenPN > 14 || info.LenPC > 12 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "BBC":
-			//			if info.LenPN > 12 && info.LenPTC > 9 {
-			//				res = 1
-			//			} else {
-			//				res = 1
-			//			}
-			res = 1
-		case "ACC":
-			//			if info.LenPC > 4 && len(compareInfo.ProjectCode) > 4 && CheckHanAndNum(info.ProjectCode) && CheckHanAndNum(compareInfo.ProjectCode) {
-			//				//未考虑MPC
-			//			} else if info.LenPN > 16 {
-			//				res = 1
-			//			}
-			//
-			res = 1
-		case "ACD":
-			//项目编号不一致
-			res = 1
-		case "ADC":
-			res = 1
-		case "BCC":
-			//项目编号不一致
-			res = 1
-		case "BCD":
-			//项目编号不一致
-			if info.LenPC > 4 && len(compareInfo.ProjectCode) > 4 && (!_numreg1.MatchString(info.ProjectCode) && !_numreg1.MatchString(compareInfo.ProjectCode)) {
-				//未考虑MPC
-			} else if info.LenPN > 18 && info.LenPC-len([]rune(compareInfo.ProjectCode)) != 0 {
-				res = 1
-			}
-		case "BDC":
-			if info.LenPN > 18 && info.LenPTC-len([]rune(compareInfo.ProjectCode)) != 0 {
-				res = 1
-			}
-		case "CAA":
-			if info.LenPC > 10 || info.LenPTC > 10 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "CAB":
-			if info.LenPC > 10 && info.LenPTC > 10 {
-				res = 2
-			} else {
-				res = 1
-			}
-		case "CAC":
-			res = 1
-		case "CAD":
-			res = 1
-		case "CBA":
-			res = 1
-		case "CBB":
-			res = 1
-		case "CBC":
-			res = 1
-		case "CBD":
-			res = 1
-		case "CCA":
-			res = 1
-		case "CCB":
-			//
-		case "CDA":
-			res = 1
-		case "CDB":
-			res = 1
-		case "DAC":
-			res = 1
-		case "DBC":
-			res = 1
-		case "DCA":
-			res = 1
-		case "DCB":
-			res = 1
-		}
-
-	}
-	return
-}

fullproject/src_dev3/new_project.go

@@ -1,181 +0,0 @@
-package main
-
-import (
-	qu "qfw/util"
-	"strings"
-	"time"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-var FIELDS = []string{"area", "city", "district", "projectname", "projectcode", "buyer", "winner", "budget", "bidamount", "bidstatus", "agency", "projectscope"}
-
-func NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *ProjectInfo) {
-	set := map[string]interface{}{}
-	for _, f := range FIELDS {
-		if tmp[f] != nil {
-			set[f] = tmp[f]
-		}
-	}
-	set["s_projectname"] = thisinfo.ProjectName
-	set["createtime"] = time.Now().Unix()
-	set["sourceinfoid"] = qu.BsonIdToSId(tmp["_id"])
-	set["sourceinfourl"] = tmp["href"]
-	set["topscopeclass"] = thisinfo.Topscopeclass
-	set["subscopeclass"] = thisinfo.Subscopeclass
-	if thisinfo.Buyerperson != "" {
-		set["buyerperson"] = thisinfo.Buyerperson
-	}
-	if thisinfo.Buyertel != "" {
-		set["buyertel"] = thisinfo.Buyertel
-	}
-	if thisinfo.Buyerclass != "" {
-		set["buyerclass"] = thisinfo.Buyerclass
-	}
-	if thisinfo.Bidopentime > 0 {
-		set["bidopentime"] = thisinfo.Bidopentime
-	}
-	if len(thisinfo.Winnerorder) > 0 {
-		set["winnerorder"] = thisinfo.Winnerorder
-	}
-	s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
-	set["s_subscopeclass"] = s_subscopeclass
-	s_winner := strings.Join(thisinfo.Winners, ",")
-	set["s_winner"] = s_winner
-	if tmp["package"] != nil {
-		set["package"] = tmp["package"] //没定义优先级
-	}
-	push := NewPushInfo(tmp)
-	set["list"] = []bson.M{
-		push,
-	}
-	pId := bson.NewObjectId()
-	set["_id"] = pId
-	///id := MongoTool.Save(ProjectColl, set)
-	p1 := NewPinfo(pId, thisinfo, set)
-	return pId.Hex(), &p1
-}
-
-func NewPushInfo(tmp map[string]interface{}) bson.M {
-	res := bson.M{
-		"comeintime":  tmp["comeintime"],
-		"publishtime": tmp["publishtime"],
-		"title":       tmp["title"],
-		"toptype":     tmp["toptype"],
-		"subtype":     tmp["subtype"],
-		"infoformat":  tmp["infoformat"],
-		"infoid":      qu.BsonIdToSId(tmp["_id"]),
-		"area":        tmp["area"],
-		"city":        tmp["city"],
-		"projectname": tmp["projectname"],
-		"projectcode": tmp["projectcode"],
-		"buyer":       tmp["buyer"],
-		"href":        tmp["href"],
-	}
-	for _, k := range []string{"winner", "budget", "bidamount"} {
-		if tmp[k] != nil {
-			res[k] = tmp[k]
-		}
-	}
-	return res
-}
-
-//生成存放在redis数组中的对象
-func NewPinfo(id bson.ObjectId, thisinfo *Info, set map[string]interface{}) ProjectInfo {
-	p1 := ProjectInfo{
-		Id:            id,
-		Ids:           []string{thisinfo.Id},
-		Topscopeclass: thisinfo.Topscopeclass,
-		Subscopeclass: thisinfo.Subscopeclass,
-		Winners:       thisinfo.Winners,
-		ProjectName:   thisinfo.ProjectName,
-		ProjectCode:   thisinfo.ProjectCode,
-		Buyer:         thisinfo.Buyer,
-		Agency:        thisinfo.Agency,
-		Area:          thisinfo.Area,
-		City:          thisinfo.City,
-		District:      thisinfo.District,
-		MPN:           []string{},
-		MPC:           []string{},
-		HasPackage:    thisinfo.HasPackage,
-		Package:       map[string]interface{}{},
-		Buyerclass:    thisinfo.Buyerclass,
-		Bidopentime:   thisinfo.Bidopentime,
-		Winnerorder:   thisinfo.Winnerorder,
-		FirstTime:     thisinfo.Publishtime,
-		LastTime:      thisinfo.Publishtime,
-		Budget:        thisinfo.Budget,
-		Bidamount:     thisinfo.Bidamount,
-	}
-	if thisinfo.LenPTC > 5 {
-		p1.MPC = append(p1.MPC, thisinfo.PTC)
-	}
-	//savePool <- set
-	MongoTool.SaveByOriID(ProjectColl, set)
-
-	return p1
-}
-
-var updatePool = make(chan []map[string]interface{}, 30)
-
-//var savePool = make(chan map[string]interface{}, 6)
-//func SaveQueue() {
-//	arr := []map[string]interface{}{}
-//	sp := make(chan bool, 3)
-//	for {
-//		select {
-//		case <-beforUpdate:
-//			if len(arr) > 0 {
-//				MongoTool.SaveBulk(ProjectColl, arr...)
-//				arr = []map[string]interface{}{}
-//			}
-//		case v := <-savePool:
-//			arr = append(arr, v)
-//			if len(arr) > 50 {
-//				sp <- true
-//				go func(arr []map[string]interface{}) {
-//					MongoTool.SaveBulk(ProjectColl, arr...)
-//					<-sp
-//				}(arr)
-//				arr = []map[string]interface{}{}
-//			}
-//		case <-time.After(80 * time.Millisecond):
-//			if len(arr) > 0 {
-//				sp <- true
-//				go func(arr []map[string]interface{}) {
-//					MongoTool.SaveBulk(ProjectColl, arr...)
-//					<-sp
-//				}(arr)
-//				arr = []map[string]interface{}{}
-//			}
-//		}
-//	}
-//}
-
-func updateQueue() {
-	arr := [][]map[string]interface{}{}
-	sp := make(chan bool, 3)
-	for {
-		select {
-		case v := <-updatePool:
-			arr = append(arr, v)
-			if len(arr) > 200 {
-				sp <- true
-				go func(arr [][]map[string]interface{}) {
-					MongoTool.UpdateBulk(ProjectColl, arr...)
-					<-sp
-				}(arr)
-				arr = [][]map[string]interface{}{}
-			}
-		case <-time.After(500 * time.Millisecond):
-			if len(arr) > 0 {
-				sp <- true
-				go func(arr [][]map[string]interface{}) {
-					MongoTool.UpdateBulk(ProjectColl, arr...)
-					<-sp
-				}(arr)
-				arr = [][]map[string]interface{}{}
-			}
-		}
-	}
-}

fullproject/src_dev3/task.go

@@ -1,207 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"log"
-	"qfw/util"
-	"regexp"
-	"sync"
-	//"strings"
-	"time"
-)
-
-const (
-	InitMinTime = int64(1325347200) //最小时间位置2012
-)
-
-//全量合并
-func taskQl(udpInfo map[string]interface{}) {
-	defer util.Catch()
-	//1、检查pubilshtime索引
-	db, _ := udpInfo["db"].(string)
-	if db == "" {
-		db = MongoTool.DbName
-	}
-	coll, _ := udpInfo["coll"].(string)
-	if coll == "" {
-		coll = ExtractColl
-	}
-	sess := MongoTool.GetMgoConn()
-	bcon := false
-	if sess.DB(db).C(coll).EnsureIndexKey("publishtime_1", "publishtime_-1") == nil {
-		bcon = true
-	} else {
-		log.Println("publishtime_1索引不存在")
-	}
-	MongoTool.DestoryMongoConn(sess)
-	thread := util.IntAllDef(udpInfo["thread"], 1)
-	if bcon {
-		//go SaveQueue()
-		go updateQueue()
-		go clearMem()
-		//获取起始时间
-		startTime, END := int64(0), int64(0)
-
-		sts, bres := MongoTool.Find(ExtractColl, `{}`, "publishtime", `{"publishtime":1}`, true, 0, 1)
-		if bres && sts != nil && len(*sts) == 1 {
-			startTime = util.Int64All((*sts)[0]["publishtime"])
-			sts, bres = MongoTool.Find(ExtractColl, `{}`, "-publishtime", `{"publishtime":1}`, true, 0, 1)
-			if bres && sts != nil && len(*sts) == 1 {
-				END = util.Int64All((*sts)[0]["publishtime"])
-			}
-			log.Println("查询到的起始时间", startTime, END)
-		} else {
-			return
-		}
-		startTime -= 1
-		sum := 0
-		if startTime < InitMinTime {
-			q := map[string]interface{}{
-				"publishtime": map[string]interface{}{
-					"$gt":  startTime,
-					"$lte": InitMinTime,
-				},
-			}
-			sum = Mql(q, thread, db, coll, sum)
-			startTime = InitMinTime
-		}
-		for {
-			if startTime >= END {
-				break
-			}
-			et := startTime + 50*86400
-			if et >= END {
-				et = END
-			}
-			q := map[string]interface{}{
-				"publishtime": map[string]interface{}{
-					"$gt":  startTime,
-					"$lte": et,
-				},
-			}
-			sum = Mql(q, thread, db, coll, sum)
-			startTime = et
-			time.Sleep(1 * time.Second)
-		}
-	}
-	log.Println("task over!!!")
-}
-
-var wg = sync.WaitGroup{}
-
-func Mql(q map[string]interface{}, thread int, db, coll string, sum int) int {
-	defer util.Catch()
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
-	query := sess.DB(db).C(coll).Find(q).Sort("publishtime").Iter()
-	pool := make(chan bool, thread)
-	count := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
-		info := ParseInfo(tmp)
-		if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
-			pool <- true
-			go func(info *Info, tmp map[string]interface{}) {
-				defer func() {
-					currentTime = info.Publishtime
-					<-pool
-				}()
-				startProjectMerge(info, tmp)
-			}(info, tmp)
-		} else {
-			//log.Println("info err:", tmp["_id"], tmp["title"], tmp["buyer"])
-		}
-		if sum%1000 == 0 {
-			log.Println("current", sum)
-		}
-		sum++
-		tmp = make(map[string]interface{})
-	}
-	//阻塞
-	for n := 0; n < thread; n++ {
-		pool <- true
-	}
-	//完成
-	log.Println("sontask over:", count, sum, q)
-	return sum
-}
-
-var (
-	titleGetPc  = regexp.MustCompile("^([-0-9a-zA-Z第号采招政询电审竞#]{8,}[-0-9a-zA-Z#]+)")
-	titleGetPc1 = regexp.MustCompile("[\\[【(（](.{0,6}(编号|编码|项号|包号|代码|标段?号)[:：为])?([-0-9a-zA-Z第号采招政询电审竞#]{5,}([\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+[\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+)?)[\\]】)）]")
-	titleGetPc2 = regexp.MustCompile("([-0-9a-zA-Z第号采政招询电审竞#]{8,}[-0-9a-zA-Z#]+)(.{0,5}公告)?$")
-	pcReplace   = regexp.MustCompile("([\\[【(（〖〔《｛﹝{](重|第?[二三四再]次.{0,4})[\\]】)）〗〕》｝﹞}])$|[\\[\\]【】()（）〖〗〔〕《》｛｝﹝﹞－{}– 　]+|(号|重|第?[二三四五再]次(招标)?)$")
-)
-
-func ParseInfo(tmp map[string]interface{}) (info *Info) {
-	bys, _ := json.Marshal(tmp)
-	var thisinfo *Info
-	json.Unmarshal(bys, &thisinfo)
-	if thisinfo == nil {
-		return nil
-	}
-	if len(thisinfo.Topscopeclass) == 0 {
-		thisinfo.Topscopeclass = []string{}
-	}
-	if len(thisinfo.Subscopeclass) == 0 {
-		thisinfo.Subscopeclass = []string{}
-	}
-
-	res := titleGetPc.FindStringSubmatch(thisinfo.Title)
-	if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
-		thisinfo.PTC = res[1]
-		thisinfo.pnbval++
-	} else {
-		res = titleGetPc1.FindStringSubmatch(thisinfo.Title)
-		if len(res) > 3 && len(res[3]) > 6 && thisinfo.ProjectCode != res[3] && !numCheckPc.MatchString(res[3]) && !_zimureg1.MatchString(res[3]) {
-			thisinfo.PTC = res[3]
-			thisinfo.pnbval++
-		} else {
-			res = titleGetPc2.FindStringSubmatch(thisinfo.Title)
-			if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
-				thisinfo.PTC = res[1]
-				thisinfo.pnbval++
-			}
-		}
-	}
-
-	if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
-		//		thisinfo.ProjectName = strings.Replace(thisinfo.ProjectName, "（", "(", -1)
-		//		thisinfo.ProjectName = strings.Replace(thisinfo.ProjectName, "）", ")", -1)
-		//		thisinfo.ProjectName = strings.Replace(thisinfo.ProjectName, "－", "", -1)
-		thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
-		if thisinfo.ProjectName != "" {
-			thisinfo.pnbval++
-		}
-	}
-
-	if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
-		if thisinfo.ProjectCode != "" {
-			thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
-			//			thisinfo.ProjectCode = strings.Replace(thisinfo.ProjectCode, "（", "(", -1)
-			//			thisinfo.ProjectCode = strings.Replace(thisinfo.ProjectCode, "）", ")", -1)
-			//			thisinfo.ProjectCode = strings.Replace(thisinfo.ProjectCode, "－", "", -1)
-		} else {
-			thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
-			//			thisinfo.PTC = strings.Replace(thisinfo.PTC, "（", "(", -1)
-			//			thisinfo.PTC = strings.Replace(thisinfo.PTC, "）", ")", -1)
-			//			thisinfo.PTC = strings.Replace(thisinfo.PTC, "－", "", -1)
-		}
-		if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
-			thisinfo.pnbval++
-		}
-	}
-	if thisinfo.ProjectCode == thisinfo.PTC {
-		thisinfo.PTC = ""
-	}
-
-	if thisinfo.Buyer != "" && len([]rune(thisinfo.Buyer)) > 2 {
-		thisinfo.pnbval++
-	} else {
-		thisinfo.Buyer = ""
-	}
-
-	thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
-	thisinfo.LenPTC = len([]rune(thisinfo.PTC))
-	thisinfo.LenPN = len([]rune(thisinfo.ProjectName))
-	return thisinfo
-}

fullproject/src_dev3/update_project.go

@@ -1,231 +0,0 @@
-package main
-
-import (
-	qu "qfw/util"
-	"sort"
-	"strings"
-)
-
-func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string) {
-	//log.Println(thisinfo.ProjectName, pInfo.ProjectName, comStr)
-	//	updateid := pInfo.Id
-	//	if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
-	//		return //updateid
-	//	}
-	set := map[string]interface{}{}
-	//res, bres := MongoTool.FindById(ProjectColl, pInfo.Id.Hex(), `{"list":0}`)
-	EqInfoUpdate(thisinfo, pInfo)
-	//if bres && res != nil && *res != nil {
-	set["topscopeclass"] = pInfo.Topscopeclass
-	set["subscopeclass"] = pInfo.Subscopeclass
-	s_subscopeclass := strings.Join(pInfo.Subscopeclass, ",")
-	if len(s_subscopeclass) > 0 {
-		s_subscopeclass = "," + s_subscopeclass + ","
-	}
-	set["s_subscopeclass"] = s_subscopeclass
-	s_winner := strings.Join(pInfo.Winners, ",")
-	if len(s_winner) > 0 {
-		s_winner = "," + s_winner + ","
-	}
-	set["s_winner"] = s_winner
-	if pInfo.Buyerperson != "" && pInfo.Buyertel != "" {
-		set["buyerperson"] = pInfo.Buyerperson
-		set["buyertel"] = pInfo.Buyertel
-	}
-	if pInfo.Buyerclass != "" {
-		set["buyerclass"] = pInfo.Buyerclass
-	}
-	if pInfo.District != "" {
-		set["district"] = pInfo.District
-	}
-	if pInfo.Bidopentime > 0 {
-		set["bidopentime"] = pInfo.Bidopentime
-	}
-	if len(pInfo.Winnerorder) > 0 {
-		set["winnerorder"] = pInfo.Winnerorder
-	}
-	if thisinfo.HasPackage {
-		set["multipackage"] = 1
-	} else {
-		set["multipackage"] = 0
-	}
-
-	if pInfo.ProjectName != "" {
-		set["projectname"] = pInfo.ProjectName
-	}
-
-	if pInfo.ProjectCode != "" {
-		set["projectcode"] = pInfo.ProjectCode
-	}
-
-	if pInfo.Buyer != "" {
-		set["buyer"] = pInfo.Buyer
-	}
-
-	//预算、中标价
-	if qu.Float64All(tmp["bidamount"]) > 0 && pInfo.Bidamount == 0 {
-		set["bidamount"] = tmp["bidamount"]
-	}
-
-	if qu.Float64All(tmp["budget"]) > 0 && pInfo.Budget == 0 {
-		set["budget"] = tmp["budget"]
-	}
-
-	set["mpn"] = pInfo.MPN
-	set["mpc"] = pInfo.MPC
-	set["area"] = pInfo.Area
-	set["city"] = pInfo.City
-
-	//e := InitEL(util.ObjToString((*res)["extractpos"]))
-	if thisinfo.dealtype == 1 {
-		var sonpackage map[string]interface{}
-		for _, obj := range tmp["package"].(map[string]interface{}) {
-			sonpackage, _ = obj.(map[string]interface{})
-		}
-		for _, v2 := range []string{"budget", "budget_w", "winner", "winner_w", "bidstatus", "bidstatus_w"} {
-			if sonpackage[v2] != nil {
-				tmp[v2] = sonpackage[v2]
-			}
-		}
-	}
-	//e.fieldpriority(&tmp, res, &set)
-	//set["extractpos"] = e.GetVal()
-	//	if thisinfo.HasPackage { //多包处理
-	//		p1, _ := (*res)["package"].(map[string]interface{})
-	//		p2, _ := tmp["package"].(map[string]interface{})
-	//		if p2 != nil {
-	//			if p1 != nil {
-	//				for pk2, pv2 := range p2 {
-	//					if p1[pk2] != nil { //合并
-	//						item1, _ := p1[pk2].(map[string]interface{})
-	//						item2, _ := pv2.(map[string]interface{})
-	//						if item1 != nil && item2 != nil { //原始项
-	//							for ik1, iv1 := range item2 {
-	//								if item1[ik1] == nil {
-	//									item1[ik1] = iv1
-	//								}
-	//							}
-	//						}
-	//					} else {
-	//						p1[pk2] = pv2
-	//					}
-	//				}
-	//			} else {
-	//				p1 = p2
-	//			}
-	//		}
-	//		set["package"] = p1
-	//	}
-	//中标候选人合并
-
-	update := map[string]interface{}{}
-	if len(set) > 0 {
-		update["$set"] = set
-	}
-	//保留原数据吧
-	push := NewPushInfo(tmp)
-	push["compareStr"] = comStr
-	update["$push"] = map[string]interface{}{
-		"list": push,
-	}
-	if len(update) > 0 {
-		updateInfo := []map[string]interface{}{
-			map[string]interface{}{
-				"_id": pInfo.Id,
-			},
-			update,
-		}
-		updatePool <- updateInfo
-		//			MongoTool.Update(ProjectColl, map[string]interface{}{
-		//				"_id": qu.StringTOBsonId(pInfo.Id.Hex()),
-		//			}, &update, false, false)
-	}
-	//}
-	//再往redis中放 index
-	//往队列中增加时间 -------------->start
-}
-
-func EqInfoUpdate(thisinfo *Info, pInfo *ProjectInfo) {
-	if thisinfo.Publishtime > pInfo.LastTime {
-		pInfo.LastTime = thisinfo.Publishtime
-	}
-	if pInfo.FirstTime == 0 || (thisinfo.Publishtime < pInfo.FirstTime && thisinfo.Publishtime > 0) {
-		pInfo.FirstTime = thisinfo.Publishtime
-	}
-	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
-	//增加发布时间结束----------------->end
-
-	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
-		pInfo.Buyer = thisinfo.Buyer
-	}
-	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
-		pInfo.Agency = thisinfo.Agency
-	}
-	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
-		pInfo.ProjectCode = thisinfo.ProjectCode
-	}
-
-	if pInfo.Area == "全国" && thisinfo.Area != "全国" {
-		pInfo.Area = thisinfo.Area
-		pInfo.City = thisinfo.City
-	}
-	if thisinfo.District != "" {
-		pInfo.District = thisinfo.District
-	}
-	if thisinfo.Buyerperson != "" && thisinfo.Buyertel != "" {
-		pInfo.Buyerperson = thisinfo.Buyerperson
-		pInfo.Buyertel = thisinfo.Buyertel
-	}
-	if thisinfo.Buyerclass != "" {
-		pInfo.Buyerclass = thisinfo.Buyerclass
-	}
-	if thisinfo.Bidopentime > 0 {
-		pInfo.Bidopentime = thisinfo.Bidopentime
-	}
-	if thisinfo.Bidamount > 0 && pInfo.Bidamount < 1 {
-		pInfo.Bidamount = thisinfo.Bidamount
-	}
-	if thisinfo.Budget > 0 && pInfo.Budget < 1 {
-		pInfo.Budget = thisinfo.Budget
-	}
-
-	if len(thisinfo.Topscopeclass) > 0 {
-		sort.Strings(pInfo.Topscopeclass)
-		for _, k := range thisinfo.Topscopeclass {
-			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
-				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
-				sort.Strings(pInfo.Topscopeclass)
-			}
-		}
-	}
-
-	if len(thisinfo.Subscopeclass) > 0 {
-		sort.Strings(pInfo.Subscopeclass)
-		for _, k := range thisinfo.Subscopeclass {
-			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
-				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
-				sort.Strings(pInfo.Subscopeclass)
-			}
-		}
-	}
-	//winner
-	if len(thisinfo.Winners) > 0 {
-		sort.Strings(pInfo.Winners)
-		for _, k := range thisinfo.Winners {
-			if BinarySearch(pInfo.Winners, k) == -1 {
-				pInfo.Winners = append(pInfo.Winners, k)
-				sort.Strings(pInfo.Winners)
-			}
-		}
-	}
-	//winnerorder
-	if len(thisinfo.Winnerorder) > 0 {
-		sort.Strings(pInfo.Winnerorder)
-		for _, k := range thisinfo.Winnerorder {
-			if BinarySearch(pInfo.Winnerorder, k) == -1 {
-				pInfo.Winnerorder = append(pInfo.Winnerorder, k)
-				sort.Strings(pInfo.Winnerorder)
-			}
-		}
-	}
-}

fullproject/src_dev4/clearmem.go

@@ -1,87 +0,0 @@
-package main
-
-import (
-	"log"
-
-	"github.com/robfig/cron"
-)
-
-//定时清理内存
-//当前位置时间，小于此时间超过6个月，则清理
-var currentTime = int64(0)
-var validTime = int64(6 * 30 * 86400)
-
-func clearMem() {
-	c := cron.New()
-	c.AddFunc("50 0/3 * * * *", func() {
-		findLock.Lock()
-		defer findLock.Unlock()
-		wg.Wait()
-		//遍历id
-		AllIdsMapLock.Lock()
-		defer AllIdsMapLock.Unlock()
-		clearNum := 0
-		for k, v := range AllIdsMap2 {
-			if currentTime-v.lastTime > validTime {
-				clearNum++
-				//删除id的map
-				delete(AllIdsMap2, k)
-				//删除pb
-				if v.P.Buyer != "" {
-					ids := mapPb[v.P.Buyer]
-					if ids != nil {
-						ids.Lock.Lock()
-						ids.Arr = deleteSlice(ids.Arr, k)
-						if len(ids.Arr) == 0 {
-							delete(mapPb, v.P.Buyer)
-						}
-						ids.Lock.Unlock()
-					}
-				}
-				//删除mapPn
-				for _, vn := range append([]string{v.P.ProjectName}, v.P.MPN...) {
-					if vn != "" {
-						ids := mapPn[vn]
-						if ids != nil {
-							ids.Lock.Lock()
-							ids.Arr = deleteSlice(ids.Arr, k)
-							if len(ids.Arr) == 0 {
-								delete(mapPn, vn)
-							}
-							ids.Lock.Unlock()
-						}
-					}
-				}
-				//删除mapPc
-				for _, vn := range append([]string{v.P.ProjectCode}, v.P.MPC...) {
-					if vn != "" {
-						ids := mapPc[vn]
-						if ids != nil {
-							ids.Lock.Lock()
-							ids.Arr = deleteSlice(ids.Arr, k)
-							if len(ids.Arr) == 0 {
-								delete(mapPc, vn)
-							}
-							ids.Lock.Unlock()
-						}
-					}
-				}
-				v = nil
-			}
-		}
-		log.Println("清除完成:", clearNum, len(AllIdsMap2))
-	})
-	c.Start()
-	defer c.Stop()
-	select {}
-}
-
-func deleteSlice(arr []string, v string) []string {
-	j := 0
-	for _, v1 := range arr {
-		if v1 != v {
-			arr[j] = v1
-		}
-	}
-	return arr[:j]
-}

fullproject/src_dev4/config.json

@@ -1,16 +0,0 @@
-{
-    "thread": 1,
-    "mongodbServers": "192.168.3.207:27082",
-    "mongodbPoolSize": 10,
-    "mongodbName": "cesuo",
-    "extractColl": "key1_biddingall",
-    "projectColl": "projectset_0809",
-    "redisaddrs": "ids=192.168.3.207:1378,keys=192.168.3.207:1378,info=192.168.3.207:1378",
-    "redisPoolSize": 20,
-    "jkmail": {
-        "to": "zhangjinkun@topnet.net.cn",
-        "api": "http://10.171.112.160:19281/_send/_mail"
-    },
-    "udpport": ":1482",
-    "nextNode": []
-}

fullproject/src_dev4/init.go

@@ -1,301 +0,0 @@
-package main
-
-import (
-	"log"
-	mu "mfw/util"
-	"qfw/util"
-	"qfw/util/mongodb"
-	"regexp"
-	"sync"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-const (
-	ProjectCache = "info" //存放每条项目信息，key为项目ID
-)
-
-var (
-	Sysconfig                map[string]interface{} //读取配置文件
-	MongoTool                mongodb.MongodbSim     //mongodb连接
-	ExtractColl, ProjectColl string                 //抽取表、项目表
-	CurrentMegerTime         int64                  //当前合并到的信息的时间
-	CurrentMegerCount        int                    //当前合并计数
-	MultiThread              = make(chan bool, 5)   //项目合并线程
-	AllIdsMap2               = map[string]*ID{}
-	AllIdsMapLock            = sync.Mutex{}
-)
-
-var (
-	//判断是日期
-	_datereg   = regexp.MustCompile("20[0-2][0-9][年-][0-9]{1,2}[月-][0-9]{1,2}[日-]([0-9]{1,2}时[0-9]{0,2})?")
-	_numreg1   = regexp.MustCompile("^[0-9-]{1,8}$")
-	_zimureg1  = regexp.MustCompile("^[a-zA-Z-]{1,7}$")
-	_nzreg     = regexp.MustCompile("^[0-9a-zA-Z-]+$")
-	_hanreg    = regexp.MustCompile(`^[\p{Han}:：【】\\[\\]（）()－-、]+$`)
-	replaceStr = regexp.MustCompile("(工程|采购|项目|[?！、【】（）—()－-]|栏标价|中标候选人|招标代理)")
-	//判断带有分包、等特定词的
-	pStr = regexp.MustCompile("(勘察|监理|施工|设计|验收|标段|分包|子包|[0-9A-Z]包|[一二三四五六七八九十0-9]批)")
-	//判断包含数值
-	nreg1 = regexp.MustCompile("[0-9]{2,}")
-	//判断包含字母
-	zreg1 = regexp.MustCompile("[a-zA-Z]{1,}")
-	//判断包含汉字
-	hreg1 = regexp.MustCompile(`[\p{Han}]+`)
-	//判断项目编号是在10以内的纯数字结构
-	numCheckPc = regexp.MustCompile("^[0-9-]{1,10}$")
-
-	//存放项目名称
-	mapPn = map[string]*Key{}
-	//存放项目编号
-	mapPc = map[string]*Key{}
-	//存放采购单位
-	mapPb = map[string]*Key{}
-	//仅初始化使用
-	compareNoPass = map[string]bool{}
-	compareAB     = map[string]bool{}
-	compareAB2D   = map[string]bool{}
-	compareABD    = map[string]bool{}
-	compareAB2CD  = map[string]bool{}
-	compareABCD   = map[string]bool{}
-)
-
-func init() {
-	util.ReadConfig(&Sysconfig)
-	MultiThread = make(chan bool, util.IntAllDef(Sysconfig["thread"], 5))
-	MongoTool = mongodb.MongodbSim{
-		MongodbAddr: Sysconfig["mongodbServers"].(string),
-		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
-		DbName:      Sysconfig["mongodbName"].(string),
-	}
-	MongoTool.InitPool()
-	ExtractColl = Sysconfig["extractColl"].(string)
-	ProjectColl = Sysconfig["projectColl"].(string)
-
-	udpport, _ := Sysconfig["udpport"].(string)
-	udpclient = mu.UdpClient{Local: udpport, BufSize: 1024}
-	udpclient.Listen(processUdpMsg)
-	log.Println("Udp服务监听", udpport)
-
-	//---不能通过
-	vm := []string{"C", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				compareNoPass[key] = true
-				//fmt.Println(key)
-			}
-		}
-	}
-	//fmt.Println("-------------------")
-
-	//三个元素一致 [AB][AB][AB]，分值最高
-	vm = []string{"A", "B"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				compareAB[key] = true
-				//fmt.Println(key)
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB))
-	//---至少两个一致，其他可能不存在
-	//[AB][AB][ABD]
-	//[AB][ABD][AB]
-	vm = []string{"A", "B"}
-	vm2 := []string{"A", "B", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 3; k++ {
-				key := vm[i] + vm[j] + vm2[k]
-				if !compareAB[key] {
-					compareAB2D[key] = true
-					//fmt.Println(key)
-
-				}
-			}
-		}
-	}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 3; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm2[j] + vm[k]
-				if !compareAB[key] {
-					compareAB2D[key] = true
-					//fmt.Println(key)
-
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB2D))
-	//---至少一个一致，其他可能不存在
-	//[ABD][ABD][ABD] //已经删除DDD
-	vm = []string{"A", "B", "D"}
-	for i := 0; i < 3; i++ {
-		for j := 0; j < 3; j++ {
-			for k := 0; k < 3; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] {
-					compareABD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareABD))
-
-	//[AB][ABCD][AB]
-	//[AB][AB][ABCD]
-	vm = []string{"A", "B"}
-	vm2 = []string{"A", "B", "C", "D"}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 4; j++ {
-			for k := 0; k < 2; k++ {
-				key := vm[i] + vm2[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-					compareAB2CD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	for i := 0; i < 2; i++ {
-		for j := 0; j < 2; j++ {
-			for k := 0; k < 4; k++ {
-				key := vm[i] + vm[j] + vm2[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
-					compareAB2CD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-	//fmt.Println("-------------------", len(compareAB2CD))
-	//[ABECD][ABECD][ABECD]  //已经删除[CD][CD][CD]   //这个要重点讨论
-	vm = []string{"A", "B", "C", "D"}
-	for i := 0; i < 4; i++ {
-		for j := 0; j < 4; j++ {
-			for k := 0; k < 4; k++ {
-				key := vm[i] + vm[j] + vm[k]
-				if !compareAB[key] && !compareAB2D[key] && !compareABD[key] && !compareNoPass[key] && !compareAB2CD[key] {
-					compareABCD[key] = true
-					//fmt.Println(key)
-				}
-			}
-		}
-	}
-}
-
-func CheckHanAndNum(str string) (b bool) {
-	return nreg1.MatchString(str) && hreg1.MatchString(str)
-}
-func CheckZimuAndNum(str string) (b bool) {
-	return zreg1.MatchString(str) && nreg1.MatchString(str)
-}
-
-type KeyMap struct {
-	Lock sync.Mutex
-	Map  map[string]*Key
-}
-
-type ID struct {
-	Id       string
-	Lock     sync.Mutex
-	lastTime int64
-	pos      int
-	P        *ProjectInfo
-}
-type Key struct {
-	Arr  []string
-	Lock sync.Mutex
-}
-type IdAndLock struct {
-	Id   string
-	Lock sync.Mutex
-}
-
-func NewKeyMap() *KeyMap {
-	return &KeyMap{
-		Map:  map[string]*Key{},
-		Lock: sync.Mutex{},
-	}
-}
-
-//招标信息实体类
-type Info struct {
-	Id            string                 `json:"_id"`
-	Href          string                 `json:"href"` //源地址
-	Publishtime   int64                  `json:"publishtime"`
-	Title         string                 `json:"title"`
-	TopType       string                 `json:"toptype"`
-	SubType       string                 `json:"subtype"`
-	ProjectName   string                 `json:"projectname"`
-	ProjectCode   string                 `json:"projectcode"`
-	Buyer         string                 `json:"buyer"`
-	Buyerperson   string                 `json:"buyerperson"`
-	Buyertel      string                 `json:"buyertel"`
-	Agency        string                 `json:"agency"`
-	Area          string                 `json:"area"`
-	City          string                 `json:"city"`
-	District      string                 `json:"district"`
-	HasPackage    bool                   `json:"haspackage"`
-	Package       map[string]interface{} `json:"package"`
-	PNum          string                 `json:"pnum"`
-	Topscopeclass []string               `json:"topscopeclass"`
-	Subscopeclass []string               `json:"subscopeclass"`
-	Buyerclass    string                 `json:"buyerclass"`
-	Bidopentime   int64                  `json:"bidopentime"`
-	Budget        float64                `json:"budget"`
-	Bidamount     float64                `json:"bidamount"`
-	Winners       []string
-	dealtype      int
-
-	Winnerorder []string
-
-	PTC    string //从标题中抽的项目编号
-	pnbval int    //项目名称、编号、采购单位存在的个数
-	LenPC  int    //项目编号长度
-	LenPN  int    //项目名称长度
-	LenPTC int    //标题抽的项目编号长度
-	//以下三个元素做对比，计算包含时候使用
-	PNBH  int //0初始，+包含，-被包含
-	PCBH  int
-	PTCBH int
-}
-
-//项目实体类
-type ProjectInfo struct {
-	Id            bson.ObjectId          `bson:"_id"`
-	FirstTime     int64                  `json:"firsttime"` //项目的最早时间
-	LastTime      int64                  `json:"lasttime"`  //项目的最后时间
-	Ids           []string               `json:"ids"`
-	Topscopeclass []string               `json:"topscopeclass"`
-	Subscopeclass []string               `json:"subscopeclass"` //子行业分类
-	Winners       []string               `json:"winners"`       //中标人
-	ProjectName   string                 `json:"projectname"`   //项目名称
-	ProjectCode   string                 `json:"projectcode"`   //项目代码唯一（纯数字的权重低）
-	Buyer         string                 `json:"buyer"`         //采购单位唯一
-	MPN           []string               `json:"mpn"`           //合并后多余的项目名称
-	MPC           []string               `json:"mpc"`           //合并后多余的项目编号
-	Buyerperson   string                 `json:"buyerperson"`   //采购联系人
-	Buyertel      string                 `json:"buyertel"`      //采购联系人电话
-	Agency        string                 `json:"agency"`        //代理机构
-	Area          string                 `json:"area"`          //地区
-	City          string                 `json:"city"`          //地市
-	District      string                 `json:"district"`      //区县
-	HasPackage    bool                   `json:"haspackage"`    //是否有分包
-	Package       map[string]interface{} `json:"package"`       //分包的对比对象
-	Buyerclass    string                 `json:"buyerclass"`    //采购单位分类
-	Bidopentime   int64                  `json:"bidopentime"`   //开标时间
-	Zbtime        int64                  `json:"zbtime"`        //招标时间
-	Jgtime        int64                  `json:"jgtime"`        //结果中标时间
-	Bidamount     float64                `json:"bidamount"`     //中标金额
-	Budget        float64                `json:"budget"`        //预算
-	Winnerorder   []string               `json:"winnerorder"`   //中标候选人
-	score         int
-	comStr        string
-}

fullproject/src_dev4/load_data.go

@@ -1,65 +0,0 @@
-package main
-
-import (
-	"time"
-	//"encoding/json"
-	"log"
-)
-
-//初始加载数据，默认加载最近6个月的数据
-
-func loadData(projectColl string, month int, bCacheRedis bool) {
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
-
-	q := map[string]interface{}{}
-	it := sess.DB(MongoTool.DbName).C(projectColl).Find(&q).Iter()
-	AllIdsMapLock.Lock()
-	tmp := &ProjectInfo{}
-	n := 0
-	for it.Next(tmp) {
-		n++
-		if n%1000 == 0 {
-			log.Println("current", n, "\n", tmp.Id, tmp)
-			time.Sleep(2 * time.Second)
-		}
-		for _, v := range append([]string{tmp.ProjectName}, tmp.MPN...) {
-			if v != "" {
-				k := mapPn[v]
-				if k == nil {
-					k = &Key{Arr: []string{}}
-					mapPn[v] = k
-				}
-				k.Arr = append(k.Arr, tmp.Id.Hex())
-			}
-		}
-		for _, v := range append([]string{tmp.ProjectCode}, tmp.MPC...) {
-			if v != "" {
-				k := mapPc[v]
-				if k == nil {
-					k = &Key{Arr: []string{}}
-					mapPc[v] = k
-				}
-				k.Arr = append(k.Arr, tmp.Id.Hex())
-			}
-		}
-		if tmp.Buyer != "" {
-			k := mapPb[tmp.Buyer]
-			if k == nil {
-				k = &Key{Arr: []string{}}
-				mapPb[tmp.Buyer] = k
-			}
-			k.Arr = append(k.Arr, tmp.Id.Hex())
-		}
-
-		AllIdsMap2[tmp.Id.Hex()] = &ID{Id: tmp.Id.Hex(), lastTime: tmp.LastTime, P: tmp}
-
-		if bCacheRedis {
-			//存入redis
-
-		}
-
-	}
-	AllIdsMapLock.Unlock()
-	log.Println("load over")
-}

fullproject/src_dev4/main.go

@@ -1,68 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"log"
-	mu "mfw/util"
-	"net"
-	"time"
-)
-
-var (
-	udpclient    mu.UdpClient //udp对象
-	SingleThread = make(chan bool, 1)
-)
-
-func main() {
-
-	//udp跑增量  id段   zl
-	//udp跑全量			ql
-	//udp跑历史数据  信息id1,id2/或id段  ls
-	//udp强制合并  信息id1,id2,id3 [项目id] 不存在时新建  qzhb
-	//udp强制拆分  项目id,信息id1,id2          qzcf
-	//udp重新合并  信息id1,id2,id3             cxhb
-	//loadData("project_0809", 1, true)
-	time.Sleep(99999 * time.Hour)
-}
-
-func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
-	switch act {
-	case mu.OP_TYPE_DATA: //上个节点的数据
-		var mapInfo map[string]interface{}
-		err := json.Unmarshal(data, &mapInfo)
-		log.Println("err:", err, "mapInfo:", mapInfo)
-		if err != nil {
-			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
-		} else if mapInfo != nil {
-			key, _ := mapInfo["key"].(string)
-			if key == "" {
-				key = "udpok"
-			}
-			go udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
-			SingleThread <- true
-			tasktype, _ := mapInfo["stype"].(string)
-			log.Println("tasktype:", tasktype)
-			switch tasktype {
-			case "ql":
-				go func() {
-					defer func() {
-						<-SingleThread
-					}()
-					taskQl(mapInfo)
-				}()
-			case "ids":
-				go func() {
-					defer func() {
-						<-SingleThread
-					}()
-					Ids(mapInfo)
-				}()
-			}
-		}
-	case mu.OP_NOOP: //下个节点回应
-		ok := string(data)
-		if ok != "" {
-			log.Println("ok:", ok)
-		}
-	}
-}

fullproject/src_dev4/merge.go

@@ -1,406 +0,0 @@
-package main
-
-import (
-	"log"
-	"math"
-	qu "qfw/util"
-	"sort"
-	//"strconv"
-	"strings"
-	"sync"
-)
-
-//单线程控制从map查找可对比的项目
-var findLock = sync.Mutex{}
-
-//从对应map中获取对比的项目id
-func getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
-	findLock.Lock()
-	defer findLock.Unlock()
-	wg.Add(1)
-	res = []*Key{}
-	//是否查找到，并标识位置。-1代表未查找到。
-	bpn, bpc, bptc, bpb = -1, -1, -1, -1
-	if pn != "" {
-		ids := mapPn[pn]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPn[pn] = ids
-			bpn = 0
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-
-	if pc != "" {
-		ids := mapPc[pc]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPc[pc] = ids
-			bpc = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-
-	if ptc != "" {
-		ids := mapPc[ptc]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPc[ptc] = ids
-			bptc = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-
-	if pb != "" {
-		ids := mapPb[pb]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPb[pb] = ids
-			bpb = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-	repeatId := map[string]bool{}
-	idArr = []string{} //项目id
-	IDArr = []*ID{}    //项目信息
-	for _, m := range res {
-		for _, id := range m.Arr {
-			if !repeatId[id] {
-				repeatId[id] = true
-				//_, _ = strconv.ParseInt(id[0:8], 16, 64)
-				AllIdsMapLock.Lock()
-				Id := AllIdsMap2[id]
-				AllIdsMapLock.Unlock()
-				if Id != nil {
-					Id.Lock.Lock()
-					idArr = append(idArr, id)
-					IDArr = append(IDArr, Id)
-				}
-			}
-		}
-	}
-	return
-}
-
-//项目合并入口
-func startProjectMerge(info *Info, tmp map[string]interface{}) {
-	//只有或没有采购单位的无法合并
-	//bpn, bpc, bptc, bpb 是否查找到，并标识位置。-1代表未查找到。
-	//pids 是项目id数组集合
-	//IDArr，是单个项目ID对象集合
-	bpn, bpc, bptc, bpb, pids, _, IDArr := getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer)
-	defer wg.Done()
-	//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock
-
-	for _, m := range pids {
-		defer m.Lock.Unlock()
-	}
-	for _, id := range IDArr {
-		defer id.Lock.Unlock()
-	}
-
-	bFindProject := false
-	findPid := ""
-	//获取完id，进行计算
-	//定义两组
-	comRes1 := []*ProjectInfo{} //优先级最高的对比结果数组
-	comRes2 := []*ProjectInfo{} //优化级其次
-	comRes3 := []*ProjectInfo{}
-	resVal, pjVal := 0, 0
-	for _, v := range IDArr {
-		comStr := ""
-		compareProject := v.P
-		//问题出地LastTime!!!!!
-		diffTime := math.Abs(float64(info.Publishtime - compareProject.LastTime))
-		if diffTime < 185*86400 {
-			//"A 相等 	B 被包含 	C 不相等	 	D不存在  E被包含
-			info.PNBH = 0
-			info.PCBH = 0
-			info.PTCBH = 0
-			compareStr, score := comparePNCB(info, compareProject)
-
-			resVal, pjVal = Select(compareStr, info, compareProject)
-			//---------------------------------------
-
-			if resVal > 0 {
-
-				compareBuyer := "D"
-				if info.Buyer != "" {
-					if info.Buyer == compareProject.Buyer {
-						compareBuyer = "A"
-						score += 3
-					} else if compareProject.Buyer != "" {
-						if strings.Contains(info.Buyer, compareProject.Buyer) || strings.Contains(compareProject.Buyer, info.Buyer) {
-							compareBuyer = "B"
-							score += 1
-						} else {
-							compareBuyer = "C"
-						}
-					}
-				}
-
-				//---------------------------------------
-
-				compareCity := ""
-				if info.Area != "全国" && info.Area != "" && info.Area == compareProject.Area {
-					compareCity += "A"
-					score += 2
-				} else if info.Area == "全国" || compareProject.Area == "全国" {
-					compareCity += "B"
-					score += 1
-				} else {
-					compareCity += "C"
-				}
-				if compareCity != "C" {
-					if info.City != "" && info.City == compareProject.City {
-						compareCity += "A"
-						score += 2
-					} else {
-						if info.Area == "全国" || compareProject.Area == "全国" {
-							compareCity += "B"
-						} else if info.City == compareCity {
-							compareCity += "B"
-						} else {
-							compareCity += "C"
-						}
-					}
-				} else {
-					compareCity += "C"
-				}
-				score2 := 0
-				if compareCity == "AA" {
-					if info.District != "" && info.District == compareProject.District {
-						score2 = 1
-					}
-				}
-
-				compareTime := "D"
-				if diffTime < 45*86400 {
-					compareTime = "A"
-					score += 2
-				} else if diffTime < 90*86400 {
-					compareTime = "B"
-					score += 1
-				}
-
-				compareAgency := "D"
-				if info.Agency != "" {
-					if info.Agency == compareProject.Agency {
-						compareAgency = "A"
-						score += 2
-						score2 += 1
-					} else if compareProject.Agency != "" {
-						if strings.Contains(info.Agency, compareProject.Agency) || strings.Contains(compareProject.Agency, info.Agency) {
-							compareAgency = "B"
-							score += 1
-							score2 += 1
-						} else {
-							compareAgency = "C"
-						}
-					}
-				}
-				compareBudget := "C"
-				if info.Budget > 0 && (info.Budget == compareProject.Budget || (compareProject.Bidamount > 0 && info.Budget > compareProject.Bidamount && (info.Budget-compareProject.Bidamount) < (0.1*info.Budget))) {
-					compareBudget = "A"
-					score += 1
-					score2 += 1
-				} else if info.Budget == 0 && compareProject.Budget == 0 {
-					compareBudget = "B"
-				}
-				compareBidmount := "C"
-				if info.Bidamount > 0 && (info.Bidamount == compareProject.Bidamount || (compareProject.Budget > 0 && compareProject.Budget > info.Bidamount && (compareProject.Budget-info.Bidamount) < 0.1*compareProject.Budget)) {
-					compareBidmount = "A"
-					score += 1
-					score2 += 1
-				} else if info.Bidamount == 0 && compareProject.Bidamount == 0 {
-					compareBidmount = "B"
-				}
-
-				//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构
-
-				comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
-				compareProject.comStr = comStr
-				eqV := 0
-				switch resVal {
-				case 3:
-					if pjVal == 3 && comStr[3:] != "CCCDCCC" {
-						eqV = 1
-					} else if compareBuyer < "C" {
-						if pjVal > 1 {
-							eqV = 1
-						} else { //if (compareCity[1:1] != "C" || compareTime != "D") && score2 > 0
-							eqV = 2
-						}
-					} else if compareBuyer == "D" {
-						if pjVal > 1 && (compareCity[1:1] != "C" || score2 > 0) {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
-							eqV = 3
-						}
-					} else {
-						if pjVal == 3 && (score2 > 0 || compareCity[1:1] != "C") {
-							eqV = 2
-						} else if pjVal == 2 && compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
-							eqV = 3
-						} else if compareCity == "AA" && compareTime == "A" && score2 > 0 {
-							eqV = 3
-						}
-					}
-				case 2:
-					if compareBuyer < "C" {
-						if pjVal > 1 {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" || score2 > 0 {
-							eqV = 3
-						}
-						//						if compareTime != "D" && "AA" == compareCity && compareAgency != "C" {
-						//							eqV = 2
-						//						} else { //if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0
-						//							eqV = 3
-						//						}
-					} else if compareBuyer == "D" {
-						if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
-							eqV = 3
-						}
-						//						if "AA" == compareCity && compareTime != "D" && score2 > 0 {
-						//							eqV = 2
-						//						} else if compareCity[1:1] != "C" && score2 > 0 && compareTime == "A" {
-						//							eqV = 3
-						//						}
-					} else {
-						if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 1) {
-							eqV = 3
-						}
-						//						if "AA" == compareCity && (compareAgency == "A" || score2 > 1) && compareTime == "A" {
-						//							eqV = 3
-						//						}
-					}
-				case 1:
-					if compareBuyer < "C" {
-						if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 1) {
-							eqV = 3
-						}
-						//						if compareTime != "D" && "AA" == compareCity && score2 > 1 {
-						//							eqV = 2
-						//						} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 1 {
-						//							eqV = 3
-						//						}
-					} else if compareBuyer == "D" {
-						if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 1) {
-							eqV = 3
-						}
-						//						if "AA" == compareCity && score2 > 1 && compareTime == "A" {
-						//							eqV = 3
-						//						}
-					} else {
-						if pjVal > 1 && compareTime == "A" && score2 > 1 && compareCity[1:1] != "C" {
-							eqV = 3
-						}
-					}
-				}
-				if eqV == 1 {
-					comRes1 = append(comRes1, compareProject)
-				} else if eqV == 2 {
-					comRes2 = append(comRes2, compareProject)
-				} else if eqV == 3 {
-					comRes3 = append(comRes3, compareProject)
-				} else if resVal == 3 || pjVal > 1 {
-					log.Println("===", resVal, pjVal, comStr, info.ProjectCode, compareProject.ProjectCode,
-						info.ProjectName, compareProject.ProjectName, info.Buyer, compareProject.Buyer, info.Id, compareProject.Id.Hex())
-				}
-			}
-		}
-	}
-	//--------------------------------对比完成-----------------------
-	//更新数组、更新项目
-	for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} {
-		if len(resN) > 0 {
-			if len(resN) > 1 {
-				sort.Slice(resN, func(i, j int) bool {
-					return resN[i].score > resN[j].score
-				})
-			}
-
-			bFindProject = true
-			findPid = resN[0].Id.Hex()
-			for k2, bv := range []int{bpn, bpc, bptc, bpb} {
-				if bv > -1 {
-					pids[bv].Arr = append(pids[bv].Arr, findPid)
-					if k2 == 0 {
-						if resN[0].ProjectName == "" {
-							resN[0].ProjectName = info.ProjectName
-						} else {
-							if resN[0].MPN == nil {
-								resN[0].MPN = []string{info.ProjectName}
-							} else {
-								resN[0].MPN = append(resN[0].MPN, info.ProjectName)
-							}
-						}
-
-					} else if k2 < 3 {
-						if resN[0].ProjectCode == "" {
-							resN[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
-						} else {
-							if resN[0].MPC == nil {
-								resN[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
-							} else {
-								resN[0].MPC = append(resN[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
-							}
-						}
-
-					} else {
-						if resN[0].Buyer == "" {
-							resN[0].Buyer = info.Buyer
-						}
-					}
-				}
-			}
-			UpdateProject(tmp, info, resN[0], kv+1, resN[0].comStr, resVal, pjVal)
-			//更新AllIdsMao中的时间
-			break
-		}
-	}
-
-	if !bFindProject {
-		//没有找到
-		id, p := NewProject(tmp, info)
-		//_, _ = strconv.ParseInt(id[0:8], 16, 64)
-		AllIdsMapLock.Lock()
-		//AllIdsMap[int(t)%idsMapSize][id] = &ID{Id: id, lastTime: info.Publishtime}
-		AllIdsMap2[id] = &ID{Id: id, lastTime: info.Publishtime, P: p}
-		AllIdsMapLock.Unlock()
-		for _, m := range pids {
-			m.Arr = append(m.Arr, id)
-		}
-	}
-
-}
-
-//二分字符串查找
-func BinarySearch(s []string, k string) int {
-	sort.Strings(s)
-	lo, hi := 0, len(s)-1
-	for lo <= hi {
-		m := (lo + hi) >> 1
-		if s[m] < k {
-			lo = m + 1
-		} else if s[m] > k {
-			hi = m - 1
-		} else {
-			return m
-		}
-	}
-	return -1
-}

fullproject/src_dev4/merge_comparepncb.go

@@ -1,182 +0,0 @@
-package main
-
-import (
-	"strings"
-)
-
-func comparePNCB(info *Info, compareProject *ProjectInfo) (compareStr string, score int) {
-	if info.ProjectName != "" {
-		pns := []string{}
-		if compareProject.ProjectName != "" {
-			pns = append(pns, compareProject.ProjectName)
-		}
-		if len(compareProject.MPN) > 0 {
-			pns = append(pns, compareProject.MPN...)
-		}
-		ifind := 0
-		templen := 0
-		for _, v := range pns {
-			if info.ProjectName == v {
-				ifind = 1
-				break
-			} else {
-				//if strings.Contains(info.ProjectName, v) || strings.Contains(v, info.ProjectName) ||
-				retv := CheckContain(info.ProjectName, v)
-				if retv == 1 {
-					ifind = 1
-					break
-				} else if retv == 2 {
-					templen = len([]rune(v))
-					ifind = 2
-				} else if ifind == 0 {
-					ifind = 3
-				}
-			}
-		}
-		switch ifind {
-		case 0:
-			compareStr = "D"
-		case 1:
-			compareStr = "A"
-			score += 4
-			if len([]rune(info.ProjectName)) > 18 {
-				score += 2
-			}
-		case 2:
-			compareStr = "B"
-			score += 2
-			if templen > info.LenPN {
-				templen = info.LenPN
-			}
-			info.PNBH = templen
-			if templen > 12 {
-				score += 1
-			}
-		case 3:
-			compareStr = "C"
-		}
-	} else {
-		compareStr = "D"
-	}
-
-	/*
-				项目编号 - －()（） 要注意
-				init_text = ["号","(重)","(第二次)","（重）"]
-		all_clean_mark = ["[","(","【","（","〖","]",")","】","）","〗","-","〔","〕","《","［","］","｛","｝","{","—"," ","－","﹝","﹞","–"]
-	*/
-	for index, pc := range []string{info.ProjectCode, info.PTC} {
-		if pc != "" {
-			pcs := []string{}
-			if compareProject.ProjectCode != "" {
-				pcs = append(pcs, compareProject.ProjectCode)
-			}
-			if len(compareProject.MPC) > 0 {
-				pcs = append(pcs, compareProject.MPC...)
-			}
-			ifind := 0
-			templen := 0
-			for _, v := range pcs {
-				if pc == v {
-					ifind = 1
-					break
-				} else {
-					// math.Abs(float64(len([]rune(pc))-len([]rune(v)))) < 6
-					//if !_numreg1.MatchString(pc) && !_zimureg1.MatchString(pc) && !_numreg1.MatchString(v) && !_zimureg1.MatchString(v)
-					if strings.Contains(pc, v) || strings.Contains(v, pc) {
-						t1 := pc
-						t2 := v
-						if len(v) > len(pc) {
-							t1 = v
-							t2 = pc
-						}
-						t3 := strings.Replace(t1, t2, "", -1)
-						t3 = _datereg.ReplaceAllString(t3, "")
-						if t3 == "" {
-							ifind = 1
-							break
-						} else {
-							ifind = 2
-							templen = len([]rune(v))
-						}
-					} else if ifind == 0 {
-						ifind = 3
-					}
-				}
-			}
-			switch ifind {
-			case 0:
-				compareStr += "D"
-			case 1:
-				compareStr += "A"
-				score += 4
-				if len([]rune(pc)) > 18 {
-					score += 2
-				}
-			case 2:
-				compareStr += "B"
-				score += 2
-				if index == 0 {
-					if templen > info.LenPC {
-						templen = info.LenPC
-					}
-					info.PCBH = templen
-					if templen > 12 {
-						score += 1
-					}
-
-				} else {
-					if templen > info.LenPTC {
-						templen = info.LenPTC
-					}
-					info.PTCBH = templen
-					if templen > 12 {
-						score += 1
-					}
-				}
-
-			case 3:
-				compareStr += "C"
-			}
-
-		} else {
-			compareStr += "D"
-		}
-	}
-	return
-}
-
-func CheckContain(b1, b2 string) (res int) {
-	b1 = replaceStr.ReplaceAllString(b1, "")
-	b2 = replaceStr.ReplaceAllString(b2, "")
-
-	if b1 == b2 {
-		res = 1 //相等
-		return
-	}
-	bs1 := []rune(b1)
-	bs2 := []rune(b2)
-	tmp := ""
-	for i := 0; i < len(bs1); i++ {
-		for j := 0; j < len(bs2); j++ {
-			if bs1[i] == bs2[j] {
-				tmp += string(bs1[i])
-			} else if tmp != "" {
-				b1 = strings.Replace(b1, tmp, "", -1)
-				b2 = strings.Replace(b2, tmp, "", -1)
-				tmp = ""
-			}
-		}
-	}
-	if tmp != "" {
-		b1 = strings.Replace(b1, tmp, "", -1)
-		b2 = strings.Replace(b2, tmp, "", -1)
-	}
-	if b1 == b2 {
-		res = 1 //相等
-	} else if b1 == "" || b2 == "" {
-		res = 2 //包含
-	} else {
-		res = 3 //不相等
-	}
-	return
-}

fullproject/src_dev4/merge_select.go

@@ -1,508 +0,0 @@
-package main
-
-func Select(compareStr string, info *Info, compareInfo *ProjectInfo) (res, pj int) {
-	//没有可对比的项目名称、或项目编号 //评级
-	if compareNoPass[compareStr] {
-
-	} else {
-		switch compareStr {
-		case "AAA":
-			res = 3
-			pj = 3
-		case "AAB":
-			res = 3
-			pj = 3
-		case "ABA":
-			res = 3
-			pj = 3
-		case "ABB":
-			if info.LenPN > 10 || info.PCBH > 8 || info.PTCBH > 8 {
-				res = 3
-			} else {
-				res = 2
-			}
-			pj = 3
-		case "BAA":
-			if info.PNBH > 10 || info.LenPC > 8 || info.LenPTC > 8 {
-				res = 3
-			} else {
-				res = 2
-			}
-			pj = 3
-		case "BAB":
-			if info.PNBH > 10 || info.LenPTC > 8 || info.PTCBH > 8 {
-				res = 3
-			} else {
-				res = 2
-			}
-			pj = 3
-		case "BBA":
-			if info.PNBH > 10 || info.PCBH > 8 || info.LenPC > 8 {
-				res = 3
-			} else {
-				res = 2
-			}
-			pj = 3
-		case "BBB":
-			v := 0
-			if info.PNBH > 10 {
-				v++
-			}
-			if info.PCBH > 8 {
-				v++
-			}
-			if info.PTCBH > 8 {
-				v++
-			}
-			if v > 1 {
-				res = 3
-			} else {
-				res = 2
-			}
-			pj = 2
-		case "AAD":
-			if info.LenPC > 8 || info.LenPN > 12 {
-				res = 3
-			} else {
-				res = 2
-			}
-			pj = 3
-		case "ABD":
-			if info.LenPN > 10 && info.PCBH > 8 {
-				res = 3
-				pj = 2
-			} else if info.LenPN > 10 || info.PCBH > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "BAD":
-			if info.LenPC > 13 || (info.PNBH > 10 && info.LenPC > 8) {
-				res = 3
-				pj = 3
-			} else if info.PNBH > 10 || info.LenPC > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "BBD":
-			if info.PNBH > 12 && info.PCBH > 10 {
-				res = 3
-				pj = 1
-			} else if info.PNBH > 10 && info.PCBH > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "ADA":
-			if info.LenPN > 12 || (info.LenPTC > 8 && !StrOrNum2.MatchString(info.PTC)) {
-				res = 3
-			} else {
-				res = 2
-			}
-			pj = 2
-		case "ADB":
-			if info.LenPN > 10 && info.PTCBH > 8 && !StrOrNum2.MatchString(info.PTC) {
-				res = 3
-				pj = 2
-			} else if info.LenPN > 10 || info.PTCBH > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "BDA":
-			if info.PNBH > 10 && info.LenPTC > 8 && !StrOrNum2.MatchString(info.PTC) {
-				res = 3
-				pj = 2
-			} else if info.PNBH > 10 || info.LenPTC > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "BDB":
-			if info.PNBH > 12 && info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
-				res = 3
-			} else if info.PNBH > 10 && info.PTCBH > 8 {
-				res = 2
-			} else {
-				res = 1
-			}
-			pj = 2
-		case "ADD":
-			if info.LenPN > 18 {
-				res = 3
-				pj = 2
-			} else if info.LenPN > 10 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 2
-			}
-		case "BDD":
-			if info.PNBH > 10 {
-				res = 2
-			} else {
-				res = 1
-			}
-			pj = 1
-		case "DAA":
-			if info.LenPTC > 8 || info.LenPC > 8 {
-				res = 3
-				pj = 2
-			} else {
-				res = 2
-				pj = 3
-			}
-		case "DAB":
-			if info.LenPC > 8 && info.PTCBH > 8 {
-				res = 3
-				pj = 2
-			} else if info.LenPC > 8 || info.PTCBH > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "DAD":
-			if info.LenPC > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
-				res = 3
-				pj = 2
-			} else if info.LenPC > 8 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 2
-			}
-		case "DBA":
-			if info.PCBH > 8 && info.LenPC > 8 && !StrOrNum2.MatchString(info.PTC) {
-				res = 3
-				pj = 2
-			} else if info.PCBH > 8 || info.LenPC > 8 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "DBB":
-			if info.PCBH > 10 && info.PTCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
-				res = 3
-				pj = 1
-			} else if info.PCBH > 8 && info.PTCBH > 8 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "DBD":
-			if info.PCBH > 12 && !StrOrNum2.MatchString(info.ProjectCode) {
-				res = 2
-				pj = 1
-			} else {
-				res = 1
-				pj = 1
-			}
-		case "DDA":
-			if info.LenPTC > 14 && !StrOrNum2.MatchString(info.PTC) {
-				res = 3
-				pj = 1
-			} else if info.LenPTC > 8 {
-				res = 2
-				pj = 1
-			} else {
-				res = 1
-				pj = 2
-			}
-		case "DDB":
-			if info.PTCBH > 12 && !StrOrNum2.MatchString(info.PTC) {
-				res = 2
-			} else {
-				res = 1
-			}
-			pj = 1
-		case "ACA":
-			if info.LenPN > 10 && info.LenPTC > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
-				res = 3
-				pj = 2
-			} else if info.LenPN > 10 || info.LenPTC > 8 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "ACB":
-			if info.LenPN > 10 && info.PTCBH > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
-				res = 3
-				pj = 2
-			} else if info.LenPN > 10 || info.PTCBH > 8 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "BCA":
-			if (info.PNBH > 10 && info.LenPTC > 8) || info.LenPTC > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
-				res = 3
-				pj = 2
-			} else if info.PNBH > 10 || info.LenPTC > 8 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "BCB":
-			if info.PNBH > 12 && info.PTCBH > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
-				res = 3
-				pj = 1
-			} else if info.PNBH > 10 || info.PTCBH > 8 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 2
-			}
-		case "AAC":
-			if (info.LenPN > 10 && info.LenPC > 8) || info.LenPN > 14 || (info.LenPC > 10 && !StrOrNum2.MatchString(info.ProjectCode)) {
-				res = 3
-				pj = 3
-			} else {
-				res = 2
-				pj = 3
-			}
-		case "ABC":
-			if info.LenPN > 14 && info.PCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
-				res = 3
-				pj = 2
-			} else if info.LenPN > 10 || info.PCBH > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "BAC":
-			if info.PNBH > 14 && info.LenPC > 8 && !StrOrNum2.MatchString(info.ProjectCode) {
-				res = 3
-				pj = 2
-			} else if info.PNBH > 10 || info.LenPC > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "BBC":
-			if info.PNBH > 14 && info.PCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
-				res = 3
-				pj = 1
-			} else if info.PNBH > 10 || info.PCBH > 8 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 2
-			}
-		case "ACC":
-			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.LenPN > 16 {
-					res = 2
-					pj = 1
-				} else {
-					res = 1
-					pj = 2
-				}
-			}
-		case "ACD":
-			//项目编号不一致
-			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.LenPN > 16 {
-
-					res = 2
-				} else {
-					res = 1
-
-				}
-				pj = 1
-			}
-
-		case "ADC":
-			if info.LenPN > 16 {
-				res = 2
-			} else {
-				res = 1
-			}
-			pj = 1
-		case "BCC":
-			//项目编号不一致
-			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.PNBH > 12 {
-					res = 1
-				}
-				pj = 1
-			}
-		case "BCD":
-			//项目编号不一致
-			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.PNBH > 8 {
-					res = 1
-				}
-				pj = 1
-			}
-		case "BDC":
-			if info.PNBH > 7 {
-				res = 1
-			}
-			pj = 1
-		case "CAA":
-			if info.LenPC > 12 || info.LenPTC > 12 {
-				res = 3
-				pj = 2
-			} else if info.LenPC > 8 || info.LenPTC > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "CAB":
-			if info.LenPC > 12 && info.PTCBH > 8 {
-				res = 3
-				pj = 2
-			} else if info.LenPC > 12 || info.PTCBH > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
-		case "CAC":
-			if info.LenPC > 9 && !StrOrNum2.MatchString(info.ProjectCode) {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 1
-			}
-		case "CAD":
-			if info.LenPC > 9 && !StrOrNum2.MatchString(info.ProjectCode) {
-				res = 2
-			} else {
-				res = 1
-			}
-			pj = 1
-		case "CBA":
-			if info.LenPTC > 14 && info.PCBH > 12 {
-				res = 3
-				pj = 2
-			} else if info.LenPTC > 12 || info.PCBH > 10 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 2
-			}
-		case "CBB":
-			if info.PCBH > 13 && info.PTCBH > 13 {
-				res = 3
-				pj = 1
-			} else if info.PCBH > 9 || info.PTCBH > 9 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 2
-			}
-		case "CBC":
-			if info.PCBH > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
-				res = 2
-			} else if info.PCBH > 5 {
-				res = 1
-			}
-			pj = 1
-		case "CBD":
-			if info.PCBH > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
-				res = 2
-			} else if info.PCBH > 5 {
-				res = 1
-			}
-			pj = 1
-		case "CCA":
-			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.LenPTC > 12 && !StrOrNum2.MatchString(info.PTC) {
-					res = 2
-				} else if info.LenPTC > 5 {
-					res = 1
-				}
-				pj = 1
-			}
-		case "CCB":
-			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
-					res = 1
-				}
-				pj = 1
-			}
-		case "CDA":
-			if info.LenPTC > 12 && !StrOrNum2.MatchString(info.PTC) {
-				res = 2
-			} else {
-				res = 1
-			}
-			pj = 1
-		case "CDB":
-			if info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
-				res = 1
-				pj = 1
-			}
-		case "DAC":
-			if info.LenPC > 13 && !StrOrNum2.MatchString(info.ProjectCode) {
-				res = 3
-			} else if info.LenPC > 8 {
-				res = 2
-			} else {
-				res = 1
-			}
-			pj = 1
-		case "DBC":
-			if info.PCBH > 8 {
-				res = 1
-			}
-			pj = 1
-		case "DCA":
-			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.LenPTC > 10 {
-					res = 2
-				} else {
-					res = 1
-				}
-				pj = 1
-			}
-		case "DCB":
-			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.PTCBH > 8 && !StrOrNum2.MatchString(info.PTC) {
-					res = 1
-				}
-				pj = 1
-			}
-		}
-
-	}
-	return
-}

fullproject/src_dev4/new_project.go

@@ -1,172 +0,0 @@
-package main
-
-import (
-	qu "qfw/util"
-	"strings"
-	"time"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-var FIELDS = []string{"area", "city", "district", "projectname", "projectcode", "buyer", "winner", "budget", "bidamount", "bidstatus", "agency", "projectscope"}
-
-func NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *ProjectInfo) {
-	set := map[string]interface{}{}
-	for _, f := range FIELDS {
-		if tmp[f] != nil {
-			set[f] = tmp[f]
-		}
-	}
-	set["s_projectname"] = thisinfo.ProjectName
-	set["createtime"] = time.Now().Unix()
-	set["sourceinfoid"] = qu.BsonIdToSId(tmp["_id"])
-	set["sourceinfourl"] = tmp["href"]
-	set["topscopeclass"] = thisinfo.Topscopeclass
-	set["subscopeclass"] = thisinfo.Subscopeclass
-	if thisinfo.Buyerperson != "" {
-		set["buyerperson"] = thisinfo.Buyerperson
-	}
-	if thisinfo.Buyertel != "" {
-		set["buyertel"] = thisinfo.Buyertel
-	}
-	if thisinfo.Buyerclass != "" {
-		set["buyerclass"] = thisinfo.Buyerclass
-	}
-	if thisinfo.Bidopentime > 0 {
-		set["bidopentime"] = thisinfo.Bidopentime
-	}
-	if len(thisinfo.Winnerorder) > 0 {
-		set["winnerorder"] = thisinfo.Winnerorder
-	}
-	s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
-	set["s_subscopeclass"] = s_subscopeclass
-	s_winner := strings.Join(thisinfo.Winners, ",")
-	set["s_winner"] = s_winner
-	if tmp["package"] != nil {
-		set["package"] = tmp["package"] //没定义优先级
-	}
-	push := NewPushInfo(tmp)
-	set["list"] = []bson.M{
-		push,
-	}
-	pId := bson.NewObjectId()
-	set["_id"] = pId
-	///id := MongoTool.Save(ProjectColl, set)
-	p1 := NewPinfo(pId, thisinfo, set)
-	return pId.Hex(), &p1
-}
-
-func NewPushInfo(tmp map[string]interface{}) bson.M {
-	res := bson.M{
-		"comeintime":  tmp["comeintime"],
-		"publishtime": tmp["publishtime"],
-		"title":       tmp["title"],
-		"toptype":     tmp["toptype"],
-		"subtype":     tmp["subtype"],
-		"infoformat":  tmp["infoformat"],
-		"infoid":      qu.BsonIdToSId(tmp["_id"]),
-		"area":        tmp["area"],
-		"city":        tmp["city"],
-		"projectname": tmp["projectname"],
-		"projectcode": tmp["projectcode"],
-		"buyer":       tmp["buyer"],
-		"href":        tmp["href"],
-	}
-	for _, k := range []string{"winner", "budget", "bidamount"} {
-		if tmp[k] != nil {
-			res[k] = tmp[k]
-		}
-	}
-	return res
-}
-
-//生成存放在redis数组中的对象
-func NewPinfo(id bson.ObjectId, thisinfo *Info, set map[string]interface{}) ProjectInfo {
-	p1 := ProjectInfo{
-		Id:            id,
-		Ids:           []string{thisinfo.Id},
-		Topscopeclass: thisinfo.Topscopeclass,
-		Subscopeclass: thisinfo.Subscopeclass,
-		Winners:       thisinfo.Winners,
-		ProjectName:   thisinfo.ProjectName,
-		ProjectCode:   thisinfo.ProjectCode,
-		Buyer:         thisinfo.Buyer,
-		Agency:        thisinfo.Agency,
-		Area:          thisinfo.Area,
-		City:          thisinfo.City,
-		District:      thisinfo.District,
-		MPN:           []string{},
-		MPC:           []string{},
-		HasPackage:    thisinfo.HasPackage,
-		Package:       map[string]interface{}{},
-		Buyerclass:    thisinfo.Buyerclass,
-		Bidopentime:   thisinfo.Bidopentime,
-		Winnerorder:   thisinfo.Winnerorder,
-		FirstTime:     thisinfo.Publishtime,
-		LastTime:      thisinfo.Publishtime,
-		Budget:        thisinfo.Budget,
-		Bidamount:     thisinfo.Bidamount,
-	}
-	if thisinfo.LenPTC > 5 {
-		p1.MPC = append(p1.MPC, thisinfo.PTC)
-	}
-	//savePool <- set
-	MongoTool.SaveByOriID(ProjectColl, set)
-
-	return p1
-}
-
-var updatePool = make(chan []map[string]interface{}, 2000)
-
-//var savePool = make(chan map[string]interface{}, 6)
-//func SaveQueue() {
-//	arr := []map[string]interface{}{}
-//	sp := make(chan bool, 3)
-//	for {
-//		select {
-//		case <-beforUpdate:
-//			if len(arr) > 0 {
-//				MongoTool.SaveBulk(ProjectColl, arr...)
-//				arr = []map[string]interface{}{}
-//			}
-//		case v := <-savePool:
-//			arr = append(arr, v)
-//			if len(arr) > 50 {
-//				sp <- true
-//				go func(arr []map[string]interface{}) {
-//					MongoTool.SaveBulk(ProjectColl, arr...)
-//					<-sp
-//				}(arr)
-//				arr = []map[string]interface{}{}
-//			}
-//		case <-time.After(80 * time.Millisecond):
-//			if len(arr) > 0 {
-//				sp <- true
-//				go func(arr []map[string]interface{}) {
-//					MongoTool.SaveBulk(ProjectColl, arr...)
-//					<-sp
-//				}(arr)
-//				arr = []map[string]interface{}{}
-//			}
-//		}
-//	}
-//}
-
-func updateQueue() {
-	arr := [][]map[string]interface{}{}
-	for {
-		select {
-		case v := <-updatePool:
-			arr = append(arr, v)
-			if len(arr) > 300 {
-				MongoTool.UpdateBulk(ProjectColl, arr...)
-				arr = [][]map[string]interface{}{}
-			}
-		case <-time.After(500 * time.Millisecond):
-			if len(arr) > 0 {
-				MongoTool.UpdateBulk(ProjectColl, arr...)
-				arr = [][]map[string]interface{}{}
-			}
-		}
-	}
-}

fullproject/src_dev4/task.go

@@ -1,222 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"log"
-	"qfw/util"
-	"regexp"
-	"strings"
-	"sync"
-	//"strings"
-	"time"
-
-	"gopkg.in/mgo.v2/bson"
-)
-
-const (
-	InitMinTime = int64(1325347200) //最小时间位置2012
-)
-
-//全量合并
-func taskQl(udpInfo map[string]interface{}) {
-	defer util.Catch()
-	//1、检查pubilshtime索引
-	db, _ := udpInfo["db"].(string)
-	if db == "" {
-		db = MongoTool.DbName
-	}
-	coll, _ := udpInfo["coll"].(string)
-	if coll == "" {
-		coll = ExtractColl
-	}
-	sess := MongoTool.GetMgoConn()
-	bcon := false
-	if sess.DB(db).C(coll).EnsureIndexKey("publishtime_1", "publishtime_-1") == nil {
-		bcon = true
-	} else {
-		log.Println("publishtime_1索引不存在")
-	}
-	MongoTool.DestoryMongoConn(sess)
-	thread := util.IntAllDef(udpInfo["thread"], 1)
-	if bcon {
-		//go SaveQueue()
-		go updateQueue()
-		go clearMem()
-		//获取起始时间
-		startTime, END := int64(0), int64(0)
-
-		sts, bres := MongoTool.Find(ExtractColl, `{}`, "publishtime", `{"publishtime":1}`, true, 0, 1)
-		if bres && sts != nil && len(*sts) == 1 {
-			startTime = util.Int64All((*sts)[0]["publishtime"])
-			sts, bres = MongoTool.Find(ExtractColl, `{}`, "-publishtime", `{"publishtime":1}`, true, 0, 1)
-			if bres && sts != nil && len(*sts) == 1 {
-				END = util.Int64All((*sts)[0]["publishtime"])
-			}
-			log.Println("查询到的起始时间", startTime, END)
-		} else {
-			return
-		}
-		startTime -= 1
-		sum := 0
-		if startTime < InitMinTime {
-			q := map[string]interface{}{
-				"publishtime": map[string]interface{}{
-					"$gt":  startTime,
-					"$lte": InitMinTime,
-				},
-			}
-			sum = Mql(q, thread, db, coll, sum)
-			startTime = InitMinTime
-		}
-		for {
-			if startTime >= END {
-				break
-			}
-			et := startTime + 50*86400
-			if et >= END {
-				et = END
-			}
-			q := map[string]interface{}{
-				"publishtime": map[string]interface{}{
-					"$gt":  startTime,
-					"$lte": et,
-				},
-			}
-			sum = Mql(q, thread, db, coll, sum)
-			startTime = et
-			time.Sleep(1 * time.Second)
-		}
-	}
-	log.Println("task over!!!")
-}
-
-var wg = sync.WaitGroup{}
-
-func Ids(udpInfo map[string]interface{}) {
-	oid := []interface{}{}
-	n1, _ := udpInfo["ids"].(string)
-	idArr := strings.Split(n1, ",")
-	for _, v := range idArr {
-		oid = append(oid, util.StringTOBsonId(v))
-	}
-	thread := util.IntAllDef(udpInfo["thread"], 1)
-	q := bson.M{"_id": bson.M{"$in": oid}}
-	go updateQueue()
-	go clearMem()
-	Mql(q, thread, MongoTool.DbName, ExtractColl, 0)
-
-}
-
-func Mql(q map[string]interface{}, thread int, db, coll string, sum int) int {
-	defer util.Catch()
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
-	query := sess.DB(db).C(coll).Find(q).Sort("publishtime").Iter()
-	pool := make(chan bool, thread)
-	count := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
-		info := ParseInfo(tmp)
-		if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
-			pool <- true
-			go func(info *Info, tmp map[string]interface{}) {
-				defer func() {
-					currentTime = info.Publishtime
-					<-pool
-				}()
-				startProjectMerge(info, tmp)
-			}(info, tmp)
-		} else {
-			//log.Println("info err:", tmp["_id"], tmp["title"], tmp["buyer"])
-		}
-		if sum%1000 == 0 {
-			log.Println("current", sum)
-		}
-		sum++
-		tmp = make(map[string]interface{})
-	}
-	//阻塞
-	for n := 0; n < thread; n++ {
-		pool <- true
-	}
-	//完成
-	log.Println("sontask over:", count, sum, q)
-	return sum
-}
-
-var (
-	titleGetPc  = regexp.MustCompile("^([-0-9a-zA-Z第号采招政询电审竞#]{8,}[-0-9a-zA-Z#]+)")
-	titleGetPc1 = regexp.MustCompile("[\\[【(（](.{0,6}(编号|编码|项号|包号|代码|标段?号)[:：为])?([-0-9a-zA-Z第号采招政询电审竞#]{5,}([\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+[\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+)?)[\\]】)）]")
-	titleGetPc2 = regexp.MustCompile("([-0-9a-zA-Z第号采政招询电审竞#]{8,}[-0-9a-zA-Z#]+)(.{0,5}公告)?$")
-	pcReplace   = regexp.MustCompile("([\\[【(（〖〔《｛﹝{](重|第?[二三四再]次.{0,4})[\\]】)）〗〕》｝﹞}])$|[\\[\\]【】()（）〖〗〔〕《》｛｝﹝﹞－{}– 　]+|(号|重|第?[二三四五再]次(招标)?)$|[ _＿]+")
-	StrOrNum    = regexp.MustCompile("^[0-9_-]{1,4}$|^[a-zA-Z_-]{1,4}$")
-	StrOrNum2   = regexp.MustCompile("^[0-9_-]+$|^[a-zA-Z_-]+$")
-)
-
-func ParseInfo(tmp map[string]interface{}) (info *Info) {
-	bys, _ := json.Marshal(tmp)
-	var thisinfo *Info
-	json.Unmarshal(bys, &thisinfo)
-	if thisinfo == nil {
-		return nil
-	}
-	if len(thisinfo.Topscopeclass) == 0 {
-		thisinfo.Topscopeclass = []string{}
-	}
-	if len(thisinfo.Subscopeclass) == 0 {
-		thisinfo.Subscopeclass = []string{}
-	}
-
-	//从标题中查找项目编号
-	res := titleGetPc.FindStringSubmatch(thisinfo.Title)
-	if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
-		thisinfo.PTC = res[1]
-	} else {
-		res = titleGetPc1.FindStringSubmatch(thisinfo.Title)
-		if len(res) > 3 && len(res[3]) > 6 && thisinfo.ProjectCode != res[3] && !numCheckPc.MatchString(res[3]) && !_zimureg1.MatchString(res[3]) {
-			thisinfo.PTC = res[3]
-		} else {
-			res = titleGetPc2.FindStringSubmatch(thisinfo.Title)
-			if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
-				thisinfo.PTC = res[1]
-			}
-		}
-	}
-
-	if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
-		thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
-		if thisinfo.ProjectName != "" {
-			thisinfo.pnbval++
-		}
-	}
-
-	if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
-		if thisinfo.ProjectCode != "" {
-			thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
-			if thisinfo.pnbval == 0 && len([]rune(thisinfo.ProjectCode)) < 5 {
-				thisinfo.ProjectCode = StrOrNum.ReplaceAllString(thisinfo.ProjectCode, "")
-			}
-		} else {
-			thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
-			if thisinfo.pnbval == 0 && len([]rune(thisinfo.PTC)) < 5 {
-				thisinfo.PTC = StrOrNum.ReplaceAllString(thisinfo.PTC, "")
-			}
-		}
-		if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
-			thisinfo.pnbval++
-		}
-	}
-	if thisinfo.ProjectCode == thisinfo.PTC || strings.Index(thisinfo.ProjectCode, thisinfo.PTC) > -1 {
-		thisinfo.PTC = ""
-	}
-
-	if thisinfo.Buyer != "" && len([]rune(thisinfo.Buyer)) > 2 {
-		thisinfo.pnbval++
-	} else {
-		thisinfo.Buyer = ""
-	}
-
-	thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
-	thisinfo.LenPTC = len([]rune(thisinfo.PTC))
-	thisinfo.LenPN = len([]rune(thisinfo.ProjectName))
-	return thisinfo
-}

fullproject/src_dev4/update_project.go

@@ -1,234 +0,0 @@
-package main
-
-import (
-	qu "qfw/util"
-	"sort"
-	"strings"
-)
-
-func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string, resVal, pjVal int) {
-	//log.Println(thisinfo.ProjectName, pInfo.ProjectName, comStr)
-	//	updateid := pInfo.Id
-	//	if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
-	//		return //updateid
-	//	}
-	set := map[string]interface{}{}
-	//res, bres := MongoTool.FindById(ProjectColl, pInfo.Id.Hex(), `{"list":0}`)
-	EqInfoUpdate(thisinfo, pInfo)
-	//if bres && res != nil && *res != nil {
-	set["topscopeclass"] = pInfo.Topscopeclass
-	set["subscopeclass"] = pInfo.Subscopeclass
-	s_subscopeclass := strings.Join(pInfo.Subscopeclass, ",")
-	if len(s_subscopeclass) > 0 {
-		s_subscopeclass = "," + s_subscopeclass + ","
-	}
-	set["s_subscopeclass"] = s_subscopeclass
-	s_winner := strings.Join(pInfo.Winners, ",")
-	if len(s_winner) > 0 {
-		s_winner = "," + s_winner + ","
-	}
-	set["s_winner"] = s_winner
-	if pInfo.Buyerperson != "" && pInfo.Buyertel != "" {
-		set["buyerperson"] = pInfo.Buyerperson
-		set["buyertel"] = pInfo.Buyertel
-	}
-	if pInfo.Buyerclass != "" {
-		set["buyerclass"] = pInfo.Buyerclass
-	}
-	if pInfo.District != "" {
-		set["district"] = pInfo.District
-	}
-	if pInfo.Bidopentime > 0 {
-		set["bidopentime"] = pInfo.Bidopentime
-	}
-	if len(pInfo.Winnerorder) > 0 {
-		set["winnerorder"] = pInfo.Winnerorder
-	}
-	if thisinfo.HasPackage {
-		set["multipackage"] = 1
-	} else {
-		set["multipackage"] = 0
-	}
-
-	if pInfo.ProjectName != "" {
-		set["projectname"] = pInfo.ProjectName
-	}
-
-	if pInfo.ProjectCode != "" {
-		set["projectcode"] = pInfo.ProjectCode
-	}
-
-	if pInfo.Buyer != "" {
-		set["buyer"] = pInfo.Buyer
-	}
-
-	//预算、中标价
-	if qu.Float64All(tmp["bidamount"]) > 0 && pInfo.Bidamount == 0 {
-		set["bidamount"] = tmp["bidamount"]
-	}
-
-	if qu.Float64All(tmp["budget"]) > 0 && pInfo.Budget == 0 {
-		set["budget"] = tmp["budget"]
-	}
-
-	set["mpn"] = pInfo.MPN
-	set["mpc"] = pInfo.MPC
-	set["area"] = pInfo.Area
-	set["city"] = pInfo.City
-
-	//e := InitEL(util.ObjToString((*res)["extractpos"]))
-	if thisinfo.dealtype == 1 {
-		var sonpackage map[string]interface{}
-		for _, obj := range tmp["package"].(map[string]interface{}) {
-			sonpackage, _ = obj.(map[string]interface{})
-		}
-		for _, v2 := range []string{"budget", "budget_w", "winner", "winner_w", "bidstatus", "bidstatus_w"} {
-			if sonpackage[v2] != nil {
-				tmp[v2] = sonpackage[v2]
-			}
-		}
-	}
-	//e.fieldpriority(&tmp, res, &set)
-	//set["extractpos"] = e.GetVal()
-	//	if thisinfo.HasPackage { //多包处理
-	//		p1, _ := (*res)["package"].(map[string]interface{})
-	//		p2, _ := tmp["package"].(map[string]interface{})
-	//		if p2 != nil {
-	//			if p1 != nil {
-	//				for pk2, pv2 := range p2 {
-	//					if p1[pk2] != nil { //合并
-	//						item1, _ := p1[pk2].(map[string]interface{})
-	//						item2, _ := pv2.(map[string]interface{})
-	//						if item1 != nil && item2 != nil { //原始项
-	//							for ik1, iv1 := range item2 {
-	//								if item1[ik1] == nil {
-	//									item1[ik1] = iv1
-	//								}
-	//							}
-	//						}
-	//					} else {
-	//						p1[pk2] = pv2
-	//					}
-	//				}
-	//			} else {
-	//				p1 = p2
-	//			}
-	//		}
-	//		set["package"] = p1
-	//	}
-	//中标候选人合并
-
-	update := map[string]interface{}{}
-	if len(set) > 0 {
-		update["$set"] = set
-	}
-	//保留原数据吧
-	push := NewPushInfo(tmp)
-	push["compareStr"] = comStr
-	push["resVal"] = resVal
-	push["pjVal"] = pjVal
-	update["$push"] = map[string]interface{}{
-		"list": push,
-	}
-	if len(update) > 0 {
-		updateInfo := []map[string]interface{}{
-			map[string]interface{}{
-				"_id": pInfo.Id,
-			},
-			update,
-		}
-		//批量更新，这块可能需要再做判重处理
-		updatePool <- updateInfo
-		//			MongoTool.Update(ProjectColl, map[string]interface{}{
-		//				"_id": qu.StringTOBsonId(pInfo.Id.Hex()),
-		//			}, &update, false, false)
-	}
-	//}
-	//再往redis中放 index
-	//往队列中增加时间 -------------->start
-}
-
-func EqInfoUpdate(thisinfo *Info, pInfo *ProjectInfo) {
-	if thisinfo.Publishtime > pInfo.LastTime {
-		pInfo.LastTime = thisinfo.Publishtime
-	}
-	if pInfo.FirstTime == 0 || (thisinfo.Publishtime < pInfo.FirstTime && thisinfo.Publishtime > 0) {
-		pInfo.FirstTime = thisinfo.Publishtime
-	}
-	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
-	//增加发布时间结束----------------->end
-
-	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
-		pInfo.Buyer = thisinfo.Buyer
-	}
-	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
-		pInfo.Agency = thisinfo.Agency
-	}
-	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
-		pInfo.ProjectCode = thisinfo.ProjectCode
-	}
-
-	if pInfo.Area == "全国" && thisinfo.Area != "全国" {
-		pInfo.Area = thisinfo.Area
-		pInfo.City = thisinfo.City
-	}
-	if thisinfo.District != "" {
-		pInfo.District = thisinfo.District
-	}
-	if thisinfo.Buyerperson != "" && thisinfo.Buyertel != "" {
-		pInfo.Buyerperson = thisinfo.Buyerperson
-		pInfo.Buyertel = thisinfo.Buyertel
-	}
-	if thisinfo.Buyerclass != "" {
-		pInfo.Buyerclass = thisinfo.Buyerclass
-	}
-	if thisinfo.Bidopentime > 0 {
-		pInfo.Bidopentime = thisinfo.Bidopentime
-	}
-	if thisinfo.Bidamount > 0 && pInfo.Bidamount < 1 {
-		pInfo.Bidamount = thisinfo.Bidamount
-	}
-	if thisinfo.Budget > 0 && pInfo.Budget < 1 {
-		pInfo.Budget = thisinfo.Budget
-	}
-
-	if len(thisinfo.Topscopeclass) > 0 {
-		sort.Strings(pInfo.Topscopeclass)
-		for _, k := range thisinfo.Topscopeclass {
-			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
-				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
-				sort.Strings(pInfo.Topscopeclass)
-			}
-		}
-	}
-
-	if len(thisinfo.Subscopeclass) > 0 {
-		sort.Strings(pInfo.Subscopeclass)
-		for _, k := range thisinfo.Subscopeclass {
-			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
-				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
-				sort.Strings(pInfo.Subscopeclass)
-			}
-		}
-	}
-	//winner
-	if len(thisinfo.Winners) > 0 {
-		sort.Strings(pInfo.Winners)
-		for _, k := range thisinfo.Winners {
-			if BinarySearch(pInfo.Winners, k) == -1 {
-				pInfo.Winners = append(pInfo.Winners, k)
-				sort.Strings(pInfo.Winners)
-			}
-		}
-	}
-	//winnerorder
-	//	if len(thisinfo.Winnerorder) > 0 {
-	//		sort.Strings(pInfo.Winnerorder)
-	//		for _, k := range thisinfo.Winnerorder {
-	//			//if BinarySearch(pInfo.Winnerorder, k) == -1 {
-	//				pInfo.Winnerorder = append(pInfo.Winnerorder, k)
-	//				sort.Strings(pInfo.Winnerorder)
-	//			//}
-	//		}
-	//	}
-}