备份-信息质量整体打分

data_quality/src1/config.json

@@ -0,0 +1,69 @@
+{
+  "udpport": ":17007",
+  "mongodb": {
+    "addrName": "172.17.4.85:27080",
+    "dbName": "qfw",
+    "collName": "zktest_score_data",
+    "pool": 10,
+    "site": {
+      "site_dbname": "qfw",
+      "site_coll": "site"
+    }
+  },
+  "qy_mongodb": {
+    "qy_addrName": "172.17.145.163:27083,172.17.4.187:27082",
+    "qy_dbName": "mixdata",
+    "qy_collName": "qyxy_std",
+    "qy_username": "zhengkun",
+    "qy_password": "zk@123123",
+    "pool": 10
+  },
+  "score_standard": {
+    "total_score": 100,
+    "core_max": 60,
+    "core_each": 10,
+    "other_max": 40,
+    "other_each": 5,
+    "deduct_each": 5
+  },
+  "core_element":[
+    {"bidamount": {"type": "float", "min": 0,"max":50000000000}},
+    {"budget": {"type": "float", "min": 0,"max":50000000000}},
+    {"projectcode": {"type": "string", "min": 4,"max": 20}},
+    {"projectname": {"type": "string", "min": 4,"max": 30}},
+    {"buyer": {"type": "string", "min":3,"max": 15}},
+    {"winner": {"type": "string", "min": 3,"max": 15}}
+  ],
+  "other_element": [
+    {"agency": {"type": "string","min": 3,"max": 15}},
+    {"bidopenaddress": {"type": "string","min": 3,"max": 40}},
+    {"winneraddr": {"type": "string","min": 5,"max": 40}},
+    {"agencyaddr": {"type": "string","min": 5,"max": 40}},
+    {"buyeraddr": {"type": "string","min": 5,"max": 40}},
+    {"projectaddr": {"type": "string","min": 5,"max": 40}},
+    {"agencyperson": {"type": "string","min": 1,"max": 10}},
+    {"buyerperson": {"type": "string","min": 1,"max": 10}},
+    {"winnerperson": {"type": "string","min": 1,"max": 10}},
+    {"winnertel": {"type": "string","min": 5,"max": 20}},
+    {"agencytel": {"type": "string","min": 5,"max": 20}},
+    {"buyertel": {"type": "string","min": 5,"max": 20}},
+    {"bidopentime": {"type": "int","min": 946656000}},
+    {"signaturedate": {"type": "int","min": 946656000}}
+  ],
+  "deduct_element": [
+    "area",
+    "city",
+    "site",
+    "toptype",
+    "subtype",
+    "title"
+  ],
+  "specialaddr": "号|楼|座|巷|街|幢|路|室|层|区|段|道|园|镇|乡|村|县|区|市|栋|厦|房|社区|单元|交叉口|服务中心",
+  "jkmail": {
+    "to": "zhengkun@topnet.net.cn",
+    "api": "http://10.171.112.160:19281/_send/_mail"
+  },
+  "nextNode": [
+
+  ]
+}

data_quality/src1/main.go

@@ -0,0 +1,182 @@
+package main
+
+import (
+	"log"
+	mu "mfw/util"
+	"net"
+	"os"
+	"qfw/common/src/qfw/util"
+	qu "qfw/util"
+	"time"
+)
+
+
+var (
+	sysconfig    map[string]interface{} //配置文件
+	mgo          *MongodbSim            //mongodb操作对象
+	qy_mgo		 *MongodbSim
+	udpclient    mu.UdpClient             //udp对象
+	nextNode     []map[string]interface{} //下节点数组
+	siteMap	 	 map[string]map[string]interface{} //站点map
+	coll_name,qy_coll_name	string
+	core_element,other_element	[]map[string]interface{}	//要素
+	deduct_element 	[]string
+	total_score,core_max,core_each,other_max,other_each ,deduct_each int
+	specialaddr   string
+)
+
+//站点配置
+func initSite()  {
+	mconf := sysconfig["mongodb"].(map[string]interface{})
+	site := mconf["site"].(map[string]interface{})
+	siteMap = make(map[string]map[string]interface{}, 0)
+	start := int(time.Now().Unix())
+	sess_site := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(sess_site)
+	res_site := sess_site.DB(site["site_dbname"].(string)).C(site["site_coll"].(string)).Find(map[string]interface{}{}).Sort("_id").Iter()
+	for site_dict := make(map[string]interface{}); res_site.Next(&site_dict); {
+		data_map := map[string]interface{}{
+			"area":     util.ObjToString(site_dict["area"]),
+			"city":     util.ObjToString(site_dict["city"]),
+		}
+		siteMap[util.ObjToString(site_dict["site"])] = data_map
+	}
+	log.Printf("new站点加载用时：%d秒,%d个\n", int(time.Now().Unix())-start, len(siteMap))
+}
+
+//mgo-配置等
+func initMgo()  {
+	mconf := sysconfig["mongodb"].(map[string]interface{})
+	log.Println(mconf)
+	mgo = &MongodbSim{
+		MongodbAddr: mconf["addrName"].(string),
+		DbName:      mconf["dbName"].(string),
+		Size:        qu.IntAllDef(mconf["pool"], 10),
+	}
+	mgo.InitPool()
+
+	qy_mconf := sysconfig["qy_mongodb"].(map[string]interface{})
+	qy_mgo = &MongodbSim{
+		MongodbAddr: qy_mconf["qy_addrName"].(string),
+		DbName:      qy_mconf["qy_dbName"].(string),
+		Size:        qu.IntAllDef(qy_mconf["pool"], 10),
+		UserName: qy_mconf["qy_username"].(string),
+		Password: qy_mconf["qy_password"].(string),
+	}
+	qy_mgo.InitPool()
+
+	coll_name = mconf["collName"].(string)
+	qy_coll_name = qy_mconf["qy_collName"].(string)
+
+	core_element = qu.ObjArrToMapArr(sysconfig["core_element"].([]interface{}))
+	other_element = qu.ObjArrToMapArr(sysconfig["other_element"].([]interface{}))
+	deduct_element  =qu.ObjArrToStringArr(sysconfig["deduct_element"].([]interface{}))
+
+	score_standard := sysconfig["score_standard"].(map[string]interface{})
+	total_score = qu.IntAll(score_standard["total_score"])
+	core_max = qu.IntAll(score_standard["core_max"])
+	core_each = qu.IntAll(score_standard["core_each"])
+	other_max = qu.IntAll(score_standard["other_max"])
+	other_each = qu.IntAll(score_standard["other_each"])
+	deduct_each = qu.IntAll(score_standard["deduct_each"])
+
+	specialaddr = sysconfig["specialaddr"].(string)
+}
+//初始化
+func init() {
+	qu.ReadConfig(&sysconfig)//加载配置文件
+	initMgo()
+	initSite()//加载站点
+}
+//正式流程
+func mainT() {
+	go checkMapJob()
+	updport := sysconfig["udpport"].(string)
+	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Println("Udp服务监听", updport)
+	time.Sleep(99999 * time.Hour)
+}
+//快速测试使用
+func main() {
+	sid := "1f0000000000000000000000"
+	eid := "9f0000000000000000000000"
+	log.Println(sid, "---", eid)
+	mapinfo := map[string]interface{}{}
+	if sid == "" || eid == "" {
+		log.Println("sid,eid参数不能为空")
+		os.Exit(0)
+	}
+	mapinfo["gtid"] = sid
+	mapinfo["lteid"] = eid
+	startScoreTask([]byte{}, mapinfo)
+	time.Sleep(99999 * time.Hour)
+}
+
+//udp监听
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case mu.OP_TYPE_DATA: //上个节点的数据
+
+	case mu.OP_NOOP: //下个节点回应
+
+	}
+}
+//打分流程-方法
+func startScoreTask(data []byte, mapInfo map[string]interface{}) {
+	log.Println("开始评分流程")
+	defer qu.Catch()
+	//区间id
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  StringTOBsonId(mapInfo["gtid"].(string)),
+			"$lte": StringTOBsonId(mapInfo["lteid"].(string)),
+		},
+	}
+	log.Println("查询条件:",q)
+	sess := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(sess)
+	it := sess.DB(mgo.DbName).C(coll_name).Find(&q).Iter()
+	updateExtract := [][]map[string]interface{}{}//更新需要
+	index:=0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		element_score,core_score,other_score,element_reason:=dealWithElementRate(tmp)
+		error_score,abnormal_score,error_reason,abnormal_reason:=dealWithErrorRate(tmp)
+
+		if index%1000 == 0 {
+			log.Println("当前数量:", index, tmp["_id"],"元素分:",element_score,"错误分:",error_score,"异常分:",abnormal_score)
+		}
+		updateExtract = append(updateExtract, []map[string]interface{}{
+			map[string]interface{}{
+				"_id": tmp["_id"],
+			},
+			map[string]interface{}{
+				"$set": map[string]interface{}{
+					"element_score": element_score,
+					"core_score": core_score,
+					"other_score": other_score,
+					"error_score":error_score,
+					"abnormal_score": abnormal_score,
+					"quality_reason":map[string]interface{}{
+						"element_reason":element_reason,
+						"error_reason":error_reason,
+						"abnormal_reason":abnormal_reason,
+					},
+				},
+			},
+		})
+		if len(updateExtract) >= 200 {
+			mgo.UpSertBulk(coll_name, updateExtract...)
+			updateExtract = [][]map[string]interface{}{}
+		}
+
+		tmp = make(map[string]interface{})
+	}
+
+	if len(updateExtract) >0 {
+		mgo.UpSertBulk(coll_name, updateExtract...)
+	}
+
+	log.Println("task quality over - 总计数量",index)
+
+}

data_quality/src1/mark

@@ -0,0 +1,165 @@
+{
+  "udpport": ":17007",
+  "mongodb": {
+    "addrName": "192.168.3.207:27092",
+    "dbName": "extract_kf",
+    "collName": "zk_move",
+    "pool": 10,
+    "site": {
+      "site_dbname": "extract_kf",
+      "site_coll": "site"
+    }
+  },
+  "score_standard": {
+    "total_score": 100,
+    "core_max": 60,
+    "core_each": 10,
+    "other_max": 40,
+    "other_each": 5,
+    "deduct_each": 5
+  },
+  "qy_mongodb": {
+    "qy_addrName": "192.168.3.207:27092",
+    "qy_dbName": "extract_kf",
+    "qy_collName": "zk",
+    "pool": 10
+  },
+  "core_element":[
+    {"bidamount": {"type": "float", "min": 0,"max":50000000000}},
+    {"budget": {"type": "float", "min": 0,"max":50000000000}},
+    {"projectcode": {"type": "string", "min": 4,"max": 20}},
+    {"projectname": {"type": "string", "min": 4,"max": 40}},
+    {"buyer": {"type": "string", "min":4,"max": 15}},
+    {"winner": {"type": "string", "min": 4,"max": 15}}
+  ],
+  "other_element": [
+    {"agency": {"type": "string","min": 3,"max": 15}},
+    {"bidopenaddress": {"type": "string","min": 3,"max": 40}},
+    {"winneraddr": {"type": "string","min": 3,"max": 40}},
+    {"agencyaddr": {"type": "string","min": 3,"max": 40}},
+    {"buyeraddr": {"type": "string","min": 3,"max": 40}},
+    {"projectaddr": {"type": "string","min": 2,"max": 40}},
+    {"agencyperson": {"type": "string","min": 0,"max": 10}},
+    {"buyerperson": {"type": "string","min": 0,"max": 10}},
+    {"winnerperson": {"type": "string","min": 0,"max": 10}},
+    {"winnertel": {"type": "string","min": 4,"max": 20}},
+    {"agencytel": {"type": "string","min": 4,"max": 20}},
+    {"buyertel": {"type": "string","min": 4,"max": 20}},
+    {"bidopentime": {"type": "int","min": 946656000}},
+    {"signaturedate": {"type": "int","min": 946656000}}
+  ],
+  "deduct_element": [
+    "area",
+    "city",
+    "site",
+    "toptype",
+    "subtype",
+    "title"
+  ],
+  "specialaddr": "号|楼|座|巷|街|幢|路|室|层|区|段|道|园|镇|乡|村|县|区|市|栋|厦|房|社区|单元|交叉口|服务中心",
+  "jkmail": {
+    "to": "zhengkun@topnet.net.cn",
+    "api": "http://10.171.112.160:19281/_send/_mail"
+  },
+  "nextNode": [
+
+  ]
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+{
+  "udpport": ":17007",
+  "mongodb": {
+    "addrName": "172.17.4.85:27080",
+    "dbName": "qfw",
+    "collName": "result_20200715",
+    "pool": 10,
+    "site": {
+      "site_dbname": "qfw",
+      "site_coll": "site"
+    }
+  },
+  "qy_mongodb": {
+    "qy_addrName": "172.17.4.187:27081",
+    "qy_dbName": "mixdata",
+    "qy_collName": "qyxy_std",
+    "pool": 10
+  },
+  "score_standard": {
+    "total_score": 100,
+    "core_max": 60,
+    "core_each": 10,
+    "other_max": 40,
+    "other_each": 5,
+    "deduct_each": 5
+  },
+  "core_element":[
+    {"bidamount": {"type": "float", "min": 0,"max":50000000000}},
+    {"budget": {"type": "float", "min": 0,"max":50000000000}},
+    {"projectcode": {"type": "string", "min": 4,"max": 20}},
+    {"projectname": {"type": "string", "min": 4,"max": 40}},
+    {"buyer": {"type": "string", "min":4,"max": 15}},
+    {"winner": {"type": "string", "min": 4,"max": 15}}
+  ],
+  "other_element": [
+    {"agency": {"type": "string","min": 3,"max": 15}},
+    {"bidopenaddress": {"type": "string","min": 3,"max": 40}},
+    {"winneraddr": {"type": "string","min": 3,"max": 40}},
+    {"agencyaddr": {"type": "string","min": 3,"max": 40}},
+    {"buyeraddr": {"type": "string","min": 3,"max": 40}},
+    {"projectaddr": {"type": "string","min": 2,"max": 40}},
+    {"agencyperson": {"type": "string","min": 0,"max": 10}},
+    {"buyerperson": {"type": "string","min": 0,"max": 10}},
+    {"winnerperson": {"type": "string","min": 0,"max": 10}},
+    {"winnertel": {"type": "string","min": 4,"max": 20}},
+    {"agencytel": {"type": "string","min": 4,"max": 20}},
+    {"buyertel": {"type": "string","min": 4,"max": 20}},
+    {"bidopentime": {"type": "int","min": 946656000}},
+    {"signaturedate": {"type": "int","min": 946656000}}
+  ],
+  "deduct_element": [
+    "area",
+    "city",
+    "site",
+    "toptype",
+    "subtype",
+    "title"
+  ],
+  "specialaddr": "号|楼|座|巷|街|幢|路|室|层|区|段|道|园|镇|乡|村|县|区|市|栋|厦|房|社区|单元|交叉口|服务中心",
+  "jkmail": {
+    "to": "zhengkun@topnet.net.cn",
+    "api": "http://10.171.112.160:19281/_send/_mail"
+  },
+  "nextNode": [
+
+  ]
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

data_quality/src1/mgo.go

@@ -0,0 +1,328 @@
+package main
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type MgoSess struct {
+	Db     string
+	Coll   string
+	Query  interface{}
+	Sorts  []string
+	fields interface{}
+	limit  int64
+	skip   int64
+	M      *MongodbSim
+}
+
+type MgoIter struct {
+	Cursor *mongo.Cursor
+}
+
+func (mt *MgoIter) Next(result interface{}) bool {
+	if mt.Cursor != nil {
+		if mt.Cursor.Next(nil) {
+			err := mt.Cursor.Decode(result)
+			if err != nil {
+				log.Println("mgo cur err", err.Error())
+				mt.Cursor.Close(nil)
+				return false
+			}
+			return true
+		} else {
+			mt.Cursor.Close(nil)
+			return false
+		}
+	} else {
+		return false
+	}
+
+}
+
+func (ms *MgoSess) DB(name string) *MgoSess {
+	ms.Db = name
+	return ms
+}
+
+func (ms *MgoSess) C(name string) *MgoSess {
+	ms.Coll = name
+	return ms
+}
+
+func (ms *MgoSess) Find(q interface{}) *MgoSess {
+	ms.Query = q
+	return ms
+}
+
+func (ms *MgoSess) Select(fields interface{}) *MgoSess {
+	ms.fields = fields
+	return ms
+}
+
+func (ms *MgoSess) Limit(limit int64) *MgoSess {
+	ms.limit = limit
+	return ms
+}
+func (ms *MgoSess) Skip(skip int64) *MgoSess {
+	ms.skip = skip
+	return ms
+}
+
+func (ms *MgoSess) Sort(sorts ...string) *MgoSess {
+	ms.Sorts = sorts
+	return ms
+}
+
+func (ms *MgoSess) Iter() *MgoIter {
+	it := &MgoIter{}
+	find := options.Find()
+	if ms.skip > 0 {
+		find.SetSkip(ms.skip)
+	}
+	if ms.limit > 0 {
+		find.SetLimit(ms.limit)
+	}
+	find.SetBatchSize(100)
+	if len(ms.Sorts) > 0 {
+		sort := bson.M{}
+		for _, k := range ms.Sorts {
+			switch k[:1] {
+			case "-":
+				sort[k[1:]] = -1
+			case "+":
+				sort[k[1:]] = 1
+			default:
+				sort[k] = 1
+			}
+		}
+		find.SetSort(sort)
+	}
+	if ms.fields != nil {
+		find.SetProjection(ms.fields)
+	}
+	cur, err := ms.M.C.Database(ms.Db).Collection(ms.Coll).Find(ms.M.Ctx, ms.Query, find)
+	if err != nil {
+		log.Println("mgo find err", err.Error())
+	} else {
+		it.Cursor = cur
+	}
+	return it
+}
+
+type MongodbSim struct {
+	MongodbAddr string
+	Size        int
+	//	MinSize     int
+	DbName   string
+	C        *mongo.Client
+	Ctx      context.Context
+	ShortCtx context.Context
+	pool     chan bool
+	UserName string
+	Password string
+}
+
+func (m *MongodbSim) GetMgoConn() *MgoSess {
+	//m.Open()
+	ms := &MgoSess{}
+	ms.M = m
+	return ms
+}
+
+func (m *MongodbSim) DestoryMongoConn(ms *MgoSess) {
+	//m.Close()
+	ms.M = nil
+	ms = nil
+}
+
+func (m *MongodbSim) InitPool() {
+	opts := options.Client()
+	opts.SetConnectTimeout(3 * time.Second)
+	opts.ApplyURI("mongodb://" + m.MongodbAddr)
+	opts.SetMaxPoolSize(uint64(m.Size))
+	m.pool = make(chan bool, m.Size)
+
+	if m.UserName !="" && m.Password !="" {
+		cre := options.Credential{
+			Username:m.UserName,
+			Password:m.Password,
+		}
+		opts.SetAuth(cre)
+	}
+
+
+
+	opts.SetMaxConnIdleTime(2 * time.Hour)
+	m.Ctx, _ = context.WithTimeout(context.Background(), 99999*time.Hour)
+	m.ShortCtx, _ = context.WithTimeout(context.Background(), 1*time.Minute)
+	client, err := mongo.Connect(m.ShortCtx, opts)
+	if err != nil {
+		log.Println("mgo init error:", err.Error())
+	} else {
+		m.C = client
+		log.Println("init success")
+	}
+}
+
+func (m *MongodbSim) Open() {
+	m.pool <- true
+}
+func (m *MongodbSim) Close() {
+	<-m.pool
+}
+
+//批量插入
+func (m *MongodbSim) UpSertBulk(c string, doc ...[]map[string]interface{}) (map[int64]interface{}, bool) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	for _, d := range doc {
+		write := mongo.NewUpdateOneModel()
+		write.SetFilter(d[0])
+		write.SetUpdate(d[1])
+		write.SetUpsert(true)
+		writes = append(writes, write)
+	}
+	r, e := coll.BulkWrite(m.Ctx, writes)
+	if e != nil {
+		log.Println("mgo upsert error:", e.Error())
+		return nil, false
+	}
+	//	else {
+	//		if r.UpsertedCount != int64(len(doc)) {
+	//			log.Println("mgo upsert uncomplete:uc/dc", r.UpsertedCount, len(doc))
+	//		}
+	//		return true
+	//	}
+	return r.UpsertedIDs, true
+}
+
+//批量插入
+func (m *MongodbSim) SaveBulk(c string, doc ...map[string]interface{}) bool {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	for _, d := range doc {
+		write := mongo.NewInsertOneModel()
+		write.SetDocument(d)
+		writes = append(writes, write)
+	}
+	_, e := coll.BulkWrite(m.Ctx, writes)
+	if e != nil {
+		log.Println("mgo savebulk error:", e.Error())
+		return false
+	}
+	return true
+}
+
+//保存
+func (m *MongodbSim) Save(c string, doc map[string]interface{}) interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.InsertOne(m.Ctx, doc)
+	if err != nil {
+		return nil
+	}
+	return r.InsertedID
+}
+
+//更新by Id
+func (m *MongodbSim) UpdateById(c, id string, doc map[string]interface{}) bool {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	_, err := coll.UpdateOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)}, doc)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+//删除by id
+func (m *MongodbSim) DeleteById(c, id string) int64 {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.DeleteOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
+	if err != nil {
+		return 0
+	}
+	return r.DeletedCount
+}
+
+//通过条件删除
+func (m *MongodbSim) Delete(c string, query map[string]interface{}) int64 {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.DeleteMany(m.Ctx, query)
+	if err != nil {
+		return 0
+	}
+	return r.DeletedCount
+}
+
+//findbyid
+func (m *MongodbSim) FindById(c, id string) map[string]interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r := coll.FindOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
+	v := map[string]interface{}{}
+	r.Decode(&v)
+	return v
+}
+
+//findone
+func (m *MongodbSim) FindOne(c string, query map[string]interface{}) map[string]interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r := coll.FindOne(m.Ctx, query)
+	v := map[string]interface{}{}
+	r.Decode(&v)
+	return v
+}
+
+//find
+func (m *MongodbSim) Find(c string, query map[string]interface{}, sort, fields interface{}) ([]map[string]interface{}, error) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	op := options.Find()
+	r, err := coll.Find(m.Ctx, query, op.SetSort(sort), op.SetProjection(fields))
+	if err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+	var results []map[string]interface{}
+	if err = r.All(m.Ctx, &results); err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+	return results, nil
+}
+
+//创建_id
+func NewObjectId() primitive.ObjectID {
+	return primitive.NewObjectID()
+}
+
+func StringTOBsonId(id string) primitive.ObjectID {
+	objectId, _ := primitive.ObjectIDFromHex(id)
+	return objectId
+}
+
+func BsonTOStringId(id interface{}) string {
+	return id.(primitive.ObjectID).Hex()
+}

data_quality/src1/scoreExpError.go

@@ -0,0 +1,306 @@
+package main
+
+import (
+	qu "qfw/util"
+	"regexp"
+	"sync"
+	"time"
+	"unicode/utf8"
+)
+
+var sitelock sync.Mutex         //锁
+var error_reason map[string]interface{}
+var abnormal_reason map[string]interface{}
+
+func dealWithErrorRate(tmp map[string]interface{}) (int , int, map[string]interface{}, map[string]interface{}) {
+	error_reason = map[string]interface{}{}
+	abnormal_reason = map[string]interface{}{}
+	//错误 ， 异常error_score  abnormal_score,
+	err_num,ab_num:=0,0
+	//金额类
+	budget:=qu.Float64All(tmp["budget"])
+	bidamount:=qu.Float64All(tmp["bidamount"])
+	if tmp["budget"]==nil&&tmp["bidamount"]==nil {
+		//均不存在
+		ab_num++
+		abnormal_reason["money"] = "budget-bidamount均不存在"
+	}else if tmp["budget"]!=nil&&tmp["bidamount"]!=nil&&budget!=0&&bidamount!=0 {
+		//均存在
+		err,ab:=amountAnalysis(budget,bidamount)
+		err_num = err_num+err
+		ab_num = err_num+ab
+	}else {
+		//二者存在一个
+		if budget==0 && qu.ObjToString(tmp["toptype"])=="招标"{
+			ab_num++
+			abnormal_reason["money"] = "招标：budget空"
+		}
+		if bidamount==0 && qu.ObjToString(tmp["toptype"])=="结果"{
+			ab_num++
+			abnormal_reason["money"] = "结果：bidamount空"
+		}
+	}
+
+	//采购单位
+	buyer:=qu.ObjToString(tmp["buyer"])
+	if buyer=="" {
+		ab_num++
+		abnormal_reason["buyer"] = "buyer空"
+	}else {
+		if utf8.RuneCountInString(buyer)<4 {
+			err_num++
+			error_reason["buyer"] = "buyer长度"
+		}else {
+			if !buyerAnalysis(buyer) {
+				ab_num++
+				abnormal_reason["buyer"] = "buyer-企业无"
+			}
+		}
+	}
+
+	//中标单位
+	if qu.ObjToString(tmp["toptype"])=="结果" {
+		winner:=qu.ObjToString(tmp["winner"])
+		if winner=="" {
+			ab_num++
+			abnormal_reason["winner"] = "winner空"
+		}else {
+			if utf8.RuneCountInString(winner)<4 {
+				err_num++
+				error_reason["winner"] = "winner长度"
+			}else {
+				if !winnerAnalysis(winner) {
+					ab_num++
+					abnormal_reason["winner"] = "winner-企业无"
+				}
+			}
+		}
+	}
+
+
+	//标题，名称
+	title:=qu.ObjToString(tmp["title"])
+	if title=="" {
+		err_num++
+		error_reason["title"] = "空"
+	}else {
+		if utf8.RuneCountInString(title)<4 {
+			ab_num++
+			abnormal_reason["title"] = "title长度"
+		}else {
+
+		}
+	}
+	projectname:=qu.ObjToString(tmp["projectname"])
+	if projectname=="" {
+		err_num++
+		error_reason["projectname"] = "空"
+	}else {
+		if utf8.RuneCountInString(projectname)<4 {
+			ab_num++
+			abnormal_reason["projectname"] = "projectname长度"
+		}else {
+
+		}
+	}
+
+
+	//编号组
+	projectcode:=qu.ObjToString(tmp["projectcode"])
+	contractnumber:=qu.ObjToString(tmp["contractnumber"])
+	if projectcode==""&&contractnumber=="" {
+		ab_num++
+		abnormal_reason["code"] = "code-空"
+	}else {
+		if !codesAnalysis(projectcode,contractnumber) {
+			ab_num++
+			abnormal_reason["code"] = "code-不符"
+		}
+	}
+
+	//发布时间
+	publishtime:=qu.Int64All(tmp["publishtime"])
+	now:=time.Now().Unix()
+	if publishtime<=0||publishtime-now>0 {
+		err_num++
+		error_reason["publishtime"] = "publishtime-(超前)（0）"
+	}
+
+	//省份，城市
+	area := qu.ObjToString(tmp["area"])
+	if area == "A" {
+		area = "全国"
+	}
+	city := qu.ObjToString(tmp["city"])
+	site := qu.ObjToString(tmp["site"])
+	if !citysAnalysis(area,city,site) {
+		ab_num++
+		abnormal_reason["city"] = "area-站点不一致"
+	}
+
+	//招标时间-地点
+	if qu.Int64All(tmp["bidopentime"])==0 && qu.ObjToString(tmp["bidopenaddress"])=="" &&
+		qu.ObjToString(tmp["toptype"])=="招标" {
+		ab_num++
+		abnormal_reason["bidopen"] = "bidopen-时间-地点-空"
+	}
+
+
+
+	//类别问题
+	if qu.ObjToString(tmp["toptype"]) == ""{
+		err_num++
+		error_reason["toptype"] = "toptype:空"
+	}else {
+		if !categoryAnalysis(tmp) {
+			ab_num++
+			abnormal_reason["toptype"] = "toptype:内容>>"
+		}
+	}
+
+
+	return err_num,ab_num,error_reason,abnormal_reason
+}
+
+
+
+
+
+//分析-金额
+func amountAnalysis(budget float64,bidamount float64) (int ,int) {
+	err_num,ab_num:=0,0
+	proportion := bidamount/budget
+	if proportion>=0.1&&proportion<=10 {
+
+	}else if (proportion>=0.01&&proportion<0.1)||(proportion>10&&proportion<=100){
+		ab_num++
+		abnormal_reason["money"] = "bidamount/budget间隔异常"
+	}else {
+		err_num++
+		error_reason["money"] = "bidamount/budget-比例错误"
+	}
+
+	return err_num,ab_num
+}
+
+//分析-采购单位
+func buyerAnalysis(buyer string) bool{
+
+	q := map[string]interface{}{
+		"company_name": buyer,
+	}
+	data :=qy_mgo.FindOne(qy_coll_name,q)
+	//data,_:=qy_mgo.Find(qy_coll_name,q,nil,map[string]interface{}{"company_name":1})
+	if data==nil {
+		return false
+	}
+	return true
+}
+
+
+//分析-中标单位
+func winnerAnalysis(winner string) bool {
+
+	q := map[string]interface{}{
+		"company_name": winner,
+	}
+	data :=qy_mgo.FindOne(qy_coll_name,q)
+	//data,_:=qy_mgo.Find(qy_coll_name,q,nil,map[string]interface{}{"company_name":1})
+	if data==nil {
+		return false
+	}
+	return true
+}
+
+//分析-编号组
+func codesAnalysis(projectcode string,contractnumber string) bool {
+
+	if projectcode!="" {
+		if utf8.RuneCountInString(projectcode)<4 {
+			return false
+		}
+		//符合-8长度-日期格式 yyyyMMdd
+		if !regAnalysis(projectcode) &&utf8.RuneCountInString(projectcode)==8 {
+			return false
+		}
+	}
+
+	if contractnumber!="" {
+		if utf8.RuneCountInString(contractnumber)<4 {
+			return false
+		}
+
+		if !regAnalysis(contractnumber)  && utf8.RuneCountInString(projectcode)==8  {
+			return false
+		}
+	}
+	return true
+}
+
+//分析-省份，城市
+func citysAnalysis(area string,city string,site string) bool {
+
+	if site != "" {//站点对比
+		sitelock.Lock()
+		dict := siteMap[site]
+		sitelock.Unlock()
+		if dict != nil {
+			if (area == "全国" && dict["area"] != "") ||
+				(city == "" && dict["city"] != "") {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+
+
+
+//分析-类别
+func categoryAnalysis(tmp map[string]interface{}) bool {
+	toptype:=qu.ObjToString(tmp["toptype"])
+	if toptype=="招标" {
+		winner:=qu.ObjToString(tmp["winner"])
+		bidamount:=qu.Float64All(tmp["bidamount"])
+		if winner!=""||bidamount!=0 {
+			//结果超前
+			return false
+		}
+	}
+	return true
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+func regAnalysis(str string) bool {
+	reg:=`\d{8}`
+	regx,_ := regexp.Compile(reg)
+	if regx.FindString(str)!="" {
+		return false
+	}
+	return true
+}
+
+
+
+
+
+
+
+
+
+
+
+
+

data_quality/src1/scoreLogic.go

@@ -0,0 +1,191 @@
+package main
+
+import (
+	qu "qfw/util"
+	"strings"
+	"unicode/utf8"
+)
+
+var element_reason map[string]interface{}
+
+func dealWithElementRate(tmp map[string]interface{}) (int,int,int,map[string]interface{}) {
+
+	//score_standard 打分标准  要素打分 - 需慎重
+	element_reason = map[string]interface{}{}
+	m,n,z :=0,0,0
+	core_value,other_value,deduct_value :="","",""
+
+	//第一次-配置验证
+	coreArr,otherArr:=filterConfiguration(tmp)
+	//细节过滤-验证
+	new_coreArr,new_otherArr:=[]string{},[]string{}
+	for _,v:=range coreArr{
+		if v=="projectname"||v=="buyer"||v=="winner" {
+			if isChinese(qu.ObjToString(tmp[v])) {
+				new_coreArr = append(new_coreArr,v)
+			}
+		}else {
+			new_coreArr = append(new_coreArr,v)
+		}
+	}
+
+	for _,v:=range otherArr{
+		if v=="agencyperson"||v=="buyerperson"||v=="winnerperson" {
+			if isChinese(qu.ObjToString(tmp[v])) {
+				new_otherArr = append(new_otherArr,v)
+			}
+		}else if v=="agencytel"||v=="buyertel"||v=="winnertel" {
+			if !isChinese(qu.ObjToString(tmp[v])) {
+				if isTelephone(qu.ObjToString(tmp[v])) {
+					new_otherArr = append(new_otherArr,v)
+				}
+			}
+		}else if v=="bidopentime"||v=="signaturedate" {
+			if isTimestamp(qu.Int64All(tmp[v])) {
+				new_otherArr = append(new_otherArr,v)
+			}
+		}else if v=="bidopenaddress"||v=="winneraddr"||v=="agencyaddr"||
+			v=="buyeraddr" {
+			if isPlaceAddr(qu.ObjToString(tmp[v])) {
+				new_otherArr = append(new_otherArr,v)
+			}
+		}else if v=="projectaddr" {
+			if isChinese(qu.ObjToString(tmp[v])) {
+				new_otherArr = append(new_otherArr,v)
+			}
+		}else {
+			new_otherArr = append(new_otherArr,v)
+		}
+	}
+
+	core_value = strings.Join(new_coreArr, ",")
+	m = len(new_coreArr)
+
+	other_value = strings.Join(new_otherArr, ",")
+	n = len(new_otherArr)
+
+
+
+	//扣分项
+	for _,v:=range deduct_element{
+		if qu.ObjToString(tmp[v])=="" {
+			z--
+			if deduct_value == "" {
+				deduct_value = v
+			}else {
+				deduct_value = deduct_value+","+v
+			}
+		}
+	}
+
+	total_s,core_s,other_s:=calculateScore(m,n,z)
+	return total_s,core_s,other_s,map[string]interface{}{
+		"coreElement":map[string]interface{}{
+			"key":core_value,
+		},
+		"otherElement":map[string]interface{}{
+			"key":other_value,
+		},
+		"deductElement":map[string]interface{}{
+			"key":deduct_value,
+		},
+	}
+}
+
+func filterConfiguration(tmp map[string]interface{})([]string,[]string)  {
+	coreArr ,otherArr:= []string{},[]string{}
+	//核心要素   int - 时间-   float-金额区间  string-字符串长度
+	for _,v:=range core_element{
+		for k1,v1:=range v{
+			if tmp[k1]==nil {
+				continue
+			}
+			dict :=*qu.ObjToMap(v1)
+			element_type := qu.ObjToString(dict["type"])
+
+			if element_type=="int" {
+				min:=qu.IntAll(dict["min"])
+				if qu.IntAll(tmp[k1])>min {
+					coreArr = append(coreArr,k1)
+				}
+			}else if element_type=="float" {
+				min:=qu.Float64All(dict["min"])
+				max:=qu.Float64All(dict["max"])
+				if qu.Float64All(tmp[k1])>min && qu.Float64All(tmp[k1])<max{
+					coreArr = append(coreArr,k1)
+				}
+			}else if element_type=="string" {
+				min:=qu.IntAll(dict["min"])
+				max:=qu.IntAll(dict["max"])
+
+				if utf8.RuneCountInString(qu.ObjToString(tmp[k1]))>min &&
+					utf8.RuneCountInString(qu.ObjToString(tmp[k1]))<max{
+					coreArr = append(coreArr,k1)
+				}
+			}else {
+
+			}
+		}
+	}
+
+
+	for _,v:=range other_element{
+		for k1,v1:=range v{
+			if tmp[k1]==nil {
+				continue
+			}
+			dict :=*qu.ObjToMap(v1)
+			element_type := qu.ObjToString(dict["type"])
+
+			if element_type=="int" {
+				min:=qu.IntAll(dict["min"])
+				if qu.IntAll(tmp[k1])>min {
+					otherArr = append(otherArr,k1)
+				}
+			}else if element_type=="float" {
+				min:=qu.Float64All(dict["min"])
+				max:=qu.Float64All(dict["max"])
+				if qu.Float64All(tmp[k1])>min && qu.Float64All(tmp[k1])<max{
+					otherArr = append(otherArr,k1)
+				}
+			}else if element_type=="string" {
+				min:=qu.IntAll(dict["min"])
+				max:=qu.IntAll(dict["max"])
+
+				if utf8.RuneCountInString(qu.ObjToString(tmp[k1]))>min &&
+					utf8.RuneCountInString(qu.ObjToString(tmp[k1]))<max{
+					otherArr = append(otherArr,k1)
+				}
+			}else {
+
+			}
+		}
+	}
+
+	return coreArr,otherArr
+}
+
+
+
+
+func calculateScore(core_num int,other_num int,deduct_num int) (int,int,int)  {
+
+	m ,core_s:=core_each*core_num,core_each*core_num
+	if m>core_max {
+		m = core_max
+	}
+
+	n ,other_s:=other_each*other_num,other_each*other_num
+
+	if n > other_max {
+		n = other_max
+	}
+
+	z := deduct_each*deduct_num
+	t :=m+n+z
+	if t > total_score {
+		t=total_score
+	}
+
+	return t,core_s,other_s
+}

data_quality/src1/scoreMethod.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"regexp"
+	"time"
+	"unicode"
+)
+
+func isPlaceAddr(str string) bool {
+
+	if isTelephone(str) {
+		return false
+	}
+	regx,_ := regexp.Compile(specialaddr)
+	result:=regx.FindString(str)
+	if result !="" {
+		return true
+	}
+	return false
+}
+
+
+
+func isTimestamp(i int64) bool {
+	now:=time.Now().Unix()
+	if i<now+86400*180 {
+		return true
+	}
+	return false
+}
+
+func isTelephone(str string) bool {
+
+	reg1:=`^1[3|4|5|6|7|8|9][0-9]\d{8}$`
+	regx1,_ := regexp.Compile(reg1)
+
+	arr1:=regx1.FindAllString(str,-1)
+	if len(arr1)>0 {
+		return true
+	}
+
+	reg2:=`^(\d{2,4}-)?\d{7,8}$`
+	regx2,_ := regexp.Compile(reg2)
+	arr2:=regx2.FindAllString(str,-1)
+	if len(arr2)>0 {
+		return true
+	}
+
+	return false
+}
+
+
+
+
+func isChinese(str string) bool {
+	var count int
+	for _, v := range str {
+		if unicode.Is(unicode.Han, v) {
+			count++
+			break
+		}
+	}
+	return count > 0
+}
+
+func isChineseChar(str string) bool {
+	for _, r := range str {
+		if unicode.Is(unicode.Scripts["Han"], r) || (regexp.MustCompile("[\u3002\uff1b\uff0c\uff1a\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b]").MatchString(string(r))) {
+			return true
+		}
+	}
+	return false
+}

data_quality/src1/sendmail.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	mu "mfw/util"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+)
+
+var udptaskmap = &sync.Map{}
+var tomail string
+var api string
+
+type udpNode struct {
+	data      []byte
+	addr      *net.UDPAddr
+	timestamp int64
+	retry     int
+}
+
+func checkMapJob() {
+	//阿里云内网无法发送邮件
+	jkmail, _ := sysconfig["jkmail"].(map[string]interface{})
+	if jkmail != nil {
+		tomail, _ = jkmail["to"].(string)
+		api, _ = jkmail["api"].(string)
+	}
+	log.Println("start checkMapJob", tomail, sysconfig["jkmail"])
+	for {
+		udptaskmap.Range(func(k, v interface{}) bool {
+			now := time.Now().Unix()
+			node, _ := v.(*udpNode)
+			if now-node.timestamp > 120 {
+				node.retry++
+				if node.retry > 5 {
+					log.Println("udp重试失败", k)
+					udptaskmap.Delete(k)
+					res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, tomail, "pingfen-send-fail", k.(string)))
+					if err == nil {
+						defer res.Body.Close()
+						read, err := ioutil.ReadAll(res.Body)
+						log.Println("邮件发发送：", string(read), err)
+					}
+				} else {
+					log.Println("udp重发", k)
+					udpclient.WriteUdp(node.data, mu.OP_TYPE_DATA, node.addr)
+				}
+			} else if now-node.timestamp > 10 {
+				log.Println("udp任务超时中..", k)
+			}
+			return true
+		})
+		time.Sleep(60 * time.Second)
+	}
+}