fieldproject/tieta_data/init.go

package main

import (
	"fmt"
	"go.mongodb.org/mongo-driver/bson/primitive"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
	"os"
	"sync"
	"tieta_data/config"
)

var (
	MongoTool *mongodb.MongodbSim

	findLock            sync.Mutex
	mapPb, mapPn, mapPc map[string]*Key
	AllIdsMapLock       sync.Mutex
	AllIdsMap           map[string]*ID

	//仅初始化使用
	compareNoPass = map[string]bool{}
	compareAB     = map[string]bool{}
	compareAB2D   = map[string]bool{}
	compareABD    = map[string]bool{}
	compareAB2CD  = map[string]bool{}
	compareABCD   = map[string]bool{}

	FieldArr = []string{"项目id", "事件id", "业务场景", "事件标题", "招标单位", "中标单位", "省份", "地市", "县（区）", "项目编号", "招标infoId",
		"招标单位运营商标签", "招标单位一级行业", "招标单位二级行业", "招标单位联系人", "招标单位联系电话", "招标数据更新时间", "招标信息信息类型",
		"预算金额（元）", "招标代理机构", "招标代理机构联系人", "招标代理机构联系电话", "公告地址", "招标剑鱼标讯链接", "中标infoId", "中标单位运营商标签",
		"中标单位联系人", "中标单位联系电话", "中标数据更新时间", "中标信息信息类型", "中标金额（元）", "公告地址（URL）", "剑鱼标讯链接"}
	FieldMap = map[string]string{
		"项目id":       "_id",
		"事件id":       "sourceinfoid",
		"业务场景":       "scenetag",
		"事件标题":       "projectname",
		"招标单位":       "buyer",
		"中标单位":       "s_winner",
		"省份":         "area",
		"地市":         "city",
		"县（区）":       "district",
		"项目编号":       "projectcode",
		"招标infoId":   "zb_infoid",
		"招标单位运营商标签":  "buyertag",
		"招标单位一级行业":   "tagname",
		"招标单位二级行业":   "buyerclass",
		"招标单位联系人":    "buyerperson",
		"招标单位联系电话":   "buyertel",
		"招标数据更新时间":   "zb_updatetime",
		"招标信息信息类型":   "bidtype",
		"预算金额（元）":    "budget",
		"招标代理机构":     "agency",
		"招标代理机构联系人":  "agencyperson",
		"招标代理机构联系电话": "agencytel",
		"公告地址":       "zb_href",
		"招标剑鱼标讯链接":   "zb_jybxhref",
		"中标infoId":   "infoid",
		"中标单位运营商标签":  "tagname_3",
		"中标单位联系人":    "winnerperson",
		"中标单位联系电话":   "winnertel",
		"中标数据更新时间":   "updatetime",
		"中标信息信息类型":   "bidstatus",
		"中标金额（元）":    "bidamount",
		"公告地址（URL）":  "href",
		"剑鱼标讯链接":     "jybxhref",
	}
	FieldArr1 = []string{"信息id", "词包", "项目id", "事件id", "标文关键词", "业务场景", "招标方式", "信息类型", "二级信息类型",
		"信息标题", "省", "市", "县", "发布时间", "招标/项目编号", "招标单位", "招标单位行业归类", "招标单位一级标签",
		"招标单位二级行业", "招标预算", "招标单位联系人", "招标单位联系方式", "中标单位", "中标单位行业归类", "中标金额",
		"中标单位联系人", "中标单位联系方式", "代理机构", "代理联系人", "代理联系电话", "URL", "剑鱼标讯链接", "标书获取时间", "标书截止时间",
		"投标开始时间", "投标截止时间", "开标时间", "是否电子招标"}
	FieldMap1 = map[string]string{
		"信息id":     "_id",
		"词包":       "rulename",
		"项目id":     "pid",
		"事件id":     "id",
		"标文关键词":    "matchkey",
		"业务场景":     "scenetag",
		"招标方式":     "subtype",
		"信息类型":     "toptype",
		"二级信息类型":   "subtype",
		"信息标题":     "title",
		"省":        "area",
		"市":        "city",
		"县":        "district",
		"发布时间":     "publishtime",
		"招标/项目编号":  "projectcode",
		"招标单位":     "buyer",
		"招标单位行业归类": "buyertag",
		"招标单位一级标签": "tagname",
		"招标单位二级行业": "buyerclass",
		"招标预算":     "budget",
		"招标单位联系人":  "buyerperson",
		"招标单位联系方式": "buyertel",
		"中标单位":     "s_winner",
		"中标单位行业归类": "tagname_3",
		"中标金额":     "bidamount",
		"中标单位联系人":  "winnerperson",
		"中标单位联系方式": "winnertel",
		"代理机构":     "agency",
		"代理联系人":    "agencyperson",
		"代理联系电话":   "agencytel",
		"URL":      "href",
		"剑鱼标讯链接":   "jybxhref",
		"标书获取时间":   "docstarttime",
		"标书截止时间":   "bidendtime",
		"投标开始时间":   "bidstarttime",
		"投标截止时间":   "bidendtime",
		"开标时间":     "bidopentime",
		"是否电子招标":   "bidway",
	}
)

type ID struct {
	Id   string
	Lock sync.Mutex
	P    *Project
}

type Key struct {
	Arr  []string
	Lock sync.Mutex
}

func init() {

	wg = sync.WaitGroup{}
	mapPn = make(map[string]*Key, 5000000)
	AllIdsMap = make(map[string]*ID, 5000000)
	mapPb = make(map[string]*Key, 1500000)
	mapPn = make(map[string]*Key, 5000000)
	mapPc = make(map[string]*Key, 5000000)

	//加载项目数据
	//---不能通过
	vm := []string{"C", "D"}
	for i := 0; i < 2; i++ {
		for j := 0; j < 2; j++ {
			for k := 0; k < 2; k++ {
				key := vm[i] + vm[j] + vm[k]
				compareNoPass[key] = true
				//fmt.Println(key)
			}
		}
	}
	//fmt.Println("-------------------")

	//三个元素一致 [AB][AB][AB]，分值最高
	vm = []string{"A", "B"}
	for i := 0; i < 2; i++ {
		for j := 0; j < 2; j++ {
			for k := 0; k < 2; k++ {
				key := vm[i] + vm[j] + vm[k]
				compareAB[key] = true
				//fmt.Println(key)
			}
		}
	}
	//fmt.Println("-------------------", len(compareAB))
	//---至少两个一致，其他可能不存在
	//[AB][AB][ABD]
	//[AB][ABD][AB]
	vm = []string{"A", "B"}
	vm2 := []string{"A", "B", "D"}
	for i := 0; i < 2; i++ {
		for j := 0; j < 2; j++ {
			for k := 0; k < 3; k++ {
				key := vm[i] + vm[j] + vm2[k]
				if !compareAB[key] {
					compareAB2D[key] = true
					//fmt.Println(key)

				}
			}
		}
	}
	for i := 0; i < 2; i++ {
		for j := 0; j < 3; j++ {
			for k := 0; k < 2; k++ {
				key := vm[i] + vm2[j] + vm[k]
				if !compareAB[key] {
					compareAB2D[key] = true
					//fmt.Println(key)

				}
			}
		}
	}
	//fmt.Println("-------------------", len(compareAB2D))
	//---至少一个一致，其他可能不存在
	//[ABD][ABD][ABD] //已经删除DDD
	vm = []string{"A", "B", "D"}
	for i := 0; i < 3; i++ {
		for j := 0; j < 3; j++ {
			for k := 0; k < 3; k++ {
				key := vm[i] + vm[j] + vm[k]
				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] {
					compareABD[key] = true
					//fmt.Println(key)
				}
			}
		}
	}
	//fmt.Println("-------------------", len(compareABD))

	//[AB][ABCD][AB]
	//[AB][AB][ABCD]
	vm = []string{"A", "B"}
	vm2 = []string{"A", "B", "C", "D"}
	for i := 0; i < 2; i++ {
		for j := 0; j < 4; j++ {
			for k := 0; k < 2; k++ {
				key := vm[i] + vm2[j] + vm[k]
				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
					compareAB2CD[key] = true
					//fmt.Println(key)
				}
			}
		}
	}
	for i := 0; i < 2; i++ {
		for j := 0; j < 2; j++ {
			for k := 0; k < 4; k++ {
				key := vm[i] + vm[j] + vm2[k]
				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
					compareAB2CD[key] = true
					//fmt.Println(key)
				}
			}
		}
	}
	//fmt.Println("-------------------", len(compareAB2CD))
	//[ABECD][ABECD][ABECD]  //已经删除[CD][CD][CD]   //这个要重点讨论
	vm = []string{"A", "B", "C", "D"}
	for i := 0; i < 4; i++ {
		for j := 0; j < 4; j++ {
			for k := 0; k < 4; k++ {
				key := vm[i] + vm[j] + vm[k]
				if !compareAB[key] && !compareAB2D[key] && !compareABD[key] && !compareNoPass[key] && !compareAB2CD[key] {
					compareABCD[key] = true
					//fmt.Println(key)
				}
			}
		}
	}
}

func InitMgo() {
	MongoTool = &mongodb.MongodbSim{
		MongodbAddr: config.Conf.DB.Mongo.Addr,
		Size:        config.Conf.DB.Mongo.Size,
		DbName:      config.Conf.DB.Mongo.Dbname,
		UserName:    config.Conf.DB.Mongo.User,
		Password:    config.Conf.DB.Mongo.Password,
	}
	MongoTool.InitPool()
}

func InitLog() {
	logcfg := config.Conf.Log

	err := log.InitLog(
		log.Path(logcfg.LogPath),
		log.Level(logcfg.LogLevel),
		log.Compress(logcfg.Compress),
		log.MaxSize(logcfg.MaxSize),
		log.MaxBackups(logcfg.MaxBackups),
		log.MaxAge(logcfg.MaxAge),
		log.Format(logcfg.Format),
	)
	if err != nil {
		fmt.Printf("InitLog failed: %v\n", err)
		os.Exit(1)
	}
}

type Project struct {
	Id            primitive.ObjectID `json:"_id"`
	Ids           []string           `json:"ids,omitempty"`
	FirstTime     int64              `json:"firsttime,omitempty"`   //项目的最早时间
	LastTime      int64              `json:"lasttime,omitempty"`    //项目的最后时间
	ProjectName   string             `json:"projectname,omitempty"` //项目名称
	ProjectCode   string             `json:"projectcode,omitempty"` //项目代码
	Buyer         string             `json:"buyer,omitempty"`       //采购单位
	Agency        string             `json:"agency"`                //代理机构
	Area          string             `json:"area"`                  //地区
	City          string             `json:"city"`                  //地市
	District      string             `json:"district"`              //区县
	Bidamount     float64            `json:"bidamount,omitempty"`   //中标金额
	Budget        float64            `json:"budget,omitempty"`      //预算
	score         int
	comStr        string
	MPN           []string `json:"mpn,omitempty"` //合并后多余的项目名称
	MPC           []string `json:"mpc,omitempty"` //合并后多余的项目编号
	resVal, pjVal int

	Topscopeclass []string `json:"topscopeclass,omitempty"`
	Subscopeclass []string `json:"subscopeclass,omitempty"` //子行业分类
	Winners       string   `json:"s_winner,omitempty"`      //中标人
	Buyerperson   string   `json:"buyerperson"`             //采购联系人
	Buyertel      string   `json:"buyertel"`                //采购联系人电话
	AgencyPerson  string   `json:"agencyperson"`
	AgencyTel     string   `json:"agencytel"`
	WinnerPerson  string   `json:"winnerperson"`
	WinnerTel     string   `json:"winnertel"`
	Buyerclass    string   `json:"buyerclass"`            //采购单位分类
	Bidopentime   int64    `json:"bidopentime,omitempty"` //开标时间
	Jgtime        int64    `json:"jgtime"`                //结果中标时间
	Zbtime        int64    `json:"zbtime"`                //招标时间

	SceneTag string `json:"scenetag"`  //业务场景
	BuyerTag string `json:"buyertag"`  //招标单位行业归类
	TagName  string `json:"tagname"`   //招标单位一级标签
	TagName2 string `json:"tagname_2"` //招标单位二级行业
	TagName3 string `json:"tagname_3"` //中标单位行业归类
}

type Info struct {
	Id            string   `json:"id"`
	Href          string   `json:"href"` //源地址
	Publishtime   int64    `json:"publishtime"`
	Comeintime    int64    `json:"comeintime"`
	Title         string   `json:"title"`
	TopType       string   `json:"toptype"`
	SubType       string   `json:"subtype"`
	ProjectName   string   `json:"projectname"`
	ProjectCode   string   `json:"projectcode"`
	Buyer         string   `json:"buyer"`
	Buyerperson   string   `json:"buyerperson"`
	Buyertel      string   `json:"buyertel"`
	Agency        string   `json:"agency"`
	Area          string   `json:"area"`
	City          string   `json:"city"`
	District      string   `json:"district"`
	Topscopeclass []string `json:"topscopeclass"`
	Subscopeclass []string `json:"subscopeclass"`
	Buyerclass    string   `json:"buyerclass"`
	Bidopentime   int64    `json:"bidopentime"`
	Budget        float64  `json:"budget"`
	Bidamount     float64  `json:"bidamount"`
	Winners       []string
	PTC           string //从标题中抽的项目编号
	pnbval        int    //项目名称、编号、采购单位存在的个数
	LenPC         int    //项目编号长度
	LenPN         int    //项目名称长度
	LenPTC        int    //标题抽的项目编号长度
	//以下三个元素做对比，计算包含时候使用
	PNBH  int //0初始，+包含，-被包含
	PCBH  int
	PTCBH int
}