|
|
@@ -4,13 +4,16 @@ import (
|
|
|
"encoding/json"
|
|
|
"esindex/config"
|
|
|
"fmt"
|
|
|
+ "github.com/cespare/xxhash/v2"
|
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
|
"go.uber.org/zap"
|
|
|
+ "jygit.jydev.jianyu360.cn/data_processing/common_utils"
|
|
|
"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
|
|
|
"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
|
|
|
"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
|
|
|
"math"
|
|
|
"net"
|
|
|
+ "os"
|
|
|
"regexp"
|
|
|
"sort"
|
|
|
"strconv"
|
|
|
@@ -501,3 +504,224 @@ func deletePreEsData(preId string) {
|
|
|
}
|
|
|
|
|
|
}
|
|
|
+
|
|
|
+// saveDb 文件写入
|
|
|
+func saveDb() {
|
|
|
+ mutex.Lock()
|
|
|
+ defer mutex.Unlock()
|
|
|
+ // 如果 cache 为空，则无需执行写入操作
|
|
|
+ if cache.GetCardinality() == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ fo, err := os.OpenFile(*dbfile, os.O_CREATE|os.O_RDWR|os.O_SYNC|os.O_TRUNC, 0777)
|
|
|
+ if err != nil {
|
|
|
+ log.Info("saveDb", zap.Error(err))
|
|
|
+ }
|
|
|
+ defer fo.Close()
|
|
|
+ cache.WriteTo(fo)
|
|
|
+}
|
|
|
+
|
|
|
+// getNewName 获取新的不重复名称
|
|
|
+func getNewName(tmp map[string]interface{}) string {
|
|
|
+ projectName := util.ObjToString(tmp["projectname"])
|
|
|
+ projectCode := util.ObjToString(tmp["projectcode"])
|
|
|
+ buyer := util.ObjToString(tmp["buyer"])
|
|
|
+ firsttime := util.Int64All(tmp["firsttime"])
|
|
|
+ createtime := util.Int64All(tmp["createtime"])
|
|
|
+ var projectDate, createDate string
|
|
|
+ if firsttime > 0 {
|
|
|
+ projectDate = time.Unix(firsttime, 0).Format("2006-01-02")
|
|
|
+ }
|
|
|
+ if createtime > 0 {
|
|
|
+ createDate = time.Unix(createtime, 0).Format("2006-01-02")
|
|
|
+ }
|
|
|
+
|
|
|
+ var matchWords = make([]string, 0)
|
|
|
+ if list, ok := tmp["list"].([]interface{}); ok {
|
|
|
+ if len(list) > 0 {
|
|
|
+ for _, v := range list {
|
|
|
+ if da, ok := v.(map[string]interface{}); ok {
|
|
|
+ title := util.ObjToString(da["title"])
|
|
|
+ // 使用正则表达式进行匹配
|
|
|
+ matches := GetPackages(title)
|
|
|
+ for _, v := range matches {
|
|
|
+ if !IsInStringArray(v, matchWords) {
|
|
|
+ matchWords = append(matchWords, v)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ //pks := removeDuplicates(matchWords)
|
|
|
+ packages := strings.Join(matchWords, "、")
|
|
|
+ return RenameProjectName(projectName, projectCode, packages, projectDate, buyer, createDate)
|
|
|
+}
|
|
|
+
|
|
|
+// hash 计算hash
|
|
|
+func hash(src string) uint64 {
|
|
|
+ return xxhash.Sum64String(src)
|
|
|
+}
|
|
|
+
|
|
|
+// RenameProjectName 获取新的不重复的项目名称
|
|
|
+func RenameProjectName(projectName, projectCode, packages, projectDate, buyer, createDate string) (newName string) {
|
|
|
+ //TODO 1.判断项目名称是否重复
|
|
|
+ var id uint64
|
|
|
+ defer func() {
|
|
|
+ if id > 0 && newName != "" {
|
|
|
+ cache.Add(uint32(id))
|
|
|
+ cacheModify = true
|
|
|
+ }
|
|
|
+ }()
|
|
|
+
|
|
|
+ //1.项目名称
|
|
|
+ if projectName != "" {
|
|
|
+ id = hash(projectName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ newName = projectName
|
|
|
+ return projectName
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //TODO 2.1 项目名称+项目编码
|
|
|
+ if projectCode != "" {
|
|
|
+ newName = projectName + "_" + projectCode
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //TODO 2.2 项目名称+分包信息
|
|
|
+ if packages != "" {
|
|
|
+ newName = projectName + "_" + packages
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //TODO 2.3 项目名称+项目时间
|
|
|
+ if projectDate != "" {
|
|
|
+ newName = projectName + "_" + projectDate
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ //TODO 2.4 项目名称+采购单位名称
|
|
|
+ if buyer != "" {
|
|
|
+ newName = projectName + "_" + buyer
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //TODO 3.1 项目名称+项目编码+分包信息
|
|
|
+ if projectCode != "" && packages != "" {
|
|
|
+ newName = projectName + "_" + projectCode + "_" + packages
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //TODO 3.2 项目名称+项目编码+项目时间
|
|
|
+ if projectCode != "" && projectDate != "" {
|
|
|
+ newName = projectName + "_" + projectCode + "_" + projectDate
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //TODO 3.3 项目名称+项目编码+采购单位
|
|
|
+ if projectCode != "" && buyer != "" {
|
|
|
+ newName = projectName + "_" + projectCode + "_" + buyer
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ //TODO 3.4 项目名称+分包+项目时间
|
|
|
+ if packages != "" && projectDate != "" {
|
|
|
+ newName = projectName + "_" + packages + "_" + projectDate
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //TODO 3.5 项目名称+分包+采购单位
|
|
|
+ if packages != "" && buyer != "" {
|
|
|
+ newName = projectName + "_" + packages + "_" + buyer
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //TODO 3.6 项目名称+项目时间+采购单位
|
|
|
+ if projectDate != "" && buyer != "" {
|
|
|
+ newName = projectName + "_" + projectDate + "_" + buyer
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ //TODO 4.1 项目名称+项目编码+分包信息+项目时间
|
|
|
+ if projectCode != "" && packages != "" && projectDate != "" {
|
|
|
+ newName = projectName + "_" + projectCode + "_" + packages + "_" + projectDate
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //TODO 4.2 项目名称+项目编码+分包信息+采购单位
|
|
|
+ if projectCode != "" && packages != "" && buyer != "" {
|
|
|
+ newName = projectName + "_" + projectCode + "_" + packages + "_" + buyer
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ //TODO 5 项目名称+项目编码+分包信息+项目时间+采购单位
|
|
|
+ if projectCode != "" && packages != "" && projectDate != "" && buyer != "" {
|
|
|
+ newName = projectName + "_" + projectCode + "_" + packages + "_" + projectDate + "_" + buyer
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ newName = projectName + "_" + projectCode + "_" + packages + "_" + projectDate + "_" + buyer + "_" + createDate
|
|
|
+ id = hash(newName)
|
|
|
+ if !cache.Contains(uint32(id)) {
|
|
|
+ return newName
|
|
|
+ } else {
|
|
|
+ newName = ""
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// GetPackages 获取对应的分包
|
|
|
+func GetPackages(title string) (res []string) {
|
|
|
+ // 定义正则表达式
|
|
|
+ rea := regexp.MustCompile(`包\d{1,2}[-~、]\d{1,2}|\d{1,2}[-~、]\d{1,2}包`) //1-6包；01-06包；01、02包;包1、包2
|
|
|
+ //text := "中国绿发投资集团有限公司直属项目公司2023年第20批集中采购非招标项目（包10、12、14、17、18、19"
|
|
|
+ packages := rea.FindAllString(util.ObjToString(title), -1) //匹配的包
|
|
|
+
|
|
|
+ if len(packages) > 0 {
|
|
|
+ res = append(res, packages...)
|
|
|
+ }
|
|
|
+
|
|
|
+ reb := regexp.MustCompile(`(标段[1-9一二三四五六七八九]|[1-9一二三四五六七八九]标段|包[1-9一二三四五六七八九]?[0-9]|[1-9一二三四五六七八九]?[0-9]包|[a-kA-K]包)`) // 标题只有一个包2
|
|
|
+ pgs := reb.FindAllString(title, -1)
|
|
|
+ if len(pgs) > 0 {
|
|
|
+ for _, v := range pgs {
|
|
|
+ if !IsInStringArray(v, res) {
|
|
|
+ res = append(res, v)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return res
|
|
|
+}
|
wcc