package main import ( "context" "encoding/json" "fmt" "github.com/olivere/elastic/v7" "io" util "jygit.jydev.jianyu360.cn/data_processing/common_utils" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb" "log" "regexp" "strings" ) // getProject 获取项目数据 func getProject() { MgoB := &mongodb.MongodbSim{ MongodbAddr: "172.17.189.140:27080", //MongodbAddr: "127.0.0.1:27083", Size: 10, DbName: "qfw", UserName: "SJZY_RWbid_ES", Password: "SJZY@B4i4D5e6S", //Direct: true, } MgoB.InitPool() url := "http://172.17.4.184:19805" //url := "http://127.0.0.1:19805" username := "es_all" password := "TopJkO2E_d1x" index := "projectset" //索引名称 // 创建 Elasticsearch 客户端 client, err := elastic.NewClient( elastic.SetURL(url), elastic.SetBasicAuth(username, password), elastic.SetSniff(false), ) if err != nil { log.Fatalf("创建 Elasticsearch 客户端失败:%s", err) } rangeQuery := elastic.NewRangeQuery("pici").Gt("1685548800").Lte("1704038400") //termQ := elastic.NewTermQuery("multipackage", 0) //rangeQuery := elastic.NewRangeQuery("id").Gt("657b08556977356f5578cb25").Lte("657b08556977356f5578cb26") query := elastic.NewBoolQuery().Must(rangeQuery) ctx := context.Background() //开始滚动搜索 scrollID := "" scroll := "1m" searchSource := elastic.NewSearchSource(). Query(query). Size(100). Sort("_doc", true) //升序排序 //Sort("_doc", false) //降序排序 searchService := client.Scroll(index). Size(100). Scroll(scroll). SearchSource(searchSource) res, err := searchService.Do(ctx) if err != nil { if err == io.EOF { fmt.Println("没有数据") } else { panic(err) } } defer client.ClearScroll().ScrollId(scrollID).Do(ctx) // 在退出时清理资源 fmt.Println("总数是:", res.TotalHits()) total := 0 for len(res.Hits.Hits) > 0 { for _, hit := range res.Hits.Hits { var doc map[string]interface{} err := json.Unmarshal(hit.Source, &doc) if err != nil { log.Printf("解析文档失败:%s", err) continue } //id := util.ObjToString(doc["id"]) //log.Println(id) var matchWords = make([]string, 0) var matchList = make([]interface{}, 0) if list, ok := doc["list"].([]interface{}); ok { for _, v := range list { if da, ok := v.(map[string]interface{}); ok { if util.ObjToString(da["toptype"]) != "招标" { continue } title := util.ObjToString(da["title"]) // 使用正则表达式进行匹配 matches := GetMatches(title) if len(matches) > 0 { matchList = append(matchList, da) } matchWords = append(matchWords, matches...) } } } insert := make(map[string]interface{}) insert["project_id"] = doc["id"] insert["_id"] = doc["id"] insert["multipackage"] = doc["multipackage"] insert["list"] = doc["list"] insert["projectname"] = doc["projectname"] insert["sourceinfourl"] = doc["sourceinfourl"] if len(matchWords) > 0 { insert["matchList"] = matchList insert["package_name"] = util.ObjToString(doc["projectname"]) + "-" + strings.Join(matchWords, "、") insert["type"] = 1 } MgoB.SaveByOriID("wcc_project_20240304", insert) } total = total + len(res.Hits.Hits) scrollID = res.ScrollId res, err = client.Scroll().ScrollId(scrollID).Scroll(scroll).Do(ctx) log.Println("current count:", total) if err != nil { if err == io.EOF { // 滚动到最后一批数据,退出循环 break } log.Printf("滚动搜索失败:%s", err) break // 处理错误时退出循环 } } fmt.Println("结束~~~~~~~~~~~~~~~") } func GetMatches(title string) (res []string) { // 编译正则表达式 re := regexp.MustCompile(`包\d+`) // 标题只有一个包2 // 定义正则表达式 re2 := regexp.MustCompile(`包(\d+(?:、\d+)*)`) //标题含有多个包;包10、12、14、17、18、19 re3 := regexp.MustCompile(`\d+包`) //冀中股份2023年12月标准件计划框架协议采购2包12.7 re4 := regexp.MustCompile(`标包\d+`) //2024一季度水火电一般工序类中小金工件(标包1)-招标公告 //re4 := regexp.MustCompile(`\d+标段`) //text := "中国绿发投资集团有限公司直属项目公司2023年第20批集中采购非招标项目(包10、12、14、17、18、19" //matches := re3.FindAllString(util.ObjToString(text), -1) matches := re2.FindAllString(util.ObjToString(title), -1) if len(matches) > 0 { return matches } matches = re4.FindAllString(util.ObjToString(title), -1) if len(matches) > 0 { return matches } matches = re3.FindAllString(util.ObjToString(title), -1) if len(matches) > 0 { return matches } matches = re.FindAllString(util.ObjToString(title), -1) return matches }