jyps/getEs/project.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/olivere/elastic/v7"
	"io"
	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
	"log"
	"regexp"
	"strings"
)

// getProject 获取项目数据
func getProject() {
	MgoB := &mongodb.MongodbSim{
		MongodbAddr: "172.17.189.140:27080",
		//MongodbAddr: "127.0.0.1:27083",
		Size:     10,
		DbName:   "qfw",
		UserName: "SJZY_RWbid_ES",
		Password: "SJZY@B4i4D5e6S",
		//Direct:   true,
	}
	MgoB.InitPool()

	url := "http://172.17.4.184:19805"
	//url := "http://127.0.0.1:19805"
	username := "es_all"
	password := "TopJkO2E_d1x"
	index := "projectset" //索引名称
	// 创建 Elasticsearch 客户端
	client, err := elastic.NewClient(
		elastic.SetURL(url),
		elastic.SetBasicAuth(username, password),
		elastic.SetSniff(false),
	)
	if err != nil {
		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
	}

	rangeQuery := elastic.NewRangeQuery("pici").Gt("1685548800").Lte("1704038400")
	//termQ := elastic.NewTermQuery("multipackage", 0)
	//rangeQuery := elastic.NewRangeQuery("id").Gt("657b08556977356f5578cb25").Lte("657b08556977356f5578cb26")
	query := elastic.NewBoolQuery().Must(rangeQuery)

	ctx := context.Background()
	//开始滚动搜索
	scrollID := ""
	scroll := "1m"
	searchSource := elastic.NewSearchSource().
		Query(query).
		Size(100).
		Sort("_doc", true) //升序排序
	//Sort("_doc", false) //降序排序
	searchService := client.Scroll(index).
		Size(100).
		Scroll(scroll).
		SearchSource(searchSource)

	res, err := searchService.Do(ctx)

	if err != nil {
		if err == io.EOF {
			fmt.Println("没有数据")
		} else {
			panic(err)
		}

	}
	defer client.ClearScroll().ScrollId(scrollID).Do(ctx) // 在退出时清理资源
	fmt.Println("总数是：", res.TotalHits())
	total := 0
	for len(res.Hits.Hits) > 0 {
		for _, hit := range res.Hits.Hits {
			var doc map[string]interface{}
			err := json.Unmarshal(hit.Source, &doc)
			if err != nil {
				log.Printf("解析文档失败：%s", err)
				continue
			}
			//id := util.ObjToString(doc["id"])
			//log.Println(id)
			var matchWords = make([]string, 0)
			var matchList = make([]interface{}, 0)
			if list, ok := doc["list"].([]interface{}); ok {
				for _, v := range list {
					if da, ok := v.(map[string]interface{}); ok {
						if util.ObjToString(da["toptype"]) != "招标" {
							continue
						}
						title := util.ObjToString(da["title"])
						// 使用正则表达式进行匹配
						matches := GetMatches(title)
						if len(matches) > 0 {
							matchList = append(matchList, da)
						}
						matchWords = append(matchWords, matches...)
					}
				}
			}

			insert := make(map[string]interface{})
			insert["project_id"] = doc["id"]
			insert["_id"] = doc["id"]
			insert["multipackage"] = doc["multipackage"]
			insert["list"] = doc["list"]
			insert["projectname"] = doc["projectname"]
			insert["sourceinfourl"] = doc["sourceinfourl"]
			if len(matchWords) > 0 {
				insert["matchList"] = matchList
				insert["package_name"] = util.ObjToString(doc["projectname"]) + "-" + strings.Join(matchWords, "、")
				insert["type"] = 1
			}

			MgoB.SaveByOriID("wcc_project_20240304", insert)

		}
		total = total + len(res.Hits.Hits)
		scrollID = res.ScrollId
		res, err = client.Scroll().ScrollId(scrollID).Scroll(scroll).Do(ctx)
		log.Println("current count:", total)
		if err != nil {
			if err == io.EOF {
				// 滚动到最后一批数据，退出循环
				break
			}
			log.Printf("滚动搜索失败：%s", err)
			break // 处理错误时退出循环
		}
	}

	fmt.Println("结束~~~~~~~~~~~~~~~")

}

func GetMatches(title string) (res []string) {
	// 编译正则表达式
	re := regexp.MustCompile(`包\d+`) // 标题只有一个包2
	// 定义正则表达式
	re2 := regexp.MustCompile(`包(\d+(?:、\d+)*)`) //标题含有多个包；包10、12、14、17、18、19
	re3 := regexp.MustCompile(`\d+包`)            //冀中股份2023年12月标准件计划框架协议采购2包12.7
	re4 := regexp.MustCompile(`标包\d+`)           //2024一季度水火电一般工序类中小金工件（标包1）-招标公告
	//re4 := regexp.MustCompile(`\d+标段`)

	//text := "中国绿发投资集团有限公司直属项目公司2023年第20批集中采购非招标项目（包10、12、14、17、18、19"
	//matches := re3.FindAllString(util.ObjToString(text), -1)
	matches := re2.FindAllString(util.ObjToString(title), -1)
	if len(matches) > 0 {
		return matches
	}

	matches = re4.FindAllString(util.ObjToString(title), -1)
	if len(matches) > 0 {
		return matches
	}

	matches = re3.FindAllString(util.ObjToString(title), -1)
	if len(matches) > 0 {
		return matches
	}

	matches = re.FindAllString(util.ObjToString(title), -1)
	return matches
}