data_ai/main.go

package main

import (
	"data_ai/clean"
	"data_ai/extract"
	"data_ai/udp"
	"data_ai/ul"
	log "github.com/donnie4w/go-logger/logger"
	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"sync"
	"unicode/utf8"
)

func init() {
	ul.IsTool = false //是否工具
	ul.IsLocal = true //是否本地
	ul.IsFull = false //是否全量
	if ul.IsTool {
		ul.InitToolVar()
	} else {
		ul.InitGlobalVar()
		if !ul.IsFull {
			udp.InitProcessVar()
		}
	}
	ul.InitOss(ul.IsLocal)
}

func main() {
	if ul.IsTool {
		//tool.StartToolUpdateInfo()
		return
	}
	//extract.TestSingleFieldInfo("bidding", "6722de29b25c3e1debe624c9")
	test1()
	lock := make(chan bool)
	<-lock
}

func test() {
	log.Debug("开始大模型验证内存数据···")
	q := map[string]interface{}{}
	pool_mgo := make(chan bool, 500)
	wg_mgo := &sync.WaitGroup{}
	sess := ul.SourceMgo.GetMgoConn()
	defer ul.SourceMgo.DestoryMongoConn(sess)
	total, isok := 0, 0
	it := sess.DB(ul.SourceMgo.DbName).C(ul.Bid_Name).Find(&q).Sort("-_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%1000 == 0 {
			log.Debug("cur index ", total)
		}
		isok++
		pool_mgo <- true
		wg_mgo.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-pool_mgo
				wg_mgo.Done()
			}()
			detail := qu.ObjToString(tmp["detail"])
			if utf8.RuneCountInString(detail) < 100 {
				data := extract.ResolveInfo(tmp)
				if len(data) > 0 {
					tmp["ai_zhipu"] = data
					ul.SourceMgo.Save("zktest_1031", tmp)
				}
			}
		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg_mgo.Wait()
	log.Debug("ai is over ...")
}

func test1() {
	arr := []string{
		"2024年3月4日 08:00:00",
		"2024年03月01日08时00分00秒",
		"2024年3月8日 17:00:00",
		"2024年03月08日09时30分00秒",
		"2024-01-24 09:15:00",
		"2024-02-29 00:00:00",
		"2024-02-23 00:00:00",
		"2024-03-05 09:30:00",
		"2024-02-21 09:30:00",
		"2024-02-28",
		"2024-02-28 09:00:00",
		"2024-03-01 00:00:00",
		"2024-02-06 17:30:00",
		"2024-03-01 00:00:00",
		"2024-02-06 17:30:00",
		"2024-02-29 00:00:00",
		"2024-02-29 00:00:00",
		"2024-02-29 00:00:00",
		"2024-02-28 08:30",
		"2024-02-29 19:56:29",
		"2024-03-29 10:30:00",
		"2024-03-29 10:30:00",
		"2024-01-12 00:00:00",
		"2024-03-23 09:00:00",
		"2024-02-28",
		"2024-02-28 09:00:00",
		"2024-02-02 10:00:00",
		"2024-02-28 00:00:00",
		"2024-02-27 00:00:00",
		"2024-02-28",
		"2024-02-29 15:30:00",
		"2024-02-29 12:00:00",
		"2024-02-28 09:00:00",
		"2024-02-27 09:00:00",
		"2024-02-27 09:00:00",
		"2024-02-29 00:00:00",
		"2024-02-29 00:00:00",
		"2024-01-30 09:30:00",
		"2024-02-27 09:00:00",
		"2024-02-27 09:30:00",
		"2024-03-01 00:00:00",
		"2024-02-28 09:00:00",
		"2024-02-27 09:00:00",
		"2024-02-28 09:00",
		"2024-02-26 08:30",
		"2024-02-27 09:00:00",
		"2024-02-27 09:30:00",
		"2024-02-28 09:00:00",
		"2024-02-27 09:00:00",
		"2024-02-28 09:00",
		"2024-02-26 08:30",
		"2024-03-05 09:30:00",
		"2024-02-27 17:54:28",
		"2024-03-01 10:00:00",
		"2024-03-01 10:00:00",
		"2024-02-08 00:00:00",
		"2024-03-01 09：00：00",
		"2024-03-01 09:00:00",
		"2024-03-08 19:30:00",
		"2024-03-08 19:30:00",
		"2024年3月4日 08:00:00",
		"2024年03月01日08时00分00秒",
		"2024年3月8日 17:00:00",
		"2024年03月08日09时30分00秒",
		"2024-03-07 17：00：00",
		"2024-03-07 17:00:00",
		"2024-02-26 08:30",
		"2024-02-28 09:00",
		"2024-02-27 09:00:00",
		"2024-02-28 09:00:00",
		"2024-02-27 09:30:00",
		"2024-02-27 09:00:00",
		"2024-01-30 09:30:00",
		"2024-02-29 00:00:00",
		"2024-02-29 00:00:00",
		"2024-02-27 09:00:00",
		"2024-02-27 09:00:00",
		"2024-02-28 09:00:00",
		"2024-02-29 12:00:00",
		"2024-02-29 16:00:00",
		"2024-02-29 15:30:00",
		"2024-02-28 00:00:00",
		"2024-02-27 00:00:00",
		"2024-02-28 00:00:00",
		"2024-02-02 10:00:00",
		"2024-02-28 09:00:00",
		"2024-02-28",
		"2024-03-23 09:00:00",
		"2024-01-12 00:00:00",
		"2024-03-29 10:30:00",
		"2024-03-29 10:30:00",
		"2024-02-29 19:56:29",
		"2024-02-28 08:30",
		"2024-02-29 00:00:00",
		"2024-02-29 00:00:00",
		"2024-02-29 00:00:00",
		"2024-02-06 17:30:00",
		"2024-03-01 00:00:00",
		"2024-02-06 17:30:00",
		"2024-03-01 00:00:00",
		"2024-02-28 09:00:00",
		"2024-02-28",
		"2024-02-21 09:30:00",
		"2024-03-05 09:30:00",
		"2024-02-23 00:00:00",
		"2024-02-29 00:00:00",
		"2024-01-24 09:15:00",
		"2024-03-06 14:00:00",
		"2024-03-06 00:00:00",
		"2024-03-07 23:59:00",
	}
	for _, v := range arr {
		nv := clean.CleanTime(v)
		log.Debug(v, "~~~", nv)
	}

}

func test2() {
	arr := []string{
		"ZFGG包件",
		"GC530100202400091001001",
		"GC532300202400053001001",
		"广州市白云区金沙街沙凤经济联合社留用地项目S2地块（金沙洲AB3708005地块）新建商业综合体全过程造价咨询服务",
		"一标段",
		"一标段",
		"E6111013556240003001001",
		"E6111013556240003001002",
		"一标段",
		"二标段",
		"一标段",
		"三标段",
		"二标段",
		"E61042235168rf556pk7001",
		"第三标段",
		"第二标段",
		"第三标段",
		"GC530500202400037001001",
		"第17标段",
		"第18标段",
		"1",
		"第23标段",
		"标段二",
		"标段二",
		"一标段",
		"一标段",
		"一标段",
		"e3100000151040117001001",
		"e3100000151027733043001",
		"e3100000151032001003001",
		"2024ANNGZ00007",
		"SYS20240206NTGC05002004",
		"SYS20240206NTGC05002010",
		"SYS20240206NTGC05002003",
		"E341522001002634001",
		"第1标段",
		"第三标段",
		"KJ06",
		"四川省遂宁市第一中学西校区第三期建设项目及附属工程（遂宁一中新校区改、扩建工程）主体工程增加建设内容-施工-1",
		"四川省遂宁市第一中学西校区第三期建设项目及附属工程（遂宁一中新校区改、扩建工程）增加建设内容-监理-1",
		"射洪市绿色农业产业带建设项目（一期）-监理-1",
		"HJFFJSZ2024020002001",
		"一标段",
		"E3702002313017682001001",
		"E3702002313017722001001",
		"1",
		"1",
		"标包一",
		"C01",
		"KJ06",
		"五标段",
		"二标段",
		"1标段",
	}
	for _, v := range arr {
		nv := clean.CleanOtherCode(v)
		log.Debug(nv, "~~~", v)
	}
}