yysdgSync/main.go

package main

import (
	. "app.yhyue.com/moapp/jybase/common"
	. "app.yhyue.com/moapp/jybase/mongodb"
	"context"
	"flag"
	"fmt"
	"github.com/gogf/gf/v2/container/garray"
	"github.com/gogf/gf/v2/container/gmap"
	"github.com/gogf/gf/v2/container/gset"
	"github.com/gogf/gf/v2/frame/g"
	"github.com/gogf/gf/v2/os/gctx"
	"github.com/gogf/gf/v2/os/gtime"
	"github.com/gogf/gf/v2/text/gstr"
	"github.com/gogf/gf/v2/util/gconv"
	"golang.org/x/net/html"
	"net/rpc"
	"regexp"
	"strings"
	"sync"
	"time"
)

var (
	TaskConfig  *taskConfig
	mgo         *MongodbSim
	mgoBid      *MongodbSim
	allSpaceReg = regexp.MustCompile(`([\s　  ])+`)
	spaceReg    = regexp.MustCompile(`\s+`)
	brReg       = regexp.MustCompile(`(<br/>(\s+)?)+`)
)

type taskConfig struct {
	LastTime int64
}

type BidDataItem struct {
	Zbgs   *BidDataObj //中标公示
	Zcgs   *BidDataObj //招采公示
	Yxsjgs *BidDataObj //意向商机公示

}

type BidDataObj struct {
	Winners        *gset.Set
	Purchasinglist *gset.Set
	BidDataDetails *garray.SortedArray
}

type BidDataDetail struct {
	ProjectName       string  //项目名称
	ProjectCode       string  //项目编号
	Area              string  //项目地区
	Buyer             string  //招采单位
	BudgetOriginal    float64 //
	Budget            string  //预算金额
	Amount            float64 //金额
	Winner            string  //中标单位
	BidamountOriginal float64 //
	Bidamount         string  //中标金额
	Detail            string  //建设内容
	Bidendtime        string  //投标截止时间
	PurchaseTime      string  //预计采购时间
	Projectscope      string  //需求概况 projectscope
}

type Article struct {
	Title   string
	Content string
}

func newBidDataObj() *BidDataObj {
	return &BidDataObj{Winners: gset.New(true), Purchasinglist: gset.New(true), BidDataDetails: garray.NewSortedArray(func(a, b interface{}) int {
		if a.(*BidDataDetail).Amount > b.(*BidDataDetail).Amount {
			return -1
		} else if a.(*BidDataDetail).Amount < b.(*BidDataDetail).Amount {
			return 1
		}
		return 0
	}, true)}
}

func init() {
	ctx := gctx.New()
	mgo = &MongodbSim{
		MongodbAddr: g.Config().MustGet(ctx, "mongodb.main.address").String(),
		Size:        g.Config().MustGet(ctx, "mongodb.main.size").Int(),
		DbName:      g.Config().MustGet(ctx, "mongodb.main.dbName").String(),
		UserName:    g.Config().MustGet(ctx, "mongodb.main.userName").String(),
		Password:    g.Config().MustGet(ctx, "mongodb.main.password").String(),
	}
	mgo.InitPool()
	mgoBid = &MongodbSim{
		MongodbAddr: g.Config().MustGet(ctx, "mongodb.bidding.address").String(),
		Size:        g.Config().MustGet(ctx, "mongodb.bidding.size").Int(),
		DbName:      g.Config().MustGet(ctx, "mongodb.bidding.dbName").String(),
		UserName:    g.Config().MustGet(ctx, "mongodb.bidding.userName").String(),
		Password:    g.Config().MustGet(ctx, "mongodb.bidding.password").String(),
	}
	mgoBid.InitPool()
	ReadConfig("./task.json", &TaskConfig)
}

func main() {
	model := flag.Int("m", 0, "1：上传素材 2：非定时任务")
	startTime := flag.Int64("s", 0, "开始时间")
	endTime := flag.Int64("e", 0, "结束时间")
	flag.Parse()
	//log.Println(detailCapture(filterHtml(`一、项目编号：ZL24011540000（招标文件编号：ZL24011540000）<br/>二、项目名称：西藏低温冷害、干旱、病虫害气象预警信息化平台<br/>三、中标（成交）信息<br/>供应商名称：重庆阿里九九科技有限公司<br/>供应商地址：拉萨市北京西路回民巷2号<br/>中标（成交）金额：61.8000000（万元）<br/>四、主要标的信息<br/><table border=\"1\">\t<tbody>\t\t<tr>\t\t\t<td>序号</td>\t\t\t<td>   供应商名称  </td>\t\t\t<td>   服务名称  </td>\t\t\t<td>   服务范围  </td>\t\t\t<td>   服务要求  </td>\t\t\t<td>   服务时间  </td>\t\t\t<td>   服务标准  </td>\t\t</tr>\t\t<tr>\t\t\t<td>1</td>\t\t\t<td>   重庆阿里九九科技有限公司  </td>\t\t\t<td>   详见响应文件  </td>\t\t\t<td>   详见响应文件  </td>\t\t\t<td>   详见响应文件  </td>\t\t\t<td>   详见响应文件  </td>\t\t\t<td>   详见响应文件  </td>\t\t</tr>\t\t<tr>\t\t\t<td> </td>\t\t\t<td> </td>\t\t\t<td> </td>\t\t\t<td> </td>\t\t\t<td> </td>\t\t\t<td> </td>\t\t\t<td> </td>\t\t</tr>\t</tbody></table><br/>五、评审专家（单一来源采购人员）名单：<br/>程标、程伟伟、姚曌臻<br/>六、代理服务收费标准及金额：<br/>本项目代理费收费标准：按发改价格【2015】299号文相关规定向中标人收取<br/>本项目代理费总金额：0.927000 万元（人民币）<br/>七、公告期限<br/>自本公告发布之日起1个工作日。<br/>八、其它补充事宜<br/>九、凡对本次公告内容提出询问，请按以下方式联系。<br/>1.采购人信息<br/>名 称：西藏自治区气候中心　　　　　<br/>地址：西藏自治区拉萨市城关区林廓北路1号　　　　　　　　<br/>联系方式：张先生 0891-6330101　　　　　　<br/>2.采购代理机构信息<br/>名 称：西藏则立项目管理有限公司　　　　　　　　　　　　<br/>地　址：拉萨市柳梧新区海亮天城商铺11-7号　　　　　　　　　　　　<br/>联系方式：陈先生 17708911306　　　　　　　　　　　　<br/>3.项目联系方式<br/>项目联系人：陈先生<br/>电　话：　　17708911306`)))
	//addDraft(0, 1736289976) //f_sourceinfo_2025Jslt_msx 江苏 西藏
	//addDraft(1740731400, 1840731400) //f_sourceinfo_2025Sjcs 广东 上海
	//return
	if *model == 1 {
		addMaterial()
	} else if *model == 2 {
		addDraft(*startTime, *endTime)
	} else {
		SimpleCrontab(false, g.Config().MustGet(gctx.New(), "timeTask").String(), func() {
			et := time.Now().Unix()
			addDraft(TaskConfig.LastTime, et)
			TaskConfig.LastTime = et
			WriteSysConfig("./task.json", *TaskConfig)
		})
		<-chan bool(nil)
	}
}

func addMaterial() {
	ctx := gctx.New()
	var path = "./thumb.png"
	result, err := g.Client().Post(ctx, "https://api.weixin.qq.com/cgi-bin/material/add_material?access_token="+getAccessToken(ctx)+"&type=image", "media=@file:"+path)
	if err != nil {
		g.Log().Error(ctx, err)
		return
	}
	defer result.Close()
	g.Log().Info(ctx, "新增永久素材请求返回结果", result.ReadAllString())
}

// 新增草稿箱
func addDraft(startTime, endTime int64) {
	ctx := gctx.New()
	g.Log().Info(ctx, "开始新增草稿箱任务。。。")
	sess := mgo.GetMgoConn()
	defer mgo.DestoryMongoConn(sess)
	it := sess.DB(g.Config().MustGet(ctx, "mongodb.main.dbName").String()).C(g.Config().MustGet(ctx, "mongodb.main.collection").String()).Find(map[string]interface{}{
		"i_updatetime": map[string]interface{}{
			"$gte": startTime,
			"$lt":  endTime,
		},
		"i_ckdata": map[string]interface{}{
			"$gt": 0,
		},
		"v_baseinfo.informatized": "是",
	}).Select(map[string]interface{}{"id": 1, "v_baseinfo": 1}).Iter()
	all := gmap.New(true)
	pool := make(chan bool, 5)
	wait := &sync.WaitGroup{}
	index := 0
	areas := map[string]bool{}
	for _, v := range g.Config().MustGet(ctx, "areas").Strings() {
		areas[v] = true
	}
	for mt := make(map[string]interface{}); it.Next(&mt); {
		pool <- true
		wait.Add(1)
		go func(mn map[string]interface{}) {
			defer func() {
				<-pool
				wait.Done()
			}()
			v_baseinfo, _ := mn["v_baseinfo"].(map[string]interface{})
			area, _ := v_baseinfo["area"].(string)
			toptype, _ := v_baseinfo["toptype"].(string)
			subtype, _ := v_baseinfo["subtype"].(string)
			if !areas[area] || (toptype != "招标" && toptype != "采购意向" && subtype != "中标" && subtype != "成交") {
				return
			}
			var bdi *BidDataItem
			if all.Get(area) == nil {
				bdi = &BidDataItem{Zbgs: newBidDataObj(), Zcgs: newBidDataObj(), Yxsjgs: newBidDataObj()}
				all.Set(area, bdi)
			} else {
				bdi = all.Get(area).(*BidDataItem)
			}
			var bdo *BidDataObj
			if toptype == "招标" {
				bdo = bdi.Zcgs
			} else if toptype == "结果" {
				bdo = bdi.Zbgs
			} else {
				bdo = bdi.Yxsjgs
			}
			purchasinglist, _ := v_baseinfo["purchasinglist"].([]interface{})
			bidendtime := gconv.Int64(v_baseinfo["bidendtime"])
			procurementlist, _ := v_baseinfo["procurementlist"].([]interface{})
			if id := gconv.String(mn["id"]); id != "" && (purchasinglist == nil || len(purchasinglist) == 0 || v_baseinfo["bidendtime"] == nil || procurementlist == nil || len(procurementlist) == 0) {
				bidding, ok := mgoBid.FindById("bidding", id, `{"purchasinglist":1,"bidendtime":1,"procurementlist":1}`)
				if ok && bidding != nil && len(*bidding) > 0 {
					if purchasinglist == nil {
						purchasinglist, _ = (*bidding)["purchasinglist"].([]interface{})
					}
					if bidendtime == 0 {
						bidendtime = gconv.Int64((*bidding)["bidendtime"])
					}
					if procurementlist == nil {
						procurementlist, _ = (*bidding)["procurementlist"].([]interface{})
					}
				}
			}
			//
			if len(purchasinglist) > 0 {
				firstOne, _ := purchasinglist[0].(map[string]interface{})
				itemname := gconv.String(firstOne["itemname"])
				if itemname != "" {
					bdo.Purchasinglist.Add(itemname)
				}
			}
			//
			purchaseTime := ""
			if len(procurementlist) > 0 {
				firstOne, _ := procurementlist[0].(map[string]interface{})
				purchaseTime = gconv.String(firstOne["expurasingtime"])
			}
			//
			winner, _ := v_baseinfo["winner"].(string)
			bdo.Winners.Add(winner)
			//
			detail := filterHtml(gconv.String(v_baseinfo["projectscope"]))
			newDetail := detailCapture(detail)
			if detail != newDetail {
				newDetail += g.Config().MustGet(ctx, "template.captureAppend").String()
			}
			bdd := &BidDataDetail{
				ProjectName:       gconv.String(v_baseinfo["projectname"]),
				ProjectCode:       gconv.String(v_baseinfo["projectcode"]),
				Area:              area,
				Buyer:             gconv.String(v_baseinfo["buyer"]),
				BudgetOriginal:    gconv.Float64(v_baseinfo["budget"]),
				Winner:            winner,
				BidamountOriginal: gconv.Float64(v_baseinfo["bidamount"]),
				Detail:            newDetail,
				PurchaseTime:      purchaseTime,
				Projectscope:      gconv.String(v_baseinfo["projectscope"]),
			}
			if bdd.BudgetOriginal > 0 {
				bdd.Budget = gconv.String(RetainDecimal(gconv.Float64(v_baseinfo["budget"])/10000, 2)) + "万元"
			}
			if bdd.BidamountOriginal > 0 {
				bdd.Bidamount = gconv.String(RetainDecimal(gconv.Float64(v_baseinfo["bidamount"])/10000, 2)) + "万元"
			}
			if v_baseinfo["bidamount"] != nil {
				bdd.Amount = bdd.BidamountOriginal
			} else {
				bdd.Amount = bdd.BudgetOriginal
			}
			if bdd.ProjectName == "" {
				bdd.ProjectName = gconv.String(v_baseinfo["title"])
			}
			if bidendtime > 0 {
				bdd.Bidendtime = gtime.New(bidendtime).Layout(time.DateTime)
			}
			bdo.BidDataDetails.Add(bdd)
		}(mt)
		index++
		if index%500 == 0 {
			g.Log().Info(ctx, "加载清洗数据。。。", index)
		}
		mt = make(map[string]interface{})
	}
	wait.Wait()
	g.Log().Info(ctx, "清洗数据加载结束。。。", index)
	articles := []*Article{}
	ymd := gtime.Now().Format("Ymd")
	purchasingSize := g.Config().MustGet(ctx, "template.purchasingSize").Int()
	winnerSize := g.Config().MustGet(ctx, "template.winnerSize").Int()
	htmlBody := g.Config().MustGet(ctx, "template.htmlBody").String()
	htmlTopImg := g.Config().MustGet(ctx, "template.htmlTopImg").String()
	htmlProjectSpit := g.Config().MustGet(ctx, "template.htmlProjectSpit").String()
	htmlProjectName := g.Config().MustGet(ctx, "template.htmlProjectName").String()
	htmlProjectOther := g.Config().MustGet(ctx, "template.htmlProjectOther").String()
	htmlBottom := g.Config().MustGet(ctx, "template.htmlBottom").String()
	all.Iterator(func(k, v interface{}) bool {
		kk := k.(string)
		vv := v.(*BidDataItem)
		//
		if !vv.Zbgs.BidDataDetails.IsEmpty() {
			vv.Zbgs.BidDataDetails.Sort()
			content := fmt.Sprintf(g.Config().MustGet(ctx, "template.zbgs.content").String(), kk, garray.NewArrayFrom(vv.Zbgs.Purchasinglist.Pops(purchasingSize)).Join("、"), garray.NewArrayFrom(vv.Zbgs.Winners.Pops(winnerSize)).Join("、"))
			content += htmlTopImg
			for k, v := range vv.Zbgs.BidDataDetails.PopLefts(g.Config().MustGet(ctx, "template.zhongProjectSize").Int()) {
				if k > 0 {
					content += htmlProjectSpit
				}
				vv := v.(*BidDataDetail)
				content += fmt.Sprintf(htmlProjectName, NumToChinese(k+1), vv.ProjectName)
				if vv.Buyer != "" {
					content += fmt.Sprintf(htmlProjectOther, "招采单位", vv.Buyer)
				}
				if vv.Winner != "" {
					content += fmt.Sprintf(htmlProjectOther, "中标单位", vv.Winner)
				}
				if vv.Bidamount != "" {
					content += fmt.Sprintf(htmlProjectOther, "中标金额", vv.Bidamount)
				}
				if vv.Detail != "" {
					content += fmt.Sprintf(htmlProjectOther, "建设内容", vv.Detail)
				}
			}
			articles = append(articles, &Article{
				Title:   fmt.Sprintf("%s_%s_中标标讯", ymd, kk),
				Content: fmt.Sprintf(htmlBody, content+htmlBottom),
			})
		}
		//
		if !vv.Zcgs.BidDataDetails.IsEmpty() {
			vv.Zcgs.BidDataDetails.Sort()
			content := ""
			var allBudget float64
			for k, v := range vv.Zcgs.BidDataDetails.PopLefts(g.Config().MustGet(ctx, "template.zhaoProjectSize").Int()) {
				if k > 0 {
					content += htmlProjectSpit
				}
				vv := v.(*BidDataDetail)
				allBudget += vv.BudgetOriginal
				content += fmt.Sprintf(htmlProjectName, NumToChinese(k+1), vv.ProjectName)
				if vv.ProjectCode != "" {
					content += fmt.Sprintf(htmlProjectOther, "项目编号", vv.ProjectCode)
				}
				if vv.Buyer != "" {
					content += fmt.Sprintf(htmlProjectOther, "采购单位", vv.Buyer)
				}
				if vv.Area != "" {
					content += fmt.Sprintf(htmlProjectOther, "项目地区", vv.Area)
				}
				if vv.Budget != "" {
					content += fmt.Sprintf(htmlProjectOther, "预算金额", vv.Budget)
				}
				if vv.Bidendtime != "" {
					content += fmt.Sprintf(htmlProjectOther, "投标截止时间", vv.Bidendtime)
				}
				if vv.Detail != "" {
					content += fmt.Sprintf(htmlProjectOther, "项目建设内容", vv.Detail)
				}
			}
			content = fmt.Sprintf(g.Config().MustGet(ctx, "template.zcgs.content").String(), kk, garray.NewArrayFrom(vv.Zcgs.Purchasinglist.Pops(purchasingSize)).Join("、"), gconv.String(RetainDecimal(allBudget/10000, 2))+"万元") + htmlTopImg + content
			articles = append(articles, &Article{
				Title:   fmt.Sprintf("%s_%s_招标标讯", ymd, kk),
				Content: fmt.Sprintf(htmlBody, content+htmlBottom),
			})
		}
		//
		if !vv.Yxsjgs.BidDataDetails.IsEmpty() {
			vv.Yxsjgs.BidDataDetails.Sort()
			content := fmt.Sprintf(g.Config().MustGet(ctx, "template.yxsjgs.content").String(), kk)
			content += htmlTopImg
			for k, v := range vv.Yxsjgs.BidDataDetails.PopLefts(g.Config().MustGet(ctx, "template.caiProjectSize").Int()) {
				if k > 0 {
					content += htmlProjectSpit
				}
				vv := v.(*BidDataDetail)
				content += fmt.Sprintf(htmlProjectName, NumToChinese(k+1), vv.ProjectName)
				content += fmt.Sprintf(htmlProjectOther, "项目地区", vv.Area)
				if vv.Buyer != "" {
					content += fmt.Sprintf(htmlProjectOther, "采购单位", vv.Buyer)
				}
				if vv.Budget != "" {
					content += fmt.Sprintf(htmlProjectOther, "预算金额", vv.Budget)
				}
				if vv.PurchaseTime != "" {
					content += fmt.Sprintf(htmlProjectOther, "预计采购时间", vv.PurchaseTime)
				}
				if vv.Detail != "" {
					content += fmt.Sprintf(htmlProjectOther, "需求概况", vv.Detail)
				}
			}
			articles = append(articles, &Article{
				Title:   fmt.Sprintf("%s_%s_采购意向", ymd, kk),
				Content: fmt.Sprintf(htmlBody, content+htmlBottom),
			})
		}
		return true
	})
	//
	if len(articles) > 0 {
		list := g.List{}
		for _, v := range articles {
			list = append(list, g.Map{
				"article_type": "news",
				"title":        v.Title,
				//"author":AUTHOR,
				//"digest":DIGEST,
				"content": v.Content,
				//"content_source_url":CONTENT_SOURCE_URL,
				"thumb_media_id": g.Config().MustGet(ctx, "mediaId").String(),
				//"need_open_comment":0,
				//"only_fans_can_comment":0,
				//"pic_crop_235_1":X1_Y1_X2_Y2,
				//"pic_crop_1_1":X1_Y1_X2_Y2
			})
		}
		g.Log().Info(ctx, "提交草稿数据", len(articles), "条", gconv.String(list))
		result, err := g.Client().ContentType("application/json").Post(ctx, "https://api.weixin.qq.com/cgi-bin/draft/add?access_token="+getAccessToken(ctx), g.Map{"articles": list})
		if err != nil {
			g.Log().Error(ctx, "提交草稿出错", err)
			return
		}
		defer result.Close()
		g.Log().Info(ctx, "新增草稿箱请求返回结果", result.ReadAllString())
	} else {
		g.Log().Info(ctx, "没有需要提交草稿的数据")
	}
	g.Log().Info(ctx, "新增草稿箱任务结束。。。")
}

func getAccessToken(ctx context.Context) string {
	code := g.Config().MustGet(ctx, "appid").String()
	var repl string
	client, err := rpc.DialHTTP("tcp", g.Config().MustGet(ctx, "wxTokenRpc").String())
	if err != nil {
		g.Log().Error(ctx, code, err)
		return repl
	}
	defer client.Close()
	err = client.Call("WxTokenRpc.GetAccessToken", code, &repl)
	if err != nil {
		g.Log().Error(ctx, code, err)
		return repl
	}
	if repl == "" {
		g.Log().Error(ctx, code, "未获取到accessToken")
	} else {
		g.Log().Info(ctx, code, "获取到accessToken", repl)
	}
	return repl
}
func filterHtml(text string) string {
	doc, err := html.Parse(strings.NewReader(text))
	if err != nil {
		text = regexp.MustCompile(`<[^>]+>`).ReplaceAllString(text, "")
	} else {
		var textBuilder strings.Builder
		var traverse func(*html.Node)
		traverse = func(n *html.Node) {
			if n.Type == html.TextNode {
				textBuilder.WriteString(n.Data)
			} else if n.Type == html.ElementNode && n.Data == "br" {
				textBuilder.WriteString("<br/>")
			}
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				traverse(c)
			}
		}
		traverse(doc)
		text = textBuilder.String()
	}
	text = strings.ReplaceAll(text, `\t`, " ")
	text = strings.ReplaceAll(text, `\n`, " ")
	text = allSpaceReg.ReplaceAllString(text, " ")
	text = spaceReg.ReplaceAllString(text, " ")
	text = brReg.ReplaceAllString(text, "<br/>")
	text = strings.TrimSpace(text)
	return text
}

func detailCapture(text string) string {
	text = gstr.SubStrRune(text, 0, g.Config().MustGet(gctx.New(), "template.detailLen").Int())
	text = strings.TrimSuffix(text, "<br/")
	text = strings.TrimSuffix(text, "<br")
	text = strings.TrimSuffix(text, "<b")
	text = strings.TrimSuffix(text, "<")
	text = strings.TrimSpace(text)
	return text
}