data_monitor/data_export_yx/src/cgyx.go

package main

import (
	"fmt"
	log "github.com/donnie4w/go-logger/logger"
	"github.com/tealeg/xlsx"
	"go.mongodb.org/mongo-driver/bson/primitive"
	"os"
	qu "qfw/util"
	"sync"
	"time"
)

func exportExcle(){
	sess := save_mgo.GetMgoConn()
	defer save_mgo.DestoryMongoConn(sess)
	q,total := map[string]interface{}{},0

	os.Remove("rj_data.xlsx")
	f :=xlsx.NewFile()
	sheet, _ := f.AddSheet("标的物项目信息")
	row := sheet.AddRow()
	row.AddCell().Value = "信息匹配词"
	row.AddCell().Value = "大区"
	row.AddCell().Value = "省份（锐捷划分）"
	row.AddCell().Value = "省份"
	row.AddCell().Value = "城市"
	row.AddCell().Value = "行业"
	row.AddCell().Value = "项目名称"
	row.AddCell().Value = "项目金额（万元）"
	row.AddCell().Value = "公告地址"
	row.AddCell().Value = "剑鱼标讯地址"
	row.AddCell().Value = "采购意向名称"
	row.AddCell().Value = "采购内容"
	row.AddCell().Value = "采购单位"
	row.AddCell().Value = "采购时间"
	row.AddCell().Value = "附件下载地址"
	row.AddCell().Value = "采购单位联系人"
	row.AddCell().Value = "采购单位联系电话"
	row.AddCell().Value = "发布时间"


	it := sess.DB(save_mgo.DbName).C(export_coll).Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp);total++ {
		if total%1000 == 0 {
			log.Debug("curent index ", total, tmp["_id"])
		}
		row = sheet.AddRow()
		row.AddCell().Value = qu.ObjToString(tmp["matchkey"])
		row.AddCell().Value = qu.ObjToString(tmp["region"])
		row.AddCell().Value = qu.ObjToString(tmp["region_pro"])
		row.AddCell().Value = qu.ObjToString(tmp["area"])
		row.AddCell().Value = qu.ObjToString(tmp["city"])
		row.AddCell().Value = qu.ObjToString(tmp["buyerclass"])
		row.AddCell().Value = qu.ObjToString(tmp["p_name"])
		row.AddCell().Value = qu.ObjToString(tmp["p_price"])
		row.AddCell().Value = qu.ObjToString(tmp["href"])
		row.AddCell().Value = qu.ObjToString(tmp["jybxhref"])
		row.AddCell().Value = qu.ObjToString(tmp["title"])
		row.AddCell().Value = qu.ObjToString(tmp["p_scope"])
		row.AddCell().Value = qu.ObjToString(tmp["buyer"])
		row.AddCell().Value = qu.ObjToString(tmp["p_time"])
		row.AddCell().Value = qu.ObjToString(tmp["file_url"])
		row.AddCell().Value = qu.ObjToString(tmp["buyerperson"])
		row.AddCell().Value = qu.ObjToString(tmp["buyertel"])
		row.AddCell().Value = qu.ObjToString(tmp["publishtime"])

		tmp = make(map[string]interface{})
	}

	err := f.Save("rj_data.xlsx")
	if err != nil {
		log.Debug("保存xlsx失败:", err)
	}else {
		log.Debug("保存xlsx成功:", err)
	}

}

func dealWithCgyxData()  {

	sess := save_mgo.GetMgoConn()
	defer save_mgo.DestoryMongoConn(sess)
	q := map[string]interface{}{}
	log.Debug("查询条件~",q)
	total,isok:= 0,0
	repair_pool := make(chan bool, 5)
	repair_wg := &sync.WaitGroup{}
	it := sess.DB(save_mgo.DbName).C(save_coll).Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
		if total%1000==0 {
			log.Debug("curent index ",total,tmp["_id"],isok)
		}
		subtype := qu.ObjToString(tmp["subtype"])
		if subtype!="采购意向" {
			tmp = make(map[string]interface{})
			continue
		}

		repair_pool <- true
		repair_wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-repair_pool
				repair_wg.Done()
			}()
			p_list := []map[string]interface{}{}
			if purchasinglist, ok := tmp["purchasinglist"].(primitive.A); ok {
				p_list = qu.ObjArrToMapArr(purchasinglist)
			}else {
				if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok {
					p_list = qu.ObjArrToMapArr(purchasinglist)
				}
			}

			f_list := []map[string]interface{}{}
			if filehref, ok := tmp["filehref"].(primitive.A); ok {
				f_list = qu.ObjArrToMapArr(filehref)
			}else {
				if filehref, ok := tmp["filehref"].([]interface{}); ok {
					f_list = qu.ObjArrToMapArr(filehref)
				}
			}

			//其他字段需要
			file_url := ""
			if len(f_list)>0 {
				file_url = qu.ObjToString(f_list[0]["url"])
			}
			area := qu.ObjToString(tmp["area"])
			city := qu.ObjToString(tmp["city"])
			publishtime := qu.Int64All(tmp["publishtime"])
			p_time_str := time.Unix(publishtime, 0).Format(timeLayout_1)


			region := ""
			region_pro := ""

			if city_map[city]!=nil {
				region = city_map[city]["region"]
				region_pro = city_map[city]["region_pro"]
			}else {
				if area_map[area]!=nil {
					region = area_map[area]["region"]
					region_pro = area_map[area]["region_pro"]
				}
			}




			if len(p_list)>0 {
				new_list := comparisonList(p_list,tmp)
				if len(new_list)>0 {
					isok+=len(new_list) //计数使用
					for _,list := range new_list {
						data := map[string]interface{}{}
						data["matchkey"] = qu.ObjToString(tmp["matchkey"])
						data["region"] = region
						data["region_pro"] = region_pro
						data["area"] = area
						data["city"] = city
						data["buyerclass"] = qu.ObjToString(tmp["buyerclass"])
						data["p_name"] = qu.ObjToString(list["p_name"])
						data["p_price"] = qu.ObjToString(list["p_price"])
						data["href"] = qu.ObjToString(tmp["href"])
						data["jybxhref"] = qu.ObjToString(tmp["jybxhref"])
						data["title"] = qu.ObjToString(tmp["title"])
						data["p_scope"] = qu.ObjToString(list["p_scope"])
						data["buyer"] = qu.ObjToString(tmp["buyer"])
						data["p_time"] = qu.ObjToString(list["p_time"])
						data["file_url"] = file_url
						data["buyerperson"] = qu.ObjToString(tmp["buyerperson"])
						data["buyertel"] = qu.ObjToString(tmp["buyertel"])
						data["publishtime"] = p_time_str
						save_mgo.Save(export_coll,data)
					}
				}
			}
		}(tmp)
		tmp = make(map[string]interface{})
	}
	repair_wg.Wait()
	log.Debug("is export over ",total,isok)

}


//对比-有效数据
func comparisonList(p_list []map[string]interface{} ,tmp map[string]interface{}) []map[string]interface{} {
	new_list := []map[string]interface{}{}
	for _,list := range p_list {
		b,data := isValidList(list,tmp)
		if b {
			new_list = append(new_list,data)
		}
	}
	return new_list
}


func isValidList(list map[string]interface{},tmp map[string]interface{}) (bool,map[string]interface{}) {
	publishtime := qu.Int64All(tmp["publishtime"])
	expurasingtime := qu.ObjToString(list["expurasingtime"])
	totalprice := qu.Float64All(list["totalprice"])
	if expurasingtime=="" {
		return false,nil
	}
	//时间格式转换
	exp_time := cleanStrToTimestamp(expurasingtime,publishtime)
	if exp_time>0 && exp_time<end_time && exp_time>=start_time{
	}else {
		return false,nil
	}
	//时间格式转换
	if totalprice<300000.0 {
		return false,nil
	}

	//项目名称-过滤词相关  tmp 任意可选
	//for _,v :=range words_arr {
	//	projectname:=""
	//	keywords := qu.ObjToString(v["keywords"])
	//	outwords := qu.ObjToString(v["outwords"])
	//	k_reg := regexp.MustCompile(keywords)
	//	o_reg := regexp.MustCompile(outwords)
	//	if k_reg.MatchString(projectname) && !o_reg.MatchString(projectname) {
	//		return true
	//	}
	//}

	data := map[string]interface{}{}
	exp_time_str := time.Unix(exp_time, 0).Format(timeLayout)
	data["p_name"]= qu.ObjToString(list["projectname"])
	data["p_price"]= fmt.Sprintf("%.2f",totalprice/10000.0)
	data["p_scope"]= qu.ObjToString(list["projectscope"])
	data["p_time"] = exp_time_str

	return true,data
}