jypkg/common/src/qfw/util/dataexport/entdataexport.go

package dataexport

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	util "app.yhyue.com/moapp/jybase/common"
	. "app.yhyue.com/moapp/jybase/date"
	. "app.yhyue.com/moapp/jybase/encrypt"
	elastic "app.yhyue.com/moapp/jybase/es"
	"app.yhyue.com/moapp/jybase/mongodb"
	mg "app.yhyue.com/moapp/jybase/mongodb"
	"app.yhyue.com/moapp/jybase/mysql"

	"github.com/tealeg/xlsx"
	"go.mongodb.org/mongo-driver/bson"
)

// 作者：一组开发
type Filters struct {
	FilterId string
}

func GetEntDataExportCount(sim, bid mg.MongodbSim, bidMgoDBName, elasticAddress, _id string, entId, entUserId int, isFirst bool, url string, maxCount int) (count, newCount int, data *[]map[string]interface{}) {
	defer util.Catch()
	var (
		searchsWaitGroup = &sync.WaitGroup{}
	)
	count = GetDataExportSearchCountByScdId(sim, bid, bidMgoDBName, elasticAddress, _id)
	if count > maxCount || count == -1 {
		count = maxCount
	}
	log.Println("count", count)
	dataType := "2"
	//数据导出数据查询
	res, err := GetDataExportSearchResultByScdId(sim, bid, bidMgoDBName, elasticAddress, _id, dataType, count)
	if err != nil {
		log.Println("企业数据导出错误 ", err)
		return 0, 0, nil
	}
	//  20210716 由原来的redis判重改为调用判重中台接口进行判重
	m := map[string]bool{}
	infoIdList := []string{}
	insertFlag := "false"
	if !isFirst {
		insertFlag = "true"
	}
	for _, v := range *res {
		if util.IntAll(v["signendtime"]) != 0 {
			date := v["signendtime"]
			v["signendtime"] = FormatDateWithObj(&date, Date_Short_Layout)
		}
		if util.IntAll(v["bidendtime"]) != 0 {
			date := v["bidendtime"]
			v["bidendtime"] = FormatDateWithObj(&date, Date_Short_Layout)
		}
		id := util.ObjToString(v["_id"])
		if m[id] {
			continue
		}
		m[id] = true
		//  20210716  redis判重调整为调用判重中台接口  每一千个调用一次
		infoIdList = append(infoIdList, id)
		if len(infoIdList) > 1000 {
			//	 调接口
			rs, err5 := Post(url, map[string]string{
				"personId": "0", // 没有使用这个参数
				"infoId":   strings.Join(infoIdList, ","),
				"entId":    fmt.Sprintf("%d", entId),
				"isInsert": insertFlag,
				"isEnt":    "true",
			})
			log.Println("响应结果：", rs)
			if err5 != nil || util.IntAll(rs["code"]) != 0 {
				log.Println("企业订阅数据导出接口判重失败", err5)
				log.Println("企业订阅数据导出接口判重失败rs:", rs)
				log.Println("企业订阅数据导出接口判重失败rs[code]:", rs["code"])
				log.Println("企业订阅数据导出接口判重失败code是否为0", util.IntAll(rs["code"]) != 0)
				log.Println("企业订阅数据导出接口判重失败", err5, "rs:", rs, " rs[code]:", rs["code"], " ", util.IntAll(rs["code"]), "code是否为0", util.IntAll(rs["code"]) != 0)
			} else {
				log.Println("企业订阅数据导出")
				// 置空
				infoIdList = []string{}
				// 本次数据累计
				returnData := rs["data"].(map[string]interface{})
				log.Println(newCount, "加之前")
				newCount += util.IntAll(returnData["newCount"])
				//newCount += int(returnData["newCount"].(float64))
				log.Println(newCount, "加之后")
			}

		}
		if !isFirst {
			delete(v, "_id")
			v["entid"] = entId
			v["userid"] = entUserId
			v["infoid"] = id
			v["createtime"] = time.Now().Unix()
		}
	}
	if len(infoIdList) > 0 {
		rs, err5 := Post(url, map[string]string{
			"personId": "0", // 没有使用这个参数
			"infoId":   strings.Join(infoIdList, ","),
			"entId":    fmt.Sprintf("%d", entId),
			"isInsert": insertFlag,
			"isEnt":    "true",
		})
		log.Println(rs)
		if err5 != nil || util.IntAll(rs["code"]) != 0 {
			log.Println("企业订阅数据导出接口判重失败", err5)
			log.Println("企业订阅数据导出接口判重失败rs:", rs)
			log.Println("企业订阅数据导出接口判重失败rs[code]:", rs["code"])
			log.Println("企业订阅数据导出接口判重失败code是否为0", util.IntAll(rs["code"]) != 0)
			log.Println("企业订阅数据导出接口判重失败", err5, "rs:", rs, " rs[code]:", rs["code"], " ", util.IntAll(rs["code"]), "code是否为0", util.IntAll(rs["code"]) != 0)
		} else {
			log.Println("企业订阅数据导出")
			// 置空
			infoIdList = []string{}
			// 本次数据累计
			returnData := rs["data"].(map[string]interface{})
			log.Println(newCount, "加之前")
			newCount += util.IntAll(returnData["newCount"])
			//newCount += int(returnData["newCount"].(float64))
			log.Println(newCount, "加之后")

		}
	}
	searchsWaitGroup.Wait()
	log.Println("企业数据导出--数据遍历完成")
	//newCount = len(newCountPool)
	log.Println("new", newCount)
	data = res
	return
}

func FormatExportDatas(Mgo_Ent mongodb.MongodbSim, data *[]map[string]interface{}, webdomain string, dataType string, entId int) *[]map[string]interface{} {
	//格式化输出
	var (
		entexportPool      = make(chan bool, 20)
		entexportWaitGroup = &sync.WaitGroup{}
	)
	sort.Slice(*data, func(i, j int) bool {
		time1 := util.Int64All((*data)[i]["publishtime"])
		time2 := util.Int64All((*data)[j]["publishtime"])
		return time1 > time2
	})
	log.Println("补充信息开始")
	for _, v := range *data {
		entexportWaitGroup.Add(1)
		entexportPool <- true
		go func(v map[string]interface{}) {
			defer func() {
				entexportWaitGroup.Done()
				<-entexportPool
			}()
			//有中标企业 且 高级字段查询
			if dataType == "2" {
				//查询企业公示 法人 公司电话 公司邮箱地址
				s_winner := strings.Split(util.ObjToString(v["s_winner"]), ",")[0]
				if entData, ok := Mgo_Ent.Find("winner_enterprise", bson.M{"company_name": s_winner}, nil,
					`{"company_name":1,"company_email":1,"legal_person":1,"company_phone":1}`, false, -1, -1); ok {
					if entData != nil && *entData != nil && len(*entData) > 0 {
						for _, ev := range *entData {
							if v["s_winner"] == ev["company_name"] {
								legal_person := ""
								if ev["legal_person"] != nil {
									legal_person = ev["legal_person"].(string)
								}
								company_phone := ""
								if ev["company_phone"] != nil {
									company_phone = ev["company_phone"].(string)
								}
								company_email := ""
								if ev["company_email"] != nil && ev["company_email"] != "无" {
									company_email = ev["company_email"].(string)
								}
								v["legal_person"] = legal_person
								v["company_phone"] = company_phone
								v["company_email"] = company_email
							}
						}
					}
				}
			}
			//====================字段补漏=========================
			if v["toptype"] == "结果" && dataType == "2" && !(v["agency"] != nil && v["budget"] != nil && v["buyerperson"] != nil && v["buyertel"] != nil) {
				r := elastic.Get("projectset", "projectset", fmt.Sprintf(`{"query":{"term":{"list.infoid":"%s"}},"_source": ["list"]}`, v["_id"]))
				if len(*r) > 0 {
					MsgList := (*r)[0]["list"]
					if MsgList != nil {
						list := util.ObjArrToMapArr(MsgList.([]interface{}))
						for _, vv := range list {
							if vv["subtype"] == "招标" {
								if v["agency"] == nil && vv["agency"] != nil {
									v["agency"] = vv["agency"]
								}
								if v["budget"] == nil && vv["budget"] != nil {
									v["budget"] = vv["budget"]
								}
								if v["buyerperson"] == nil && vv["buyerperson"] != nil {
									v["buyerperson"] = vv["buyerperson"]
								}
								if v["buyertel"] == nil && vv["buyertel"] != nil {
									v["buyertel"] = vv["buyertel"]
								}
								break
							}
						}
					}
				}
			}
			if v["area"] == "A" {
				v["area"] = "全国"
			}
			if v["publishtime"] != nil {
				date := v["publishtime"]
				v["publishtime"] = FormatDateWithObj(&date, Date_Short_Layout)
			}
			if v["bidopentime"] != nil {
				date := v["bidopentime"]
				v["bidopentime"] = FormatDateWithObj(&date, Date_Short_Layout)
			}
			if util.IntAll(v["signendtime"]) != 0 {
				date := v["signendtime"]
				v["signendtime"] = FormatDateWithObj(&date, Date_Short_Layout)
			}
			if util.IntAll(v["bidendtime"]) != 0 {
				date := v["bidendtime"]
				v["bidendtime"] = FormatDateWithObj(&date, Date_Short_Layout)
			}
			if v["currency"] == "" || v["currency"] == nil {
				v["currency"] = "人民币"
			}
			if v["subtype"] == nil && v["toptype"] != nil {
				v["subtype"] = v["toptype"]
			}
			if v["detail"] != "" && v["detail"] != nil {
				str := ClearHtml.ReplaceAllString(v["detail"].(string), "")
				str = ClearOther.ReplaceAllString(str, "")
				str = strings.Replace(str, " ", "", -1)
				v["detail"] = str
			}
			if v["infoid"] != nil {
				v["url"] = webdomain + "/article/content/" + CommonEncodeArticle("content", v["infoid"].(string)) + ".html"
			}
		}(v)
	}
	entexportWaitGroup.Wait()
	log.Println("补充信息结束")
	return data
}

func Post(url string, form map[string]string) (data map[string]interface{}, err error) {
	str := ""
	for k, v := range form {
		str += "&" + k + "=" + v
	}
	//log.Println(str)
	res, err1 := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(str))
	log.Println(res)
	if err1 != nil {
		log.Println("post err:", err1.Error())
		return nil, err1

	} else if res.Body != nil {
		defer res.Body.Close()
		bs, _ := ioutil.ReadAll(res.Body)
		err2 := json.Unmarshal(bs, &data)
		if err2 != nil {
			return nil, err2
		}

	}
	return data, nil
}

// 生成xlsx
func GetXlsx(mMap []map[string]interface{}, entId, entUserId int, filePath string) string {
	xf, err := xlsx.OpenFile("./web/staticres/fields.xlsx")
	if err != nil {
		log.Println("fields file not foud", err.Error())
	}
	sh := xf.Sheets[1]
	for _, v := range mMap {
		row := sh.AddRow()
		row.AddCell().SetValue(v["keyword"])
		row.AddCell().SetValue(v["area"])
		row.AddCell().SetValue(v["city"])
		row.AddCell().SetValue(v["district"])
		row.AddCell().SetValue(v["title"])
		row.AddCell().SetValue(v["subtype"])
		row.AddCell().SetValue(v["detail"])
		if v["publishtime"] != nil {
			row.AddCell().SetValue(v["publishtime"])
		} else {
			row.AddCell()
		}
		row.AddCell().SetValue(v["href"])
		row.AddCell().SetValue(v["url"])
		row.AddCell().SetValue(v["projectname"])
		row.AddCell().SetValue(v["projectcode"])
		row.AddCell().SetValue(v["projectscope"])
		if v["budget"] != nil {
			row.AddCell().SetFloat(util.Float64All(v["budget"]))
		} else {
			row.AddCell()
		}
		if v["bidamount"] != nil {
			row.AddCell().SetFloat(util.Float64All(v["bidamount"]))
		} else {
			row.AddCell()
		}
		if v["signendtime"] != nil {
			row.AddCell().SetValue(v["signendtime"])
		} else {
			row.AddCell()
		}
		if v["bidopentime"] != nil {
			row.AddCell().SetValue(v["bidopentime"])
		} else {
			row.AddCell()
		}
		if v["bidendtime"] != nil {
			row.AddCell().SetValue(v["bidendtime"])
		} else {
			row.AddCell()
		}
		row.AddCell().SetValue(v["buyer"])
		row.AddCell().SetValue(v["buyerperson"])
		row.AddCell().SetValue(v["buyertel"])
		row.AddCell().SetValue(v["buyeraddr"])
		row.AddCell().SetValue(v["agency"])
		row.AddCell().SetValue(v["s_winner"])
		row.AddCell().SetValue(v["winnerperson"])
		row.AddCell().SetValue(v["winnertel"])
		row.AddCell().SetValue(v["legal_person"])
		row.AddCell().SetValue(v["company_phone"])
		row.AddCell().SetValue(v["company_email"])
	}
	xf.Sheets = xf.Sheets[1:2]
	xf.Sheets[0].Name = "数据导出"
	//生文件
	t := strconv.FormatInt(time.Now().Unix(), 10)
	entIds := strconv.Itoa(entId)
	entUserIds := strconv.Itoa(entUserId)

	dir := filePath + "/entsearchexport/" + entIds + "_" + entUserIds + "_" + t + "/"
	if b, _ := PathExists(dir); !b {
		err1 := os.MkdirAll(dir, os.ModePerm)
		if err1 != nil {
			log.Println("mkdir err", dir)
		}
	}
	fname := entIds + "_" + entUserIds + "_" + "entdataexport.xlsx"

	download_url := "/entsearchexport/" + entIds + "_" + entUserIds + "_" + t + "/" + fname
	err = xf.Save(dir + fname)
	if err != nil {
		log.Println("xls error", fname)
		return ""
	}
	return download_url
}

func PathExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func SaveExportLog(mysqlSess *mysql.Mysql, entId, entUserId, count, newCount, remain_nums, export_nums int, xlsxUrl, types, filterStr string, exportPhone, exportEmail string) {
	query := map[string]interface{}{
		"id":     entUserId,
		"ent_id": entId,
	}
	set := map[string]interface{}{
		"remain_nums": remain_nums - newCount,
		"export_nums": export_nums + newCount,
	}
	ok := mysqlSess.Update("entniche_export_limit", map[string]interface{}{"ent_id": entId, "user_id": entUserId}, set)
	if !ok {
		log.Println("修改导出条数失败", query, remain_nums, newCount)
	}
	userData := mysqlSess.FindOne("entniche_user", query, "name,phone", "")
	if userData != nil {
		name := util.ObjToString((*userData)["name"])
		phone := util.ObjToString((*userData)["phone"])
		now := time.Now()
		mysqlSess.Insert("entniche_export_log", map[string]interface{}{
			"user_name":    name,
			"export_time":  FormatDate(&now, Date_Full_Layout),
			"data_source":  "2",
			"export_num":   count,
			"deduct_num":   newCount,
			"download_url": xlsxUrl,
			"ent_id":       entId,
			"phone":        phone,
			"user_id":      entUserId,
			"filter":       filterStr,
			"export_phone": exportPhone,
			"export_mail":  exportEmail,
		})
	}
}

func DeductNum(m *mysql.Mysql, qyfw mongodb.MongodbSim, entId, newCount int) {
	query := map[string]interface{}{
		"id": entId,
	}
	userData := m.FindOne("entniche_info", query, "name,phone", "")
	if userData != nil {
		name := util.ObjToString((*userData)["name"])
		phone := util.ObjToString((*userData)["phone"])
		qyfw.Update("user", map[string]interface{}{"phone": phone, "username": name}, map[string]interface{}{
			"$inc": map[string]interface{}{
				"plan.current": -newCount,
			},
		}, false, false)
	}
}
func GetCurrentCount(m *mysql.Mysql, qyfw mongodb.MongodbSim, entId int) int {
	count := 0
	userData := m.FindOne("entniche_info", map[string]interface{}{"id": entId}, "name,phone", "")
	if userData == nil {
		return count
	}
	current, ok := qyfw.FindOne("user", map[string]interface{}{"phone": util.ObjToString((*userData)["phone"]), "username": util.ObjToString((*userData)["name"])})
	if current == nil || !ok {
		return count
	}
	plan, _ := (*current)["plan"].(map[string]interface{})
	count = util.IntAll(plan["current"])
	return count
}