package dataexport import ( "encoding/json" "fmt" "io/ioutil" "log" "net/http" "os" "sort" "strconv" "strings" "sync" "time" util "app.yhyue.com/moapp/jybase/common" . "app.yhyue.com/moapp/jybase/date" . "app.yhyue.com/moapp/jybase/encrypt" elastic "app.yhyue.com/moapp/jybase/es" "app.yhyue.com/moapp/jybase/mongodb" mg "app.yhyue.com/moapp/jybase/mongodb" "app.yhyue.com/moapp/jybase/mysql" "github.com/tealeg/xlsx" "go.mongodb.org/mongo-driver/bson" ) // 作者:一组开发 type Filters struct { FilterId string } func GetEntDataExportCount(sim, bid mg.MongodbSim, bidMgoDBName, elasticAddress, _id string, entId, entUserId int, isFirst bool, url string, maxCount int) (count, newCount int, data *[]map[string]interface{}) { defer util.Catch() var ( searchsWaitGroup = &sync.WaitGroup{} ) count = GetDataExportSearchCountByScdId(sim, bid, bidMgoDBName, elasticAddress, _id) if count > maxCount || count == -1 { count = maxCount } log.Println("count", count) dataType := "2" //数据导出数据查询 res, err := GetDataExportSearchResultByScdId(sim, bid, bidMgoDBName, elasticAddress, _id, dataType, count) if err != nil { log.Println("企业数据导出错误 ", err) return 0, 0, nil } // 20210716 由原来的redis判重改为调用判重中台接口进行判重 m := map[string]bool{} infoIdList := []string{} insertFlag := "false" if !isFirst { insertFlag = "true" } for _, v := range *res { if util.IntAll(v["signendtime"]) != 0 { date := v["signendtime"] v["signendtime"] = FormatDateWithObj(&date, Date_Short_Layout) } if util.IntAll(v["bidendtime"]) != 0 { date := v["bidendtime"] v["bidendtime"] = FormatDateWithObj(&date, Date_Short_Layout) } id := util.ObjToString(v["_id"]) if m[id] { continue } m[id] = true // 20210716 redis判重调整为调用判重中台接口 每一千个调用一次 infoIdList = append(infoIdList, id) if len(infoIdList) > 1000 { // 调接口 rs, err5 := Post(url, map[string]string{ "personId": "0", // 没有使用这个参数 "infoId": strings.Join(infoIdList, ","), "entId": fmt.Sprintf("%d", entId), "isInsert": insertFlag, "isEnt": "true", }) log.Println("响应结果:", rs) if err5 != nil || util.IntAll(rs["code"]) != 0 { log.Println("企业订阅数据导出接口判重失败", err5) log.Println("企业订阅数据导出接口判重失败rs:", rs) log.Println("企业订阅数据导出接口判重失败rs[code]:", rs["code"]) log.Println("企业订阅数据导出接口判重失败code是否为0", util.IntAll(rs["code"]) != 0) log.Println("企业订阅数据导出接口判重失败", err5, "rs:", rs, " rs[code]:", rs["code"], " ", util.IntAll(rs["code"]), "code是否为0", util.IntAll(rs["code"]) != 0) } else { log.Println("企业订阅数据导出") // 置空 infoIdList = []string{} // 本次数据累计 returnData := rs["data"].(map[string]interface{}) log.Println(newCount, "加之前") newCount += util.IntAll(returnData["newCount"]) //newCount += int(returnData["newCount"].(float64)) log.Println(newCount, "加之后") } } if !isFirst { delete(v, "_id") v["entid"] = entId v["userid"] = entUserId v["infoid"] = id v["createtime"] = time.Now().Unix() } } if len(infoIdList) > 0 { rs, err5 := Post(url, map[string]string{ "personId": "0", // 没有使用这个参数 "infoId": strings.Join(infoIdList, ","), "entId": fmt.Sprintf("%d", entId), "isInsert": insertFlag, "isEnt": "true", }) log.Println(rs) if err5 != nil || util.IntAll(rs["code"]) != 0 { log.Println("企业订阅数据导出接口判重失败", err5) log.Println("企业订阅数据导出接口判重失败rs:", rs) log.Println("企业订阅数据导出接口判重失败rs[code]:", rs["code"]) log.Println("企业订阅数据导出接口判重失败code是否为0", util.IntAll(rs["code"]) != 0) log.Println("企业订阅数据导出接口判重失败", err5, "rs:", rs, " rs[code]:", rs["code"], " ", util.IntAll(rs["code"]), "code是否为0", util.IntAll(rs["code"]) != 0) } else { log.Println("企业订阅数据导出") // 置空 infoIdList = []string{} // 本次数据累计 returnData := rs["data"].(map[string]interface{}) log.Println(newCount, "加之前") newCount += util.IntAll(returnData["newCount"]) //newCount += int(returnData["newCount"].(float64)) log.Println(newCount, "加之后") } } searchsWaitGroup.Wait() log.Println("企业数据导出--数据遍历完成") //newCount = len(newCountPool) log.Println("new", newCount) data = res return } func FormatExportDatas(Mgo_Ent mongodb.MongodbSim, data *[]map[string]interface{}, webdomain string, dataType string, entId int) *[]map[string]interface{} { //格式化输出 var ( entexportPool = make(chan bool, 20) entexportWaitGroup = &sync.WaitGroup{} ) sort.Slice(*data, func(i, j int) bool { time1 := util.Int64All((*data)[i]["publishtime"]) time2 := util.Int64All((*data)[j]["publishtime"]) return time1 > time2 }) log.Println("补充信息开始") for _, v := range *data { entexportWaitGroup.Add(1) entexportPool <- true go func(v map[string]interface{}) { defer func() { entexportWaitGroup.Done() <-entexportPool }() //有中标企业 且 高级字段查询 if dataType == "2" { //查询企业公示 法人 公司电话 公司邮箱地址 s_winner := strings.Split(util.ObjToString(v["s_winner"]), ",")[0] if entData, ok := Mgo_Ent.Find("winner_enterprise", bson.M{"company_name": s_winner}, nil, `{"company_name":1,"company_email":1,"legal_person":1,"company_phone":1}`, false, -1, -1); ok { if entData != nil && *entData != nil && len(*entData) > 0 { for _, ev := range *entData { if v["s_winner"] == ev["company_name"] { legal_person := "" if ev["legal_person"] != nil { legal_person = ev["legal_person"].(string) } company_phone := "" if ev["company_phone"] != nil { company_phone = ev["company_phone"].(string) } company_email := "" if ev["company_email"] != nil && ev["company_email"] != "无" { company_email = ev["company_email"].(string) } v["legal_person"] = legal_person v["company_phone"] = company_phone v["company_email"] = company_email } } } } } //====================字段补漏========================= if v["toptype"] == "结果" && dataType == "2" && !(v["agency"] != nil && v["budget"] != nil && v["buyerperson"] != nil && v["buyertel"] != nil) { r := elastic.Get("projectset", "projectset", fmt.Sprintf(`{"query":{"term":{"list.infoid":"%s"}},"_source": ["list"]}`, v["_id"])) if len(*r) > 0 { MsgList := (*r)[0]["list"] if MsgList != nil { list := util.ObjArrToMapArr(MsgList.([]interface{})) for _, vv := range list { if vv["subtype"] == "招标" { if v["agency"] == nil && vv["agency"] != nil { v["agency"] = vv["agency"] } if v["budget"] == nil && vv["budget"] != nil { v["budget"] = vv["budget"] } if v["buyerperson"] == nil && vv["buyerperson"] != nil { v["buyerperson"] = vv["buyerperson"] } if v["buyertel"] == nil && vv["buyertel"] != nil { v["buyertel"] = vv["buyertel"] } break } } } } } if v["area"] == "A" { v["area"] = "全国" } if v["publishtime"] != nil { date := v["publishtime"] v["publishtime"] = FormatDateWithObj(&date, Date_Short_Layout) } if v["bidopentime"] != nil { date := v["bidopentime"] v["bidopentime"] = FormatDateWithObj(&date, Date_Short_Layout) } if util.IntAll(v["signendtime"]) != 0 { date := v["signendtime"] v["signendtime"] = FormatDateWithObj(&date, Date_Short_Layout) } if util.IntAll(v["bidendtime"]) != 0 { date := v["bidendtime"] v["bidendtime"] = FormatDateWithObj(&date, Date_Short_Layout) } if v["currency"] == "" || v["currency"] == nil { v["currency"] = "人民币" } if v["subtype"] == nil && v["toptype"] != nil { v["subtype"] = v["toptype"] } if v["detail"] != "" && v["detail"] != nil { str := ClearHtml.ReplaceAllString(v["detail"].(string), "") str = ClearOther.ReplaceAllString(str, "") str = strings.Replace(str, " ", "", -1) v["detail"] = str } if v["infoid"] != nil { v["url"] = webdomain + "/article/content/" + CommonEncodeArticle("content", v["infoid"].(string)) + ".html" } }(v) } entexportWaitGroup.Wait() log.Println("补充信息结束") return data } func Post(url string, form map[string]string) (data map[string]interface{}, err error) { str := "" for k, v := range form { str += "&" + k + "=" + v } //log.Println(str) res, err1 := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(str)) log.Println(res) if err1 != nil { log.Println("post err:", err1.Error()) return nil, err1 } else if res.Body != nil { defer res.Body.Close() bs, _ := ioutil.ReadAll(res.Body) err2 := json.Unmarshal(bs, &data) if err2 != nil { return nil, err2 } } return data, nil } // 生成xlsx func GetXlsx(mMap []map[string]interface{}, entId, entUserId int, filePath string) string { xf, err := xlsx.OpenFile("./web/staticres/fields.xlsx") if err != nil { log.Println("fields file not foud", err.Error()) } sh := xf.Sheets[1] for _, v := range mMap { row := sh.AddRow() row.AddCell().SetValue(v["keyword"]) row.AddCell().SetValue(v["area"]) row.AddCell().SetValue(v["city"]) row.AddCell().SetValue(v["district"]) row.AddCell().SetValue(v["title"]) row.AddCell().SetValue(v["subtype"]) row.AddCell().SetValue(v["detail"]) if v["publishtime"] != nil { row.AddCell().SetValue(v["publishtime"]) } else { row.AddCell() } row.AddCell().SetValue(v["href"]) row.AddCell().SetValue(v["url"]) row.AddCell().SetValue(v["projectname"]) row.AddCell().SetValue(v["projectcode"]) row.AddCell().SetValue(v["projectscope"]) if v["budget"] != nil { row.AddCell().SetFloat(util.Float64All(v["budget"])) } else { row.AddCell() } if v["bidamount"] != nil { row.AddCell().SetFloat(util.Float64All(v["bidamount"])) } else { row.AddCell() } if v["signendtime"] != nil { row.AddCell().SetValue(v["signendtime"]) } else { row.AddCell() } if v["bidopentime"] != nil { row.AddCell().SetValue(v["bidopentime"]) } else { row.AddCell() } if v["bidendtime"] != nil { row.AddCell().SetValue(v["bidendtime"]) } else { row.AddCell() } row.AddCell().SetValue(v["buyer"]) row.AddCell().SetValue(v["buyerperson"]) row.AddCell().SetValue(v["buyertel"]) row.AddCell().SetValue(v["buyeraddr"]) row.AddCell().SetValue(v["agency"]) row.AddCell().SetValue(v["s_winner"]) row.AddCell().SetValue(v["winnerperson"]) row.AddCell().SetValue(v["winnertel"]) row.AddCell().SetValue(v["legal_person"]) row.AddCell().SetValue(v["company_phone"]) row.AddCell().SetValue(v["company_email"]) } xf.Sheets = xf.Sheets[1:2] xf.Sheets[0].Name = "数据导出" //生文件 t := strconv.FormatInt(time.Now().Unix(), 10) entIds := strconv.Itoa(entId) entUserIds := strconv.Itoa(entUserId) dir := filePath + "/entsearchexport/" + entIds + "_" + entUserIds + "_" + t + "/" if b, _ := PathExists(dir); !b { err1 := os.MkdirAll(dir, os.ModePerm) if err1 != nil { log.Println("mkdir err", dir) } } fname := entIds + "_" + entUserIds + "_" + "entdataexport.xlsx" download_url := "/entsearchexport/" + entIds + "_" + entUserIds + "_" + t + "/" + fname err = xf.Save(dir + fname) if err != nil { log.Println("xls error", fname) return "" } return download_url } func PathExists(path string) (bool, error) { _, err := os.Stat(path) if err == nil { return true, nil } if os.IsNotExist(err) { return false, nil } return false, err } func SaveExportLog(mysqlSess *mysql.Mysql, entId, entUserId, count, newCount, remain_nums, export_nums int, xlsxUrl, types, filterStr string, exportPhone, exportEmail string) { query := map[string]interface{}{ "id": entUserId, "ent_id": entId, } set := map[string]interface{}{ "remain_nums": remain_nums - newCount, "export_nums": export_nums + newCount, } ok := mysqlSess.Update("entniche_export_limit", map[string]interface{}{"ent_id": entId, "user_id": entUserId}, set) if !ok { log.Println("修改导出条数失败", query, remain_nums, newCount) } userData := mysqlSess.FindOne("entniche_user", query, "name,phone", "") if userData != nil { name := util.ObjToString((*userData)["name"]) phone := util.ObjToString((*userData)["phone"]) now := time.Now() mysqlSess.Insert("entniche_export_log", map[string]interface{}{ "user_name": name, "export_time": FormatDate(&now, Date_Full_Layout), "data_source": "2", "export_num": count, "deduct_num": newCount, "download_url": xlsxUrl, "ent_id": entId, "phone": phone, "user_id": entUserId, "filter": filterStr, "export_phone": exportPhone, "export_mail": exportEmail, }) } } func DeductNum(m *mysql.Mysql, qyfw mongodb.MongodbSim, entId, newCount int) { query := map[string]interface{}{ "id": entId, } userData := m.FindOne("entniche_info", query, "name,phone", "") if userData != nil { name := util.ObjToString((*userData)["name"]) phone := util.ObjToString((*userData)["phone"]) qyfw.Update("user", map[string]interface{}{"phone": phone, "username": name}, map[string]interface{}{ "$inc": map[string]interface{}{ "plan.current": -newCount, }, }, false, false) } } func GetCurrentCount(m *mysql.Mysql, qyfw mongodb.MongodbSim, entId int) int { count := 0 userData := m.FindOne("entniche_info", map[string]interface{}{"id": entId}, "name,phone", "") if userData == nil { return count } current, ok := qyfw.FindOne("user", map[string]interface{}{"phone": util.ObjToString((*userData)["phone"]), "username": util.ObjToString((*userData)["name"])}) if current == nil || !ok { return count } plan, _ := (*current)["plan"].(map[string]interface{}) count = util.IntAll(plan["current"]) return count }