timed_tasks/zhima/main.go

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"strings"
	"sync"
	"unicode"

	"github.com/xuri/excelize/v2"
	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
)

var (
	// JianZhiLabel = make([]CreditLabel, 0)
	ZhimaMap = map[string]string{}
	MgoT     *mongodb.MongodbSim //测试环境链接
	Es       *elastic.Elastic
)

func Init() {
	f, err := excelize.OpenFile("./jianyu_zhima_0829.xlsx")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer func() {
		f.Save()
		if err := f.Close(); err != nil {
			fmt.Println(err)
		}
	}()
	//2.专项债详情
	rows, err := f.GetRows("0829")
	if err != nil {
		fmt.Println(err)
		return
	}

	for i := 1; i < len(rows); i++ {
		// data := CreditLabel{
		// 	ZhimaToptype: rows[i][0],
		// 	ZhimaSubtype: rows[i][1],
		// 	ZhimaName:    rows[i][2],
		// 	ZhimaValue:   rows[i][4],
		// }
		// JianZhiLabel = append(JianZhiLabel, data)
		ZhimaMap[rows[i][2]] = rows[i][1]
	}

	//测试环境
	MgoT = &mongodb.MongodbSim{
		MongodbAddr: "192.168.3.149:27102",
		//MongodbAddr: "192.168.3.206:27002",
		DbName:   "mixdata",
		Size:     10,
		UserName: "root",
		Password: "root",
		// Direct:   true,
	}
	MgoT.InitPool()

	//测试环境es
	Es = &elastic.Elastic{
		S_esurl: "http://192.168.3.149:9201",
		//S_esurl:  "http://172.17.4.184:19805",
		I_size:   10,
		Username: "",
		Password: "",
	}
	Es.InitElasticSize()

	//正式环境
	// MgoT = &mongodb.MongodbSim{
	// 	MongodbAddr: "172.17.189.140:27080",
	// 	//MongodbAddr: "127.0.0.1:27083",
	// 	Size:     10,
	// 	DbName:   "mixdata",
	// 	UserName: "SJZY_RWbid_ES",
	// 	Password: "SJZY@B4i4D5e6S",
	// 	//Direct:   true,
	// }
	// MgoT.InitPool()

	// Es = &elastic.Elastic{
	// 	//S_esurl: "http://127.0.0.1:19908",
	// 	S_esurl:  "http://172.17.4.184:19908",
	// 	I_size:   10,
	// 	Username: "jybid",
	// 	Password: "Top2023_JEB01i@31",
	// }
	// Es.InitElasticSize()

}

func main() {
	Init()

	appID := "2021003111633206"
	privateKeyPEM := "-----BEGIN RSA PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCaJnmz3N1Z43jxtmZJmPkInl3FbfudxjicThHj+Hkndp+muzZXlAPcWoYTFLAfVa6xg9xAc58fWTPUGOpouh0ZDEP8TGvD784AljDNwRXPI9+afjcxN8gzsNPlqA6KVkAu5u5uEI1HHQgtdyRoCEnXgQRXj6DqAs9olbx8HrTxHjZzjPd+HXv+v4na3zAvZsvPdhLB2ggpVW6uQN7ur8C+hVpB3ZO+gGVFHy3m1t08XgscljyOA9Vpk4T8TnKZqDvnI6ehWFCGeD11yoe3ipHJJdP/+u9/tXiKhm29mJvcM9C6gab6BOQABByQPb2nPFFyfPglbRYrYxgKp/eXUQTzAgMBAAECggEAO6BCVyZJoKlycNzC+Bjpy0Gmt+Vw/HheEWfJV8sW4gLqgk3MGVgxnhyN2/eWoKg96IA+w9d1Z2TTvWjd0yCafMPX0FGnJaUuo990hUH4s6UGqvXDV5BiKkxtylmVNkFilkHRXFWesDE0ppF9TZhXJ7rJ+Syfa6UmoSe6PrCKAPG4FD8MH1ODj5b42l/TRIFM948nTtTLFAFgd77P5ULBCKVfBB5V9XdKhLRdwG9RxugWwkxuCiMOtmk45DzA/PQgvkIzEViD/2NJX67/S88B0G63zdQizJrvxsLp9fThmjhlo/+CaF+a+nfIzg81TfQHlRuTjiYc7iE9n5K0sU82wQKBgQDydI9pEjkPH+ZEixEymdx8D4workBfa+GXgobJ2UwatqCJOCYqZJR0MhjTVeDJiiTGFiemNyZHT0WFQedAQ/O9mkRf0MRhzR6ZOiMnCG5pSSOKdmQXoqjOqcMzdgWBQRol0eLB/vto8M3bpt1VautR0qQOR7BrFsj57K3asofikQKBgQCiwwie5gyCAYB1gkp4QkLmQDSTJdZaTlzzx2oa2/ZB4Pg6wWWNJIxEfSwtuGVH4jGJl0Eer2YpHzaY7f2NfOWJCpNI/aRhM/RilQsFXTox/5bqlbAfspPMqSHnry2OgbZb9CRF3jH7kt65390SKde4A4JNE+jJ0Xb07Tzcr7+pQwKBgCDJ8KcWBs9r0flNEVtVgYLpm1vZDO2aqlySE2Zy0579OcK07Dxpc8YvDDpwb8DxxL1hO6V36d5U8yLtYbfIOMtYdIy8sAJF+jBgs2ZQ9pWYhYzxK+IiZk/kl3h7Ko1rX3zj4JW5xQhRIJXnACHQHtjm8rbXrR0ebLmX738XC9dxAoGAQfUDQ1E6dFyjb/syrnU7ow8VvyOwL7xd0rRW/TdyHdRLA5Rn0MCkzwRiqMx2zqCNLpLK4uJ1ZeiuOV8IRxP7+IL+mK+gs3tYZ7K6Wpwi/yvyPnIWg2Hcny1S/g5xSbSq76yKHn4ihy5Hb8+WSMwIDRvu68lLpq1ppWuWJGBcLP0CgYAnGtHlFY887zSI0aRdzV3EYm/sPnmYy0TWIUVLuGL8kkprcSIW2yRxx1GtpB+3A7cPu61WwRbiswJA6VmQHvn8f9ds81+aiO5KQLMk4MbrTwWhWLDpP4BxjWANVzPr19ZwmDJcYkh7YUI2iLqH7dXho5k62OYCM1T53xlql6HDcg==\n-----END RSA PRIVATE KEY-----"
	//epCertNo := "914400007211976169" //对应企业库 credit_no
	sceneCode := "JIANYU_LABEL"

	//1.查询qyxy 数据
	defer util.Catch()
	sess := MgoT.GetMgoConn()
	defer MgoT.DestoryMongoConn(sess)
	it := sess.DB("mixdata").C("qyxy_std").Find(nil).Sort("_id").Iter()

	fmt.Println("taskRun 开始")
	count := 0
	ch := make(chan bool, 5)
	wg := &sync.WaitGroup{}
	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
		if count%1000 == 0 {
			log.Println("current", count)
		}
		// 存在芝麻标签，直接跳过
		//if _, ok := tmp["zhima_labels"]; ok {
		//	continue
		//}

		if util.ObjToString(tmp["credit_no"]) == "" {
			continue
		}
		if util.ObjToString(tmp["company_type"]) == "个体工商户" {
			continue
		}

		ch <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			//
			id := mongodb.BsonIdToSId(tmp["_id"])
			epCertNo := util.ObjToString(tmp["credit_no"])
			createLabels := getCreditLabel(appID, privateKeyPEM, epCertNo, sceneCode)
			if len(createLabels) > 0 {
				updateEs := map[string]interface{}{
					"zhima_labels": convertToInterfaceSliceEs(createLabels),
				}

				updateMgo := map[string]interface{}{
					"zhima_labels": convertToInterfaceSliceMgo(createLabels),
				}
				where := map[string]interface{}{
					"_id": tmp["_id"],
				}
				MgoT.Update("qyxy_std", where, map[string]interface{}{"$set": updateMgo}, true, false)
				err := Es.UpdateDocument("qyxy", id, updateEs)
				if err != nil && err.Error() != "Document not updated: noop" {
					log.Println("qyxy es update err", err, id)
				} else {
					log.Println("ID ", id)
				}
			}

		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg.Wait()
	log.Println("企业芝麻标签结束")
}

// getCreditLabel 获取芝麻企业信用企业标签
func getCreditLabel(appID, privateKeyPEM, certNo, sceneCode string) (createLabels []CreditLabel) {
	BizContent := BizContentForLabelQuery{
		EPCertNo:  certNo,
		ShowFlag:  true,
		SceneCode: sceneCode,
	}

	curl := "https://openapi.alipay.com/gateway.do"
	method := "zhima.credit.ep.acceptance.label.query"
	bizbyte, _ := json.Marshal(BizContent)
	priKey, err := parsePrivateKey(privateKeyPEM)
	if err != nil {
		log.Fatalln("parsePrivateKey err", err)
	}

	resMap, err := DoRequest(appID, string(bizbyte), curl, method, priKey)
	if err != nil {
		log.Fatalln("request err", err)
	}
	responseData := util.ObjToMap((*resMap)["zhima_credit_ep_acceptance_label_query_response"])
	if util.ObjToString(((*responseData)["code"])) == "10000" && util.ObjToString(((*responseData)["msg"])) == "Success" {
		if (*responseData)["label_content"] == nil {
			//log.Println("请求出错，没有返回企业标签内容模型", BizContent)
		} else {
			if labelContent, ok := (*responseData)["label_content"].([]interface{}); ok {
				for _, v := range labelContent {
					label := util.ObjToMap(v)
					labelName := util.ObjToString((*label)["label_name"])         //外显
					labelCategory := util.ObjToString((*label)["label_category"]) //一级分类
					labelCode := util.ObjToString((*label)["label_code"])
					labelDe := util.ObjToString((*label)["label_definition"])
					newName, subtype := getLabel(labelName)
					// log.Println("newName,subtype ", newName, subtype)
					data := CreditLabel{}
					data.ZhimaCode = labelCode
					data.ZhimaDefinition = labelDe
					data.ZhimaToptype = labelCategory
					data.ZhimaSubtype = subtype
					data.ZhimaName = newName
					data.ZhimaValue = labelName
					// if subtype != "" {
					createLabels = append(createLabels, data)
					// }
				}
			}
		}
	} else {
		log.Println("请求失败", BizContent, (*responseData))
	}

	return
}

// getLabel 根据labelTop  labelName 获取对应芝麻标签
func getLabel(labelName string) (newName, subtype string) {
	// for _, v := range JianZhiLabel {
	// 	if v.ZhimaToptype == labelTop && v.ZhimaName == labelName {
	// 		return v
	// 	}
	// }
	newName = getNewName(labelName)
	// log.Println("old ", labelName, " new ", newName)
	subtype = ""
	if ZhimaMap[newName] != "" {
		subtype = ZhimaMap[newName]
	}
	return
}

func getNewName(name string) string {
	if strings.Contains(name, "年企业信用") {
		return "芝麻企业信用等级连续情况"
	} else if strings.Contains(name, "芝麻企业信用等级") {
		return "芝麻企业信用等级"
	} else if strings.Contains(name, "企业信用等级") {
		return "企业信用等级_个体"
	} else if strings.Contains(name, "公开项目中标") {
		return "年度公开项目中标"
	} else if ZhimaMap[name] != "" {
		return name
	} else if strings.Contains(name, "年A级纳税人") {
		return "A级纳税人连续年限"
	} else if strings.Contains(name, "年A级纳税人") {
		return "A级纳税人连续年限"
	} else if strings.Contains(name, "融资") && strings.Contains(name, "获") {
		return "获得融资"
	} else if strings.Contains(name, "融资") && strings.Contains(name, "轮") {
		return "融资轮次"
	} else if strings.Contains(name, "年网络销量前") && strings.Contains(name, "%") {
		return "年度网络销量靠前_百分比"
	} else if strings.Contains(name, "季度网络销量前") && strings.Contains(name, "%") {
		return "季度网络销量靠前_百分比"
	} else if strings.Contains(name, "月销量全网前") && strings.Contains(name, "%") {
		return "月度网络销量靠前_百分比"
	} else if strings.Contains(name, "年网络销量前") {
		return "年度网络销量靠前"
	} else if strings.Contains(name, "季度网络销量前") {
		return "季度网络销量靠前"
	} else if strings.Contains(name, "月销量全网前") {
		return "月度网络销量靠前"
	} else if strings.Contains(name, "绝对控股") {
		return "绝对控股公司数"
	} else if strings.Contains(name, "拥有") && strings.Contains(name, "作品") {
		return "拥有作品"
	} else if strings.Contains(name, "拥有") && strings.Contains(name, "著作权") {
		return "拥有软件著作权"
	} else if strings.Contains(name, "作品创作量位于同行业TOP") {
		return "作品创作量位于同行业TOP数"
	} else if strings.Contains(name, "作品创作量位于同行业前") {
		return "作品创作量位于同行业TOP"
	} else if strings.Contains(name, "年作品创作量极速增长") && strings.Contains(name, "连续") {
		return "作品创作量极速增长连续年"
	} else if strings.Contains(name, "年作品创作量增长") && strings.Contains(name, "连续") {
		return "作品创作量增长连续年"
	} else if strings.Contains(name, "个月作品创作量增长") && strings.Contains(name, "连续") {
		return "作品创作量增长连续月"
	} else if strings.Contains(name, "年软件研发量增长") && strings.Contains(name, "连续") {
		return "软件研发量增长连续年"
	} else if strings.Contains(name, "个月软件研发量增长") && strings.Contains(name, "连续") {
		return "软件研发量增长连续月"
	} else if strings.Contains(name, "货架商品数同行前") {
		return "货架商品数位于同行TOP"
	} else if strings.Contains(name, "类目直播销量前") && strings.Contains(name, "%") {
		return "直播带货销量位于平台TOP"
	} else if strings.Contains(name, "类目直播销量前") {
		return "直播带货销量位于平台TOP数"
	} else if strings.Contains(name, "个月直播销量增长") && strings.Contains(name, "连续") {
		return "直播带货销量增长连续月"
	} else if strings.Contains(name, "类目短视频销量平台前") && strings.Contains(name, "%") {
		return "短视频带货销量位于平台TOP"
	} else if strings.Contains(name, "类目短视频销量平台前") {
		return "短视频带货销量位于平台TOP数"
	} else if strings.Contains(name, "个月短视频销量增长") && strings.Contains(name, "连续") {
		return "短视频带货销量增长连续月"
	} else if strings.Contains(name, "年度网络销量同行前") && strings.Contains(name, "%") {
		return "年度网络销量位于同行TOP"
	} else if strings.Contains(name, "季度网络销量同行前") && strings.Contains(name, "%") {
		return "季度网络销量位于同行TOP"
	} else if strings.Contains(name, "月网络销量同行前") && strings.Contains(name, "%") {
		return "月网络销量位于同行TOP"
	} else if strings.Contains(name, "年度网络销量同行前") {
		return "年度网络销量位于同行TOP数"
	} else if strings.Contains(name, "季度网络销量同行前") {
		return "季度网络销量位于同行TOP数"
	} else if strings.Contains(name, "月网络销量同行前") {
		return "月网络销量位于同行TOP数"
	}
	strs := RemoveNumbers(name)
	strs = RemoveSpecialChars(strs)
	strs = strings.ReplaceAll(strs, "前", "")
	for k, _ := range ZhimaMap {
		if strings.Contains(name, k) {
			return k
		}
		if strings.Contains(k, name) {
			return k
		}
		isOk := true
		for _, v := range strings.Split(strs, "年") {
			if !strings.Contains(k, v) && v != "" {
				isOk = false
			}
		}
		if isOk {
			return k
		}
	}
	return ""
}

// RemoveSpecialChars 去除字符串中的特定字符，如%、「、」
func RemoveSpecialChars(s string) string {
	var sb strings.Builder
	for _, r := range s {
		// 检查字符是否不是%、「或」
		// 注意：「的Unicode码点是0x300C，」的Unicode码点是0x300D
		if r != '%' && r != rune(0x300C) && r != rune(0x300D) {
			sb.WriteRune(r)
		}
	}
	return sb.String()
}

// RemoveNumbers 从字符串中移除所有数字
func RemoveNumbers(s string) string {
	var sb strings.Builder
	for _, r := range s {
		if !unicode.IsDigit(r) {
			sb.WriteRune(r)
		}
	}
	return sb.String()
}

func convertToInterfaceSliceEs(labels []CreditLabel) []map[string]interface{} {
	result := make([]map[string]interface{}, len(labels))
	for i, label := range labels {
		result[i] = map[string]interface{}{
			"zhima_toptype":    label.ZhimaToptype,
			"zhima_subtype":    label.ZhimaSubtype,
			"zhima_name":       label.ZhimaName,
			"zhima_definition": label.ZhimaDefinition,
			"zhima_value":      label.ZhimaValue,
		}
	}
	return result
}

func convertToInterfaceSliceMgo(labels []CreditLabel) []map[string]interface{} {
	result := make([]map[string]interface{}, len(labels))
	for i, label := range labels {
		result[i] = map[string]interface{}{
			"zhima_toptype":    label.ZhimaToptype,
			"zhima_subtype":    label.ZhimaSubtype,
			"zhima_name":       label.ZhimaName,
			"zhima_code":       label.ZhimaCode,
			"zhima_definition": label.ZhimaDefinition,
			"zhima_value":      label.ZhimaValue,
		}
	}
	return result
}