package main import ( "crypto/sha256" "encoding/hex" "fmt" "log" "strings" "app.yhyue.com/moapp/jybase/mongodb" "github.com/PuerkitoBio/goquery" "github.com/gogf/gf/v2/util/gconv" ) //年份、项目名称、项目情况、绩效目标、实施单位 的哈希值 func Hash(year, projectname, procure_content, kpi, institution string) string { procure_content = CleanString(procure_content) kpi = CleanString(kpi) str := fmt.Sprintf("%s%s%s%s%s", year, projectname, procure_content, kpi, institution) return HashCode(str) } //刷库 更新project_yusuan func UpdateProject_yusuan() { sess := db.GetMgoConn() defer db.DestoryMongoConn(sess) it := sess.DB(cf.Collections).C(cf.ProjectItem).Find(nil).Select(map[string]interface{}{ "year": 1, "projectname": 1, "procure_content": 1, "kpi": 1, "institution": 1, "_id": 1, "hash_code": 1, }).Iter() i := 0 for m := make(map[string]interface{}); it.Next(&m); { i++ if i%1000 == 0 { log.Println("count:", i) } hash_code := gconv.String(m["hash_code"]) year := gconv.String(m["year"]) projectname := gconv.String(m["projectname"]) procure_content := gconv.String(m["procure_content"]) kpi := gconv.String(m["kpi"]) institution := gconv.String(m["institution"]) id := mongodb.BsonIdToSId(m["_id"]) //清洗 procure_content = CleanString(procure_content) kpi = CleanString(kpi) newHashCode := Hash(year, projectname, procure_content, kpi, institution) //修改hash值 if hash_code != newHashCode { db.UpdateById(cf.ProjectItem, id, map[string]interface{}{ "$set": map[string]interface{}{ "hash_code": newHashCode, }, }) } m = make(map[string]interface{}) } } //生成hashCode func HashCode(input string) string { hash := sha256.Sum256([]byte(input)) hashString := hex.EncodeToString(hash[:]) return hashString } // 纯文本 func HtmlToText(con string) string { doc2, _ := goquery.NewDocumentFromReader(strings.NewReader(con)) //log.Println(doc2.Html()) doc2.Find("tr").Each(func(i int, selection *goquery.Selection) { selection.AfterHtml(string(rune(10))) }) //log.Println(doc2.Html()) return doc2.Text() } //刷库 更新project_yusuan func UpdateProject_huipu() { huipudb := "zxl_project_huipu" sess := db.GetMgoConn() defer db.DestoryMongoConn(sess) it := sess.DB(cf.Collections).C(huipudb).Find(nil).Select(map[string]interface{}{ "year": 1, "projectname": 1, "procure_content": 1, "kpi": 1, "institution": 1, "_id": 1, "hash_code": 1, }).Iter() i := 0 for m := make(map[string]interface{}); it.Next(&m); { i++ if i%1000 == 0 { log.Println("count:", i) } hash_code := gconv.String(m["hash_code"]) year := gconv.String(m["year"]) projectname := gconv.String(m["projectname"]) procure_content := gconv.String(m["procure_content"]) kpi := gconv.String(m["kpi"]) institution := gconv.String(m["institution"]) id := mongodb.BsonIdToSId(m["_id"]) //清洗 procure_content = CleanString(procure_content) kpi = CleanString(kpi) newHashCode := Hash(year, projectname, procure_content, kpi, institution) //修改hash值 if hash_code != newHashCode { db.UpdateById(huipudb, id, map[string]interface{}{ "$set": map[string]interface{}{ "hash_code": newHashCode, }, }) } m = make(map[string]interface{}) } }