yusuan/yusuan_yunpan/deduplication.go

package main

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"log"
	"strings"

	"app.yhyue.com/moapp/jybase/mongodb"
	"github.com/PuerkitoBio/goquery"
	"github.com/gogf/gf/v2/util/gconv"
)

//年份、项目名称、项目情况、绩效目标、实施单位 的哈希值
func Hash(year, projectname, procure_content, kpi, institution string) string {
	procure_content = CleanString(procure_content)
	kpi = CleanString(kpi)
	str := fmt.Sprintf("%s%s%s%s%s", year, projectname, procure_content, kpi, institution)
	return HashCode(str)
}

//刷库 更新project_yusuan
func UpdateProject_yusuan() {
	log.Println("开始..")
	sess := db.GetMgoConn()
	defer db.DestoryMongoConn(sess)

	it := sess.DB(cf.Collections).C(cf.ProjectItem).Find(nil).Select(map[string]interface{}{
		"year":            1,
		"projectname":     1,
		"procure_content": 1,
		"kpi":             1,
		"institution":     1,
		"_id":             1,
		"hash_code":       1,
	}).Iter()
	i := 0
	for m := make(map[string]interface{}); it.Next(&m); {
		i++
		if i%1000 == 0 {
			log.Println(cf.ProjectItem, "count:", i)
		}
		hash_code := gconv.String(m["hash_code"])
		year := gconv.String(m["year"])
		projectname := gconv.String(m["projectname"])
		procure_content := gconv.String(m["procure_content"])
		kpi := gconv.String(m["kpi"])
		institution := gconv.String(m["institution"])
		id := mongodb.BsonIdToSId(m["_id"])
		//清洗
		procure_content = CleanString(procure_content)
		kpi = CleanString(kpi)
		newHashCode := Hash(year, projectname, procure_content, kpi, institution)
		//修改hash值
		if hash_code != newHashCode {
			db.UpdateById(cf.ProjectItem, id, map[string]interface{}{
				"$set": map[string]interface{}{
					"hash_code": newHashCode,
				},
			})
		}

		m = make(map[string]interface{})
	}
}

//生成hashCode
func HashCode(input string) string {
	hash := sha256.Sum256([]byte(input))
	hashString := hex.EncodeToString(hash[:])
	return hashString
}

// 纯文本
func HtmlToText(con string) string {
	doc2, _ := goquery.NewDocumentFromReader(strings.NewReader(con))
	//log.Println(doc2.Html())
	doc2.Find("tr").Each(func(i int, selection *goquery.Selection) {
		selection.AfterHtml(string(rune(10)))
	})
	//log.Println(doc2.Html())
	return doc2.Text()
}

//刷库 更新project_yusuan
func UpdateProject_huipu() {
	huipudb := "zxl_project_huipu"

	sess := db.GetMgoConn()
	defer db.DestoryMongoConn(sess)

	it := sess.DB(cf.Collections).C(huipudb).Find(nil).Select(map[string]interface{}{
		"year":            1,
		"projectname":     1,
		"procure_content": 1,
		"kpi":             1,
		"institution":     1,
		"_id":             1,
		"hash_code":       1,
	}).Iter()
	i := 0
	for m := make(map[string]interface{}); it.Next(&m); {
		i++
		if i%1000 == 0 {
			log.Println("count:", i)
		}
		hash_code := gconv.String(m["hash_code"])
		year := gconv.String(m["year"])
		projectname := gconv.String(m["projectname"])
		procure_content := gconv.String(m["procure_content"])
		kpi := gconv.String(m["kpi"])
		institution := gconv.String(m["institution"])
		id := mongodb.BsonIdToSId(m["_id"])
		//清洗
		procure_content = CleanString(procure_content)
		kpi = CleanString(kpi)
		newHashCode := Hash(year, projectname, procure_content, kpi, institution)
		//修改hash值
		if hash_code != newHashCode {
			db.UpdateById(huipudb, id, map[string]interface{}{
				"$set": map[string]interface{}{
					"hash_code": newHashCode,
				},
			})
		}

		m = make(map[string]interface{})
	}
}