jy-data-extract/versioncomparison/main.go

/*
抽取结果对比
*/
package main

import (
	"flag"
	"fmt"
	"jy/mongodbutil"
	"log"
	qu "qfw/util"

	"github.com/tealeg/xlsx"
	"gopkg.in/mgo.v2/bson"
)

var (
	SysConfig   map[string]interface{}
	Premgo      *mongodbutil.Pool //上个版本库
	Newmgo      *mongodbutil.Pool //当前版本库
	FieldData   map[string]map[string]*Data
	Compares    map[string]*Compare
	Sid, Eid    string
	Fields      []string
	FieldsQuery string
)

type Compare struct {
	Field                string //属性
	PreExtNum, NewExtNum int    //上个版、当前版有值数量
	PreNilnum, NewNilnum int    //上个版、当前版无值数量
	EqNum, NEqNum        int    //相等、不等数据量
}

type Data struct {
	Id             string
	PreVal, NewVal string
}

func init() {
	flag.StringVar(&Sid, "sid", "5d348c0ca5cb26b9b76a4bb8", "开始id")
	flag.StringVar(&Eid, "eid", "5d34ae22a5cb26b9b7850b43", "结束id")
	flag.Parse()
	qu.ReadConfig(&SysConfig)
	Premgo = mongodbutil.MgoFactory(1, 3, 120, qu.ObjToString(SysConfig["premgo"]), qu.ObjToString(SysConfig["predb"]))
	Newmgo = mongodbutil.MgoFactory(1, 3, 120, qu.ObjToString(SysConfig["newmgo"]), qu.ObjToString(SysConfig["newdb"]))
	tmp, _ := SysConfig["fields"].([]interface{})
	for k, v := range tmp {
		Fields = append(Fields, qu.ObjToString(v))
		if k < (len(tmp) - 1) {
			FieldsQuery += `"` + qu.ObjToString(v) + `":1,`
		} else {
			FieldsQuery += `"` + qu.ObjToString(v) + `":1`
		}
	}
	FieldData = map[string]map[string]*Data{}
	Compares = map[string]*Compare{}
}
func main() {
	getVersionData()
	createXlsx()
}
func createXlsx() {
	xf, err := xlsx.OpenFile("template.xlsx")
	if err != nil {
		log.Println(err)
		return
	}
	//生成第一个sheet信息
	sh := xf.Sheets[0]
	for i, field := range Fields {
		for k, row := range sh.Rows {
			if k > 2+i {
				style := (*row).Cells[1].GetStyle()
				style.Font.Color = "000000"
				(*row).Cells[0].SetString(field)
				(*row).Cells[1].SetInt(Compares[field].PreExtNum)
				(*row).Cells[1].SetStyle(style)
				(*row).Cells[2].SetInt(Compares[field].NewExtNum)
				(*row).Cells[2].SetStyle(style)
				(*row).Cells[3].SetInt(Compares[field].EqNum)
				(*row).Cells[3].SetStyle(style)
				(*row).Cells[4].SetInt(Compares[field].NEqNum)
				(*row).Cells[4].SetStyle(style)
			}
			sh.Rows[k] = row
		}
	}
	//生成信息sheet
	url := "https://www.jianyu360.com/article/content/%s.html"
	for _, field := range Fields {
		sh, _ := xf.AddSheet(field)
		rowh := sh.AddRow()
		rowh.AddCell().SetString("id")
		rowh.AddCell().SetString("preval")
		rowh.AddCell().SetString("newval")
		rowh.AddCell().SetString("url")
		tmp := FieldData[field]
		for k, v := range tmp {
			if v.NewVal != v.PreVal {
				row := sh.AddRow()
				row.AddCell().SetString(k)
				row.AddCell().SetString(v.PreVal)
				row.AddCell().SetString(v.NewVal)
				row.AddCell().SetString(fmt.Sprintf(url, qu.CommonEncodeArticle("content", v.Id)))
			}
		}
	}
	err = xf.Save("result.xlsx")
	if err != nil {
		log.Println("保存xlsx失败:", err)
		return
	}
	log.Println("xlsx保存成功")
}
func getVersionData() {
	query := bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(Sid), "$lte": bson.ObjectIdHex(Eid)}}
	log.Println(qu.ObjToString(SysConfig["prec"]), query)
	list1, _ := Premgo.Find(qu.ObjToString(SysConfig["prec"]), query, nil, `{`+FieldsQuery+`}`, false, -1, -1)
	for _, v := range *list1 {
		for _, key := range Fields {
			rd := FieldData[key]
			if rd == nil {
				rd = map[string]*Data{}
			}
			rd[qu.BsonIdToSId(v["_id"])] = &Data{
				Id:     qu.BsonIdToSId(v["_id"]),
				PreVal: fmt.Sprint(v[key]),
			}
			FieldData[key] = rd
		}
	}
	log.Println("pre version 加载完成")

	list2, _ := Newmgo.Find(qu.ObjToString(SysConfig["newc"]), query, nil, `{`+FieldsQuery+`}`, false, -1, -1)
	for _, v := range *list2 {
		for _, field := range Fields {
			rd := FieldData[field]
			if rd == nil {
				rd = map[string]*Data{}
			}
			_id := qu.BsonIdToSId(v["_id"])
			tmp := rd[_id]
			if tmp != nil {
				tmp.NewVal = fmt.Sprint(v[field])
				rd[_id] = tmp
			} else {
				rd[_id] = &Data{
					NewVal: fmt.Sprint(v[field]),
				}
			}
			FieldData[field] = rd
		}
	}
	log.Println("new version 加载完成")
	for k, v := range FieldData {
		cp := &Compare{Field: k}
		for _, d := range v {
			if d.NewVal != "" && d.PreVal != "" {
				if d.NewVal == d.PreVal {
					cp.EqNum++
				} else {
					cp.NEqNum++
				}
				cp.PreExtNum++
				cp.NewExtNum++
			} else {
				if d.NewVal == "" {
					cp.NewNilnum++
					if d.PreVal != "" {
						cp.NEqNum++
						cp.PreExtNum++
					}
				}
				if d.PreVal == "" {
					cp.PreNilnum++
					if d.NewVal != "" {
						cp.NewExtNum++
						cp.NEqNum++
					}
				}
			}
		}
		Compares[k] = cp
	}
}