优化，配置文件读取

versioncomparison/config.json

@@ -7,7 +7,7 @@
   "previousc": "result_v3",
   "newmgo": "192.168.3.207:27081",
   "newdb": "extract_v3",
-  "newc": "result_data",
+  "newc": "result_data2",
   "keyfield": {
     "bidamount": 1,
     "budget": 1,
@@ -15,5 +15,7 @@
     "projectcode":1,
     "buyer": 1,
     "projectname": 1
-  }
+  },
+  "queryNum": 1000,
+  "querySid": "57319151edbcdc7b27000aec"
 }

versioncomparison/main.go

@@ -42,17 +42,20 @@ type versionComparison struct {
 }
 
 func main() {
-	Query(1000)
+	Query(util.IntAll(SysConfig["queryNum"]), util.ObjToString(SysConfig["querySid"]))
 }
 
-func Query(num int) {
+func Query(num int, sid string) {
 	xf, err := xlsx.OpenFile("抽取结果对比.xlsx")
 	if err != nil {
 		log.Println("读取文件", err)
 		return
 	}
-	var projectcodenum, bidamountnum, winnernum, buyernum, budgetnum, projectnamenum int //不相等计数器
-	log.Println(num)
+	var projectcodenum, bidamountnum, winnernum, buyernum, budgetnum, projectnamenum int                               //不相等计数器
+	var pcodeNotNilNumP, bidamountNotNilNumP, winnerNotNilNumP, buyerNotNilNumP, budgetNotNilNumP, pnameNotNilNumP int //不相等计数器
+	var pcodeNotNilNumN, bidamountNotNilNumN, winnerNotNilNumN, buyerNotNilNumN, budgetNotNilNumN, pnameNotNilNumN int //不相等计数器
+
+	log.Println(num, sid)
 	if num < 1 {
 		log.Println("查询数量应该大于0")
 		return
@@ -61,30 +64,42 @@ func Query(num int) {
 	//if strings.TrimSpace(gteid) == "" {
 	//	gteid = "386cd3000000000000000000"
 	//}
-
-	iter := Extractmgo.DB(util.ObjToString(SysConfig["extractdb"])).C(util.ObjToString(SysConfig["extractc"])).Find(nil).Select(bson.M{"_id": 1, "href": 1}).Iter()
+	var iter *mgo.Iter
+	if sid == "" {
+		iter = Extractmgo.DB(util.ObjToString(SysConfig["extractdb"])).C(util.ObjToString(SysConfig["extractc"])).Find(nil).Select(bson.M{"_id": 1}).Iter()
+	} else {
+		iter = Extractmgo.DB(util.ObjToString(SysConfig["extractdb"])).C(util.ObjToString(SysConfig["extractc"])).Find(bson.M{"_id": bson.M{
+			"$gte": bson.ObjectIdHex(sid)},
+		}).Select(bson.M{"_id": 1}).Iter()
+	}
 	defer log.Println("关闭 iter:", iter.Close())
-	var data map[string]interface{}
+	var data map[string]bson.ObjectId
+	getdata := make([]bson.ObjectId, 0)
+	for iter.Next(&data) {
+		if num == 0 {
+			break
+		}
+		getdata = append(getdata, data["_id"])
+		num--
+	}
+	log.Println(sum, "条数据加载完成")
 	projectnames := make([]*Projectname, 0)
 	buyers := make([]*Buyer, 0)
 	projectcodes := make([]*Projectcode, 0)
 	winners := make([]*Winner, 0)
 	budgets := make([]*Budget, 0)
 	bidamounts := make([]*Bidamount, 0)
-	for iter.Next(&data) {
-		if num == 0 {
-			break
-		}
-		log.Println(data["_id"])
-		id := data["_id"].(bson.ObjectId).Hex()
-		pdata, b := Previousmgo.FindById(util.ObjToString(SysConfig["previousc"]), id, SysConfig["keyfield"])
+	for _, gv := range getdata {
+		log.Println(gv)
+		gvid := gv.Hex()
+		pdata, b := Previousmgo.FindById(util.ObjToString(SysConfig["previousc"]), gvid, SysConfig["keyfield"])
 		if !b || len(*pdata) == 0 {
 			log.Println("oldId不存在")
 			continue
 		}
 		log.Println("pdata:", pdata)
 
-		ndata, b := Newmgo.FindById(util.ObjToString(SysConfig["newc"]), id, SysConfig["keyfield"])
+		ndata, b := Newmgo.FindById(util.ObjToString(SysConfig["newc"]), gvid, SysConfig["keyfield"])
 		if !b || len(*ndata) == 0 {
 			log.Println("nweId不存在")
 			continue
@@ -92,61 +107,106 @@ func Query(num int) {
 		log.Println("ndata:", ndata)
 
 		versioncomparison := new(versionComparison)
-		versioncomparison.Id = id
-		versioncomparison.Url = "https://www.jianyu360.com/article/content/" + util.CommonEncodeArticle("content", data["_id"].(bson.ObjectId).Hex()) + ".html"
+		versioncomparison.Id = gvid
+		versioncomparison.Url = "https://www.jianyu360.com/article/content/" + util.CommonEncodeArticle("content", gvid) + ".html"
 		for k := range SysConfig["keyfield"].(map[string]interface{}) {
-			if util.ObjToString((*pdata)[k]) != util.ObjToString((*ndata)[k]) {
+			var pd interface{}
+			var nd interface{}
+			if k == "budget" || k == "bidamount" {
+				pd = util.Float64All((*pdata)[k])
+				nd = util.Float64All((*ndata)[k])
+			} else {
+				pd = strings.TrimSpace(util.ObjToString((*pdata)[k]))
+				nd = strings.TrimSpace(util.ObjToString((*ndata)[k]))
+			}
+			if pd != nd {
 				//log.Println(k)
 				switch k {
 				case "projectname":
 					projectname := new(Projectname)
 					projectname.versionComparison = *versioncomparison
-					projectname.ProjectnameOld = util.ObjToString((*pdata)[k])
-					projectname.ProjectnameNew = util.ObjToString((*ndata)[k])
+					projectname.ProjectnameOld = fmt.Sprint(pd)
+					projectname.ProjectnameNew = fmt.Sprint(nd)
 					projectnames = append(projectnames, projectname)
 					projectnamenum++
+					if strings.TrimSpace(pd.(string)) != "" {
+						pnameNotNilNumP++
+					}
+					if strings.TrimSpace(nd.(string)) != "" {
+						pnameNotNilNumN++
+					}
 				case "buyer":
 					buyer := new(Buyer)
 					buyer.versionComparison = *versioncomparison
-					buyer.BuyerOld = util.ObjToString((*pdata)[k])
-					buyer.BuyerNew = util.ObjToString((*ndata)[k])
+					buyer.BuyerOld = fmt.Sprint(pd)
+					buyer.BuyerNew = fmt.Sprint(nd)
 					buyers = append(buyers, buyer)
 					buyernum++
+					if strings.TrimSpace(pd.(string)) != "" {
+						buyerNotNilNumP++
+					}
+					if strings.TrimSpace(nd.(string)) != "" {
+						buyerNotNilNumN++
+					}
 				case "projectcode":
 					projectcode := new(Projectcode)
-					projectcode.ProjectcodeOld = util.ObjToString((*pdata)[k])
-					projectcode.ProjectcodeNew = util.ObjToString((*ndata)[k])
+					projectcode.ProjectcodeOld = fmt.Sprint(pd)
+					projectcode.ProjectcodeNew = fmt.Sprint(nd)
 					projectcode.versionComparison = *versioncomparison
 					projectcodes = append(projectcodes, projectcode)
 					projectcodenum++
+					if strings.TrimSpace(pd.(string)) != "" {
+						pcodeNotNilNumP++
+					}
+					if strings.TrimSpace(nd.(string)) != "" {
+						pcodeNotNilNumN++
+					}
 				case "winner":
 					winner := new(Winner)
-					winner.WinnerOld = util.ObjToString((*pdata)[k])
-					winner.WinnerNew = util.ObjToString((*ndata)[k])
+					winner.WinnerOld = fmt.Sprint(pd)
+					winner.WinnerNew = fmt.Sprint(nd)
 					winner.versionComparison = *versioncomparison
 					winners = append(winners, winner)
 					winnernum++
+					if strings.TrimSpace(pd.(string)) != "" {
+						winnerNotNilNumP++
+					}
+					if strings.TrimSpace(nd.(string)) != "" {
+						winnerNotNilNumN++
+					}
 				case "budget":
 					budget := new(Budget)
-					budget.BudgetOld = util.ObjToString((*pdata)[k])
-					budget.BudgetNew = util.ObjToString((*ndata)[k])
+					budget.BudgetOld = fmt.Sprint(pd)
+					budget.BudgetNew = fmt.Sprint(nd)
 					budget.versionComparison = *versioncomparison
 					budgets = append(budgets, budget)
 					budgetnum++
+					if pd.(float64) > 0 {
+						budgetNotNilNumP++
+					}
+					if nd.(float64) > 0 {
+						budgetNotNilNumN++
+					}
 				case "bidamount":
 					bidamount := new(Bidamount)
-					bidamount.BidamountOld = util.ObjToString((*pdata)[k])
-					bidamount.BidamountNew = util.ObjToString((*ndata)[k])
+					bidamount.BidamountOld = fmt.Sprint(pd)
+					bidamount.BidamountNew = fmt.Sprint(nd)
 					bidamount.versionComparison = *versioncomparison
 					bidamounts = append(bidamounts, bidamount)
 					bidamountnum++
+					if pd.(float64) > 0 {
+						bidamountNotNilNumP++
+					}
+					if nd.(float64) > 0 {
+						bidamountNotNilNumN++
+					}
 				}
 			}
 		}
-		num--
 		fmt.Println()
 	}
-	log.Println(projectcodenum, bidamountnum, winnernum, buyernum, budgetnum, projectnamenum)
+
+	//log.Println(projectcodenum, bidamountnum, winnernum, buyernum, budgetnum, projectnamenum)
 	for ins, ivs := range xf.Sheets {
 		for inr, ivr := range ivs.Rows {
 			for _, ivc := range ivr.Cells {
@@ -157,59 +217,71 @@ func Query(num int) {
 					}
 					switch strings.TrimSpace(ivc.String()) {
 					case "projectname":
-						//结果相同数量
-						ivr.Cells[1].SetValue(sum - projectnamenum)
 						style := ivr.Cells[1].GetStyle()
 						style.Font.Color = "000000"
-						ivr.Cells[1].SetStyle(style)
+						ivr.Cells[1].SetValue(pnameNotNilNumP)
+						ivr.Cells[2].SetValue(pnameNotNilNumN)
+						//结果相同数量
+						ivr.Cells[3].SetValue(sum - projectnamenum)
+						ivr.Cells[3].SetStyle(style)
 						//结果不同数量
-						ivr.Cells[2].SetValue(projectnamenum)
-						ivr.Cells[2].SetStyle(style)
+						ivr.Cells[4].SetValue(projectnamenum)
+						ivr.Cells[4].SetStyle(style)
 					case "buyer":
-						//结果相同数量
-						ivr.Cells[1].SetValue(sum - buyernum)
 						style := ivr.Cells[1].GetStyle()
 						style.Font.Color = "000000"
-						ivr.Cells[1].SetStyle(style)
+						ivr.Cells[1].SetValue(buyerNotNilNumP)
+						ivr.Cells[2].SetValue(buyerNotNilNumN)
+						//结果相同数量
+						ivr.Cells[3].SetValue(sum - buyernum)
+						ivr.Cells[3].SetStyle(style)
 						//结果不同数量
-						ivr.Cells[2].SetValue(buyernum)
-						ivr.Cells[2].SetStyle(style)
+						ivr.Cells[4].SetValue(buyernum)
+						ivr.Cells[4].SetStyle(style)
 					case "projectcode":
-						//结果相同数量
-						ivr.Cells[1].SetValue(sum - projectcodenum)
 						style := ivr.Cells[1].GetStyle()
 						style.Font.Color = "000000"
-						ivr.Cells[1].SetStyle(style)
+						ivr.Cells[1].SetValue(pcodeNotNilNumP)
+						ivr.Cells[2].SetValue(pcodeNotNilNumN)
+						//结果相同数量
+						ivr.Cells[3].SetValue(sum - projectcodenum)
+						ivr.Cells[3].SetStyle(style)
 						//结果不同数量
-						ivr.Cells[2].SetValue(projectcodenum)
-						ivr.Cells[2].SetStyle(style)
+						ivr.Cells[4].SetValue(projectcodenum)
+						ivr.Cells[4].SetStyle(style)
 					case "winner":
-						//结果相同数量
-						ivr.Cells[1].SetValue(sum - winnernum)
 						style := ivr.Cells[1].GetStyle()
 						style.Font.Color = "000000"
-						ivr.Cells[1].SetStyle(style)
+						ivr.Cells[1].SetValue(winnerNotNilNumP)
+						ivr.Cells[2].SetValue(winnerNotNilNumN)
+						//结果相同数量
+						ivr.Cells[3].SetValue(sum - winnernum)
+						ivr.Cells[3].SetStyle(style)
 						//结果不同数量
-						ivr.Cells[2].SetValue(winnernum)
-						ivr.Cells[2].SetStyle(style)
+						ivr.Cells[4].SetValue(winnernum)
+						ivr.Cells[4].SetStyle(style)
 					case "budget":
-						//结果相同数量
-						ivr.Cells[1].SetValue(sum - budgetnum)
 						style := ivr.Cells[1].GetStyle()
 						style.Font.Color = "000000"
-						ivr.Cells[1].SetStyle(style)
+						ivr.Cells[1].SetValue(budgetNotNilNumP)
+						ivr.Cells[2].SetValue(budgetNotNilNumN)
+						//结果相同数量
+						ivr.Cells[3].SetValue(sum - budgetnum)
+						ivr.Cells[3].SetStyle(style)
 						//结果不同数量
-						ivr.Cells[2].SetValue(budgetnum)
-						ivr.Cells[2].SetStyle(style)
+						ivr.Cells[4].SetValue(budgetnum)
+						ivr.Cells[4].SetStyle(style)
 					case "bidamount":
-						//结果相同数量
-						ivr.Cells[1].SetValue(sum - bidamountnum)
 						style := ivr.Cells[1].GetStyle()
 						style.Font.Color = "000000"
-						ivr.Cells[1].SetStyle(style)
+						ivr.Cells[1].SetValue(bidamountNotNilNumP)
+						ivr.Cells[2].SetValue(bidamountNotNilNumN)
+						//结果相同数量
+						ivr.Cells[3].SetValue(sum - bidamountnum)
+						ivr.Cells[3].SetStyle(style)
 						//结果不同数量
-						ivr.Cells[2].SetValue(bidamountnum)
-						ivr.Cells[2].SetStyle(style)
+						ivr.Cells[4].SetValue(bidamountnum)
+						ivr.Cells[4].SetStyle(style)
 					}
 				}
 			}
@@ -222,7 +294,7 @@ func Query(num int) {
 				row.AddCell().SetValue("dev3.2")
 				row.AddCell().SetValue("URL")
 			}
-			log.Println(ivs.Name)
+			//log.Println(ivs.Name)
 			switch strings.TrimSpace(ivs.Name) {
 			case "projectname":
 				for _, v := range projectnames {