Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

versioncomparison/config.json

@@ -1,10 +1,10 @@
 {
     "premgo": "192.168.3.207:27092",
     "predb": "extract_kf",
-    "prec": "datainfo_dev3.2",
+    "prec": "demo_data3.2",
     "newmgo": "192.168.3.207:27092",
     "newdb": "extract_kf",
-    "newc": "datainfo_dev3.4",
+    "newc": "demo_data3.4",
     "fields": [
         "projectname",
         "projectcode",

versioncomparison/demo_data.go

@@ -0,0 +1,128 @@
+// demodata
+package main
+
+import (
+	"jy/mongodbutil"
+	"log"
+	"qfw/util"
+)
+
+//30个网站，10个信息分类,共计5000条样例数据
+var sitenums = map[string]int{
+	"中国政府采购网":              1006,
+	"中国招标投标公共服务平台":         674,
+	"国家公共资源交易平台":           371,
+	"广东省政府采购网":             282,
+	"机电产品招标投标电子交易平台":       189,
+	"中国山东政府采购网":            183,
+	"中国华能集团公司":             177,
+	"湖北省政府采购网":             173,
+	"基建云采购":                166,
+	"浙江政府采购网":              141,
+	"中国大唐集团公司电子商务平台":       140,
+	"中国华电集团公司电子商务平台":       129,
+	"中国电力设备信息网":            122,
+	"安徽省政府采购网":             116,
+	"中国电信阳光采购网":            100,
+	"河北省政府采购网":             98,
+	"四川政府采购":               92,
+	"安徽省招标投标信息网":           85,
+	"中国山西政府采购网":            74,
+	"浙江省公共资源交易服务平台":        73,
+	"新疆维吾尔自治区政府采购网":        73,
+	"阿里巴巴大企业采购平台":          70,
+	"中国冶金科工集团有限公司采购电子商务平台": 64,
+	"广东省电子化采购执行平台":         62,
+	"河南省政府采购网":             60,
+	"国家能源e购":               57,
+	"铁路物资采购与招商平台":          57,
+	"兵器工业集团公司采购电子商务平台":     57,
+	"安徽合肥公共资源交易中心":         55,
+	"河北省公共资源交易信息网":         55,
+}
+
+var bidtypesP = map[string]float64{
+	"招标": 0.25,
+	"邀标": 0.03,
+	"询价": 0.10,
+	"竞谈": 0.06,
+	"单一": 0.03,
+	"竞价": 0.03,
+	"合同": 0.09,
+	"验收": 0.03,
+	"中标": 0.23,
+	"成交": 0.15,
+}
+
+var bidtypes = map[string]bool{
+	"招标": true, "邀标": true, "询价": true, "竞价": true, "竞谈": true,
+	"单一": true, "中标": true, "成交": true, "合同": true, "验收": true,
+}
+var sitebidtypesNum = map[string]int{}
+
+func mainT() {
+	ext := mongodbutil.MgoFactory(1, 3, 120, "192.168.3.207:27092", "extract_kf")
+	//	ls, _ := ext.Find("demo_data", nil, nil, nil, false, -1, -1)
+	//	bidnum := map[string]int{}
+	//	for _, v := range *ls {
+	//		subtype, _ := v["subtype"].(string)
+	//		bidnum[subtype] += 1
+	//	}
+	//	for k, v := range bidnum {
+	//		log.Println(k, v)
+	//	}
+	//	os.Exit(0)
+	var total = 0
+	for site, num := range sitenums {
+		tt := 0
+		for stype, _ := range bidtypes {
+			n := bidtypesP[stype] * float64(num)
+			sitebidtypesNum[site+"-"+stype] = int(n)
+			total += int(n)
+			tt += int(n)
+			log.Println(site+"-"+stype, int(n))
+		}
+		log.Println(site, tt)
+	}
+	log.Println(total)
+
+	db := mongodbutil.MgoFactory(1, 3, 120, "192.168.3.207:27083", "qfw")
+	it := db.Get().DB("qfw").C("bidding").Find(map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt": util.StringTOBsonId("5df507000000000000000000"),
+			"$lt": util.StringTOBsonId("5e0a1f000000000000000000"),
+		},
+	}).Iter()
+	index := 0
+	datanum := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if index%1000 == 0 {
+			log.Println(index, datanum)
+		}
+		site, _ := tmp["site"].(string)
+		subtype, _ := tmp["subtype"].(string)
+		if !bidtypes[subtype] {
+			continue
+		}
+		tp := site + "-" + subtype
+		if sitebidtypesNum[tp] > 0 {
+			sitebidtypesNum[tp] -= 1
+		} else {
+			continue
+		}
+		datanum++
+		ext.Update("demo_data", map[string]interface{}{"_id": tmp["_id"]}, tmp, true, false)
+		tmp = map[string]interface{}{}
+	}
+
+	log.Println("datanum", datanum)
+	ls, _ := ext.Find("demo_data", nil, nil, nil, false, -1, -1)
+	snum := map[string]int{}
+	for _, v := range *ls {
+		site, _ := v["site"].(string)
+		snum[site] += 1
+	}
+	for k, v := range snum {
+		log.Println(k, v)
+	}
+}

versioncomparison/main.go

@@ -40,8 +40,8 @@ type Data struct {
 }
 
 func init() {
-	flag.StringVar(&Sid, "sid", "5e17deb150b5ea296ec939d3", "开始id")
-	flag.StringVar(&Eid, "eid", "5e17e1e685a9271abf08616d", "结束id")
+	flag.StringVar(&Sid, "sid", "5df5071ce9d1f601e495fa54", "开始id")
+	flag.StringVar(&Eid, "eid", "5e09c05f0cf41612e0626abc", "结束id")
 	flag.Parse()
 	qu.ReadConfig(&SysConfig)
 	Premgo = mongodbutil.MgoFactory(1, 3, 120, qu.ObjToString(SysConfig["premgo"]), qu.ObjToString(SysConfig["predb"]))
@@ -97,8 +97,8 @@ func createXlsx() {
 			sh.Rows[k] = row
 		}
 	}
+	var idsall = map[string]bool{}
 	//生成信息sheet
-
 	for _, field := range Fields {
 		sh, _ := xf.AddSheet(field)
 		rowh := sh.AddRow()
@@ -114,9 +114,44 @@ func createXlsx() {
 				row.AddCell().SetString(v.PreVal)
 				row.AddCell().SetString(v.NewVal)
 				row.AddCell().SetString(fmt.Sprintf(Url, qu.CommonEncodeArticle("content", v.Id)))
+				idsall[v.Id] = true
+			}
+		}
+	}
+	log.Println("不同数据总量", len(idsall))
+	//生全量信息不同部分
+	shall, _ := xf.AddSheet("全量数据（不同部分）")
+	rowh := shall.AddRow()
+	rowh.AddCell().SetString("id")
+	for _, v := range Fields {
+		rowh.AddCell().SetString("preval_" + v)
+		rowh.AddCell().SetString("newval_" + v)
+	}
+	rowh.AddCell().SetString("url")
+	i := 0
+	for k, _ := range idsall {
+		i++
+		row := shall.AddRow()
+		row.AddCell().SetString(k)
+		for _, field := range Fields {
+			tmp := FieldData[field]
+			v := tmp[k]
+			if v != nil {
+				if v.NewVal != v.PreVal {
+					row.AddCell().SetString(v.PreVal)
+					row.AddCell().SetString(v.NewVal)
+				} else {
+					row.AddCell().SetString("")
+					row.AddCell().SetString("")
+				}
+			} else {
+				row.AddCell().SetString("")
+				row.AddCell().SetString("")
 			}
 		}
+		row.AddCell().SetString(fmt.Sprintf(Url, qu.CommonEncodeArticle("content", k)))
 	}
+	log.Println("数据处理完成，正在生成文件")
 	err = xf.Save("result.xlsx")
 	if err != nil {
 		log.Println("保存xlsx失败:", err)