爬虫周报

src/config.json

@@ -28,6 +28,12 @@
 		"username": "",
 		"password": ""
 	},
+	"es": {
+		"addr": "http://172.17.4.184:19800",
+		"pool": 2,
+		"index": "bidding",
+		"type": "bidding"
+	},
 	"msgservers": {
 		"comm": {
 			"addr": "spdata.jianyu360.com:801",

src/luatask/task.go

@@ -1589,8 +1589,8 @@ func CreateTaskProcess() {
 								if !moreThanLimit { //不在异常范围，不建该类型任务
 									continue
 								}
-							} else if spider.Model == 0 && spider.Working == 1 { //老模式，队列模式(7500,7700)有下载异常数据直接建任务
-								if spider.DownloadFailedNum > 0 { //只有7500、7700出现一条下载异常时，任务状态即为待处理
+							} else if spider.Model == 0 && spider.Working == 1 { //老模式，队列模式(7500,7510、7520、7700)有下载异常数据直接建任务
+								if spider.DownloadFailedNum > 0 { //只有7500,7510、7520、7700出现一条下载异常时，任务状态即为待处理
 									task.State = 1 //待处理
 									taskStateOk = true
 								} else {
@@ -1690,9 +1690,10 @@ func CreateTaskProcess() {
 				logger.Debug("Json UnMarshal Error", code)
 				return
 			}
-			//根据爬虫信息新建任务
-			CreateTask(task, spider, &upsertBulk, lock) //比对历史任务，新建任务
 			if spider.Platform == "golua平台" {
+				//根据爬虫信息新建任务(python任务无人处理，暂停)
+				CreateTask(task, spider, &upsertBulk, lock) //比对历史任务，新建任务
+
 				//列表页总下载量
 				atomic.AddInt64(&LuaListDownloadAllNum, int64(spider.RepeatDownloadAllNum))
 				//列表页总下载成功量
@@ -1735,14 +1736,23 @@ func CreateTaskProcess() {
 func CreateTask(t *Task, sp *Spider, upsertBulk *[][]map[string]interface{}, lock *sync.Mutex) {
 	defer qu.Catch()
 	if t.ErrType == 0 { //不是异常任务
+		if sp.PendState == 1 && sp.DownloadAllNum != 0 { //更新爬虫的挂起状态
+			util.MgoEB.Update("luaconfig",
+				map[string]interface{}{"code": sp.Code},
+				map[string]interface{}{"$set": map[string]interface{}{"pendstate": 0}},
+				false, false)
+			logger.Debug("挂起状态爬虫无异常任务爬虫：", sp.Code)
+		}
 		return
 	}
 	if sp.PendState == 1 {
 		if sp.DownloadAllNum == 0 { //挂起状态爬虫，且下载量为0，不建任务
+			logger.Debug("挂起状态爬虫：", sp.Code)
 			return
 		} else { //挂起状态有下载量，更新爬虫挂起状态
 			sp.PendState = 0 //影响任务i_pendstate状态
 			util.MgoEB.Update("luaconfig", map[string]interface{}{"code": sp.Code}, map[string]interface{}{"$set": map[string]interface{}{"pendstate": 0}}, false, false)
+			logger.Debug("更新挂起状态爬虫：", sp.Code)
 		}
 	}
 	diff := time.Now().Unix() - sp.AuditTime

src/main.go

@@ -23,6 +23,7 @@ func init() {
 	timetask.GetLuaUserInfo()
 	//
 	util.InitMgo()
+	util.InitEs()
 	util.InitOther() //
 	//msgclient
 	// util.InitMsgClient(

src/timetask/report.go

@@ -12,6 +12,9 @@ import (
 	"util"
 )
 
+const Compete_YBW = "元博网（采购与招标网）"
+const Compete_ZGZBYCGW = "中国招标与采购网"
+
 func SpiderWeeklyReport() {
 	defer qu.Catch()
 	eTime := util.GetTime(0)
@@ -19,13 +22,20 @@ func SpiderWeeklyReport() {
 	qu.Debug(sTime, eTime)
 	//上周统计信息
 	LastWeekAddTaskAllNum, //上周新建任务数
-		LastWeekAddLuaNum,                             //lua上周新增爬虫数
-		LastWeekAddEffectTaskNum,                      //上周新增有效任务数
-		LastWeekAddPythonNum,                          //python上周新增爬虫数
-		LastWeekCheckTaskNum,                          //上周核实任务数
-		LastWeekFinishLuaNum,                          //lua上周完成爬虫数
-		LastWeekFinishEffectTaskNum,                   //上周完成有效任务数
-		LastWeekFinishPythonNum := GetLastWeekReport() //python上周完成爬虫数
+		LastWeekAddLuaNum,                          //lua上周新增爬虫数
+		LastWeekAddEffectTaskNum,                   //上周新增有效任务数
+		LastWeekAddPythonNum,                       //python上周新增爬虫数
+		LastWeekCheckTaskNum,                       //上周核实任务数
+		LastWeekFinishLuaNum,                       //lua上周完成爬虫数
+		LastWeekFinishEffectTaskNum,                //上周完成有效任务数
+		LastWeekFinishPythonNum,                    //python上周完成爬虫数
+		LastWeekCodeAllNum,                         //现有爬虫数量
+		LastWeekSiteAllNum,                         //现有站点数量
+		LastWeekMgoNum,                             //上周mgo数量
+		LastWeekEsNum,                              //上周es数量
+		LastWeekCompeteMgoNum,                      //上周竞品采集mgo数量
+		LastWeekCompeteEsNum := GetLastWeekReport() //上周竞品采集es数量
+
 	//1、任务相关
 	//上周新建任务数量
 	qu.Debug("上周新建任务数量：", LastWeekAddTaskAllNum)
@@ -68,9 +78,10 @@ func SpiderWeeklyReport() {
 	qu.Debug("核实任务同比增减：", CheckTaskIncDecRatio)
 	//待核实任务总数
 	query = map[string]interface{}{
-		"i_state": map[string]interface{}{
-			"$lte": 1, //任务状态：待确认、待处理
-		},
+		//"i_state": map[string]interface{}{
+		//	"$lte": 1, //任务状态：待确认、待处理
+		//},
+		"i_state": 1, //待处理
 	}
 	NeedToCheckTaskAllNum := util.MgoE.Count("task", query)
 	qu.Debug("待核实任务总数：", NeedToCheckTaskAllNum)
@@ -206,7 +217,7 @@ func SpiderWeeklyReport() {
 		"platform": "python",
 		"state":    11,
 	}
-	ThisWeekFinishPythonNum := util.MgoE.Count("python_logs_auditor", query)
+	ThisWeekFinishPythonNum := util.MgoE.Count("luaconfig", query)
 	qu.Debug("python本周已完成爬虫数量：", ThisWeekFinishPythonNum)
 	//已完成爬虫数量同比增减
 	FinishPythonIncDecRatio := float64(0)
@@ -233,6 +244,83 @@ func SpiderWeeklyReport() {
 	NeedToFinishPythonAllNum := util.MgoEB.Count("luaconfig", query)
 	qu.Debug("python待完成爬虫总数：", NeedToFinishPythonAllNum)
 
+	//5、爬虫和站点信息
+	ThisWeekCodeAllNum, ThisWeekSiteAllNum := GetCodeAndSiteInfo()
+	qu.Debug("上周统计爬虫总量，站点总量：", LastWeekCodeAllNum, LastWeekSiteAllNum)
+	qu.Debug("本周统计爬虫总量，站点总量：", ThisWeekCodeAllNum, ThisWeekSiteAllNum)
+	//爬虫总量增长率
+	AllCodeIncDecRatio := float64(0)
+	if ThisWeekCodeAllNum != 0 {
+		AllCodeIncDecRatio = float64(ThisWeekCodeAllNum-LastWeekCodeAllNum) / float64(ThisWeekCodeAllNum)
+	}
+	AllCodeIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllCodeIncDecRatio), 64)
+	qu.Debug("爬虫总量增长率：", AllCodeIncDecRatio)
+	//站点总量增长率
+	AllSiteIncDecRatio := float64(0)
+	if ThisWeekSiteAllNum != 0 {
+		AllSiteIncDecRatio = float64(ThisWeekSiteAllNum-LastWeekSiteAllNum) / float64(ThisWeekSiteAllNum)
+	}
+	AllSiteIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllSiteIncDecRatio), 64)
+	qu.Debug("爬虫总量增长率：", AllSiteIncDecRatio)
+
+	//6、数据量统计
+	//mgo本周采集量
+	query = map[string]interface{}{
+		"comeintime": map[string]interface{}{
+			"$gte": sTime,
+			"$lt":  eTime,
+		},
+	}
+	ThisWeekMgoNum := util.MgoB.Count("bidding", query)
+	qu.Debug("上周，本周mgo采集总量：", LastWeekMgoNum, ThisWeekMgoNum)
+	//mgo采集量增长率
+	AllMgoNumIncDecRatio := float64(0)
+	if ThisWeekMgoNum != 0 {
+		AllMgoNumIncDecRatio = float64(ThisWeekMgoNum-LastWeekMgoNum) / float64(ThisWeekMgoNum)
+	}
+	AllMgoNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllMgoNumIncDecRatio), 64)
+	qu.Debug("mgo采集量增长率：", AllMgoNumIncDecRatio)
+
+	//本周es索引量
+	esQuery := `{"query": {"bool": {"must": [{"range": {"comeintime": {"gte": "` + fmt.Sprint(sTime) + `","lt": "` + fmt.Sprint(eTime) + `"}}}]}}}`
+	ThisWeekEsNum := util.Es.Count(util.EsIndex, util.EsType, esQuery)
+	qu.Debug("上周，本周es索引总量：", LastWeekEsNum, ThisWeekEsNum)
+	//es索引量增长率
+	AllEsNumIncDecRatio := float64(0)
+	if ThisWeekEsNum != 0 {
+		AllEsNumIncDecRatio = (float64(ThisWeekEsNum) - float64(LastWeekEsNum)) / float64(ThisWeekEsNum)
+	}
+	AllEsNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllEsNumIncDecRatio), 64)
+	qu.Debug("es索引量增长率：", AllEsNumIncDecRatio)
+
+	//竞品mgo本周采集量
+	ThisWeekCompeteMgoNum := GetThisWeekCompeteMgoNum(sTime, eTime)
+	qu.Debug("上周，本周竞品mgo采集总量：", LastWeekCompeteMgoNum, ThisWeekCompeteMgoNum)
+	//竞品mgo采集量增长率：
+	CompeteMgoNumIncDecRatio := float64(0)
+	if ThisWeekEsNum != 0 {
+		CompeteMgoNumIncDecRatio = float64(ThisWeekCompeteMgoNum-LastWeekCompeteMgoNum) / float64(ThisWeekCompeteMgoNum)
+	}
+	CompeteMgoNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", CompeteMgoNumIncDecRatio), 64)
+	qu.Debug("竞品mgo采集量增长率：", CompeteMgoNumIncDecRatio)
+
+	//竞品es本周索引量(元博网（采购与招标网）、中国招标与采购网)
+	ThisWeekCompeteEsNum := 0
+	for _, competeSite := range []string{Compete_ZGZBYCGW, Compete_YBW} {
+		esQuery = `{"query": {"bool": {"must": [{"range": {"comeintime": {"gte": "` + fmt.Sprint(sTime) + `","lt": "` + fmt.Sprint(eTime) + `"}}},{"term": {"site": "` + competeSite + `"}}]}}}`
+		tmpNum := util.Es.Count(util.EsIndex, util.EsType, esQuery)
+		qu.Debug(competeSite, "es统计量：", tmpNum)
+		ThisWeekCompeteEsNum += int(tmpNum)
+	}
+	qu.Debug("上周，本周竞品es索引总量：", LastWeekCompeteEsNum, ThisWeekCompeteEsNum)
+	//竞品es索引量增长率
+	CompeteEsNumIncDecRatio := float64(0)
+	if ThisWeekEsNum != 0 {
+		CompeteEsNumIncDecRatio = float64(ThisWeekCompeteEsNum-LastWeekCompeteEsNum) / float64(ThisWeekCompeteEsNum)
+	}
+	CompeteEsNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", CompeteEsNumIncDecRatio), 64)
+	qu.Debug("竞品es索引量增长率：", CompeteEsNumIncDecRatio)
+
 	//保存本周记录
 	util.MgoS.Save("spider_weeklyreport", map[string]interface{}{
 		"addtasknum":             ThisWeekAddTaskNum,
@@ -247,6 +335,12 @@ func SpiderWeeklyReport() {
 		"needfinishluaallnum":    NeedToFinishNewLuaAllNum,
 		"needfinishtaskallnum":   NeedToFinishEffectTaskAllNum,
 		"needfinishpythonallnum": NeedToFinishPythonAllNum,
+		"allcodenum":             ThisWeekCodeAllNum,
+		"allsitenum":             ThisWeekSiteAllNum,
+		"mgonum":                 ThisWeekMgoNum,
+		"esnum":                  ThisWeekEsNum,
+		"competemgonum":          ThisWeekCompeteMgoNum,
+		"competeesnum":           ThisWeekCompeteEsNum,
 		"comeintime":             time.Now().Unix(),
 	})
 	//生成excel
@@ -257,37 +351,66 @@ func SpiderWeeklyReport() {
 	}
 	sheet := file.Sheets[0]
 	taskRow := sheet.Rows[1]
-	taskRow.Cells[1].SetValue(LastWeekAddTaskAllNum)
-	taskRow.Cells[2].SetValue(ThisWeekAddTaskNum)
-	taskRow.Cells[3].SetValue(fmt.Sprint(AddTaskIncDecRatio*100) + "%")
-	taskRow.Cells[4].SetValue(LastWeekCheckTaskNum)
-	taskRow.Cells[5].SetValue(ThisWeekCheckTaskNum)
-	taskRow.Cells[6].SetValue(fmt.Sprint(CheckTaskIncDecRatio*100) + "%")
-	taskRow.Cells[7].SetValue(NeedToCheckTaskAllNum)
+	taskRow.Cells[1].SetValue(fmt.Sprint(LastWeekAddTaskAllNum))
+	taskRow.Cells[2].SetValue(fmt.Sprint(ThisWeekAddTaskNum))
+	taskRow.Cells[3].SetValue(fmt.Sprint(ConvertDataType(AddTaskIncDecRatio)) + "%")
+	taskRow.Cells[4].SetValue(fmt.Sprint(LastWeekCheckTaskNum))
+	taskRow.Cells[5].SetValue(fmt.Sprint(ThisWeekCheckTaskNum))
+	taskRow.Cells[6].SetValue(fmt.Sprint(ConvertDataType(CheckTaskIncDecRatio)) + "%")
+	taskRow.Cells[7].SetValue(fmt.Sprint(NeedToCheckTaskAllNum))
 	newLuaRow := sheet.Rows[2]
-	newLuaRow.Cells[1].SetValue(LastWeekAddLuaNum)
-	newLuaRow.Cells[2].SetValue(ThisWeekAddLuaNum)
-	newLuaRow.Cells[3].SetValue(fmt.Sprint(AddLuaIncDecRatio*100) + "%")
-	newLuaRow.Cells[4].SetValue(LastWeekFinishLuaNum)
-	newLuaRow.Cells[5].SetValue(ThisWeekFinishLuaNum)
-	newLuaRow.Cells[6].SetValue(fmt.Sprint(FinishLuaIncDecRatio*100) + "%")
-	newLuaRow.Cells[7].SetValue(NeedToFinishNewLuaAllNum)
+	newLuaRow.Cells[1].SetValue(fmt.Sprint(LastWeekAddLuaNum))
+	newLuaRow.Cells[2].SetValue(fmt.Sprint(ThisWeekAddLuaNum))
+	newLuaRow.Cells[3].SetValue(fmt.Sprint(ConvertDataType(AddLuaIncDecRatio)) + "%")
+	newLuaRow.Cells[4].SetValue(fmt.Sprint(LastWeekFinishLuaNum))
+	newLuaRow.Cells[5].SetValue(fmt.Sprint(ThisWeekFinishLuaNum))
+	newLuaRow.Cells[6].SetValue(fmt.Sprint(ConvertDataType(FinishLuaIncDecRatio)) + "%")
+	newLuaRow.Cells[7].SetValue(fmt.Sprint(NeedToFinishNewLuaAllNum))
 	historyLuaRow := sheet.Rows[3]
-	historyLuaRow.Cells[1].SetValue(LastWeekAddEffectTaskNum)
-	historyLuaRow.Cells[2].SetValue(ThisWeekAddEffectTaskNum)
-	historyLuaRow.Cells[3].SetValue(fmt.Sprint(AddEffectTaskIncDecRatio*100) + "%")
-	historyLuaRow.Cells[4].SetValue(LastWeekFinishEffectTaskNum)
-	historyLuaRow.Cells[5].SetValue(ThisWeekFinishEffectTaskNum)
-	historyLuaRow.Cells[6].SetValue(fmt.Sprint(FinishEffectTaskIncDecRatio*100) + "%")
-	historyLuaRow.Cells[7].SetValue(NeedToFinishEffectTaskAllNum)
+	historyLuaRow.Cells[1].SetValue(fmt.Sprint(LastWeekAddEffectTaskNum))
+	historyLuaRow.Cells[2].SetValue(fmt.Sprint(ThisWeekAddEffectTaskNum))
+	historyLuaRow.Cells[3].SetValue(fmt.Sprint(ConvertDataType(AddEffectTaskIncDecRatio)) + "%")
+	historyLuaRow.Cells[4].SetValue(fmt.Sprint(LastWeekFinishEffectTaskNum))
+	historyLuaRow.Cells[5].SetValue(fmt.Sprint(ThisWeekFinishEffectTaskNum))
+	historyLuaRow.Cells[6].SetValue(fmt.Sprint(ConvertDataType(FinishEffectTaskIncDecRatio)) + "%")
+	historyLuaRow.Cells[7].SetValue(fmt.Sprint(NeedToFinishEffectTaskAllNum))
 	pythonRow := sheet.Rows[4]
-	pythonRow.Cells[1].SetValue(LastWeekAddPythonNum)
-	pythonRow.Cells[2].SetValue(ThisWeekAddPythonNum)
-	pythonRow.Cells[3].SetValue(fmt.Sprint(AddPythonIncDecRatio*100) + "%")
-	pythonRow.Cells[4].SetValue(LastWeekFinishPythonNum)
-	pythonRow.Cells[5].SetValue(ThisWeekFinishPythonNum)
-	pythonRow.Cells[6].SetValue(fmt.Sprint(FinishPythonIncDecRatio*100) + "%")
-	pythonRow.Cells[7].SetValue(NeedToFinishPythonAllNum)
+	pythonRow.Cells[1].SetValue(fmt.Sprint(LastWeekAddPythonNum))
+	pythonRow.Cells[2].SetValue(fmt.Sprint(ThisWeekAddPythonNum))
+	pythonRow.Cells[3].SetValue(fmt.Sprint(ConvertDataType(AddPythonIncDecRatio)) + "%")
+	pythonRow.Cells[4].SetValue(fmt.Sprint(LastWeekFinishPythonNum))
+	pythonRow.Cells[5].SetValue(fmt.Sprint(ThisWeekFinishPythonNum))
+	pythonRow.Cells[6].SetValue(fmt.Sprint(ConvertDataType(FinishPythonIncDecRatio)) + "%")
+	pythonRow.Cells[7].SetValue(fmt.Sprint(NeedToFinishPythonAllNum))
+	thisWeekNumRow := sheet.Rows[6]
+	thisWeekNumRow.Cells[1].SetValue(fmt.Sprint(LastWeekCodeAllNum))
+	thisWeekNumRow.Cells[2].SetValue(fmt.Sprint(LastWeekSiteAllNum))
+	thisWeekNumRow.Cells[3].SetValue(fmt.Sprint(LastWeekMgoNum))
+	thisWeekNumRow.Cells[4].SetValue(fmt.Sprint(LastWeekEsNum))
+	thisWeekNumRow.Cells[5].SetValue(fmt.Sprint(LastWeekCompeteMgoNum))
+	thisWeekNumRow.Cells[6].SetValue(fmt.Sprint(LastWeekCompeteEsNum))
+	lastWeekNumRow := sheet.Rows[7]
+	lastWeekNumRow.Cells[1].SetValue(fmt.Sprint(ThisWeekCodeAllNum))
+	lastWeekNumRow.Cells[2].SetValue(fmt.Sprint(ThisWeekSiteAllNum))
+	lastWeekNumRow.Cells[3].SetValue(fmt.Sprint(ThisWeekMgoNum))
+	lastWeekNumRow.Cells[4].SetValue(fmt.Sprint(ThisWeekEsNum))
+	lastWeekNumRow.Cells[5].SetValue(fmt.Sprint(ThisWeekCompeteMgoNum))
+	lastWeekNumRow.Cells[6].SetValue(fmt.Sprint(ThisWeekCompeteEsNum))
+	incDecRatioRow := sheet.Rows[8]
+	incDecRatioRow.Cells[1].SetValue(fmt.Sprint(ConvertDataType(AllCodeIncDecRatio)) + "%")
+	incDecRatioRow.Cells[2].SetValue(fmt.Sprint(ConvertDataType(AllSiteIncDecRatio)) + "%")
+	incDecRatioRow.Cells[3].SetValue(fmt.Sprint(ConvertDataType(AllMgoNumIncDecRatio)) + "%")
+	incDecRatioRow.Cells[4].SetValue(fmt.Sprint(ConvertDataType(AllEsNumIncDecRatio)) + "%")
+	incDecRatioRow.Cells[5].SetValue(fmt.Sprint(ConvertDataType(CompeteMgoNumIncDecRatio)) + "%")
+	incDecRatioRow.Cells[6].SetValue(fmt.Sprint(ConvertDataType(CompeteEsNumIncDecRatio)) + "%")
+	changeNumrow := sheet.Rows[9]
+	changeNumrow.Cells[1].SetValue(fmt.Sprint(ThisWeekCodeAllNum - LastWeekCodeAllNum))
+	changeNumrow.Cells[2].SetValue(fmt.Sprint(ThisWeekSiteAllNum - LastWeekSiteAllNum))
+	changeNumrow.Cells[3].SetValue(fmt.Sprint(ThisWeekMgoNum - LastWeekMgoNum))
+	changeNumrow.Cells[4].SetValue(fmt.Sprint(ThisWeekEsNum - int64(LastWeekEsNum)))
+	changeNumrow.Cells[5].SetValue(fmt.Sprint(ThisWeekCompeteMgoNum - LastWeekCompeteMgoNum))
+	changeNumrow.Cells[6].SetValue(fmt.Sprint(ThisWeekCompeteEsNum - LastWeekCompeteEsNum))
+
 	mw := &util.MyWrite{
 		Byte: &bytes.Buffer{},
 	}
@@ -298,16 +421,120 @@ func SpiderWeeklyReport() {
 	gm.GSendMail_Bq("jy@jianyu360.cn", "zhangjinkun@topnet.net.cn,maxiaoshan@topnet.net.cn", "", "", "爬虫统计周报", "", name, bt, util.GMail)
 }
 
-func GetLastWeekReport() (int, int, int, int, int, int, int, int) {
+func GetCodeAndSiteInfo() (int, int) {
+	defer qu.Catch()
+	codeMap := map[string]bool{} //爬虫集合
+	siteMap := map[string]bool{} //站点集合
+	//1、统计爬虫和站点
+	sess := util.MgoEB.GetMgoConn()
+	defer util.MgoEB.DestoryMongoConn(sess)
+	ch := make(chan bool, 2)
+	wg := &sync.WaitGroup{}
+	lock := &sync.Mutex{}
+	query := map[string]interface{}{
+		"state": map[string]interface{}{
+			"$nin": []int{4, 10}, //查询非作废和非删除状态的爬虫
+		},
+	}
+	field := map[string]interface{}{
+		"code": 1,
+		"site": 1,
+	}
+	it := sess.DB(util.MgoEB.DbName).C("luaconfig").Find(&query).Select(&field).Iter()
+	n := 0
+	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			code := qu.ObjToString(tmp["code"])
+			site := qu.ObjToString(tmp["site"])
+			lock.Lock()
+			codeMap[code] = true
+			siteMap[site] = true
+			lock.Unlock()
+		}(tmp)
+		if n%1000 == 0 {
+			qu.Debug("current:", n)
+		}
+		tmp = map[string]interface{}{}
+	}
+	wg.Wait()
+	codeNum := len(codeMap)
+	siteNum := len(siteMap)
+	codeMap = map[string]bool{} //爬虫集合
+	siteMap = map[string]bool{} //站点集合
+	return codeNum, siteNum
+}
+
+func GetThisWeekCompeteMgoNum(sTime, eTime int64) (ThisWeekCompeteMgoNum int) {
+	defer qu.Catch()
+	sess := util.MgoB.GetMgoConn()
+	defer util.MgoB.DestoryMongoConn(sess)
+	ch := make(chan bool, 2)
+	wg := &sync.WaitGroup{}
+	lock := &sync.Mutex{}
+	query := map[string]interface{}{
+		"comeintime": map[string]interface{}{
+			"$gte": sTime,
+			"$lt":  eTime,
+		},
+	}
+	field := map[string]interface{}{
+		"site": 1,
+	}
+	it := sess.DB(util.MgoB.DbName).C("bidding").Find(&query).Select(&field).Iter()
+	n := 0
+	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			site := qu.ObjToString(tmp["site"])
+			lock.Lock()
+			if site == Compete_YBW || site == Compete_ZGZBYCGW {
+				ThisWeekCompeteMgoNum++
+			}
+			lock.Unlock()
+		}(tmp)
+		if n%1000 == 0 {
+			qu.Debug("current:", n)
+		}
+		tmp = map[string]interface{}{}
+	}
+	wg.Wait()
+	return
+}
+
+func GetLastWeekReport() (int, int, int, int, int, int, int, int, int, int, int, int, int, int) {
 	defer qu.Catch()
 	datas, _ := util.MgoS.Find("spider_weeklyreport", nil, map[string]interface{}{"_id": -1}, nil, true, 0, 1)
 	if len(*datas) == 1 {
 		data := (*datas)[0]
-		return qu.IntAll(data["addtasknum"]), qu.IntAll(data["addluanum"]), qu.IntAll(data["addeffecttasknum"]), qu.IntAll(data["addpythonnum"]), qu.IntAll(data["checktasknum"]), qu.IntAll(data["finishluanum"]), qu.IntAll(data["finisheffecttasknum"]), qu.IntAll(data["finishpythonnum"])
+		return qu.IntAll(data["addtasknum"]),
+			qu.IntAll(data["addluanum"]),
+			qu.IntAll(data["addeffecttasknum"]),
+			qu.IntAll(data["addpythonnum"]),
+			qu.IntAll(data["checktasknum"]),
+			qu.IntAll(data["finishluanum"]),
+			qu.IntAll(data["finisheffecttasknum"]),
+			qu.IntAll(data["finishpythonnum"]),
+			qu.IntAll(data["allcodenum"]),
+			qu.IntAll(data["allsitenum"]),
+			qu.IntAll(data["mgonum"]),
+			qu.IntAll(data["esnum"]),
+			qu.IntAll(data["competemgonum"]),
+			qu.IntAll(data["competeesnum"])
 	} else {
 		qu.Debug("历史周报信息查询失败")
 	}
-	return -1, -1, -1, -1, -1, -1, -1, -1
+	return -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
 }
 
 func CountNeedToFinishNewLuaAllNum(query map[string]interface{}) (result int) {
@@ -346,3 +573,9 @@ func CountNeedToFinishNewLuaAllNum(query map[string]interface{}) (result int) {
 	wg.Wait()
 	return
 }
+
+//解决float64*100后精度丢失问题
+func ConvertDataType(ratio float64) float64 {
+	ratio, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", ratio*100), 64)
+	return ratio
+}

src/util/config.go

@@ -3,6 +3,7 @@ package util
 import (
 	mgo "mongodb"
 	qu "qfw/util"
+	es "qfw/util/elastic"
 	gm "qfw/util/mail"
 	"regexp"
 )
@@ -14,6 +15,9 @@ var (
 	MgoS                       *mgo.MongodbSim //spider
 	MgoPy                      *mgo.MongodbSim //py_spider
 	MgoB                       *mgo.MongodbSim //bidding
+	Es                         *es.Elastic     //
+	EsIndex                    string          //
+	EsType                     string          //
 	CommServers                map[string]interface{}
 	BidServers                 map[string]interface{}
 	UploadEvents               map[int]string
@@ -78,6 +82,17 @@ func InitMgo() {
 	MgoEB.InitPool()
 }
 
+func InitEs() {
+	esMap := Config["es"].(map[string]interface{})
+	EsIndex = qu.ObjToString(esMap["index"])
+	EsType = qu.ObjToString(esMap["type"])
+	Es = &es.Elastic{
+		S_esurl: qu.ObjToString(esMap["addr"]),
+		I_size:  qu.IntAll(esMap["pool"]),
+	}
+	Es.InitElasticSize()
+}
+
 func InitOther() {
 	//cron
 	StartTaskCron = qu.ObjToString(Config["startaskcron"])