spider_task/src/timetask/report.go

package timetask

import (
	"bytes"
	"fmt"
	"github.com/tealeg/xlsx"
	qu "qfw/util"
	gm "qfw/util/mail"
	"strconv"
	"sync"
	"time"
	"util"
)

const Compete_YBW = "元博网（采购与招标网）"
const Compete_ZGZBYCGW = "中国招标与采购网"

func SpiderWeeklyReport() {
	defer qu.Catch()
	eTime := util.GetTime(0)
	sTime := util.GetTime(-7)
	qu.Debug(sTime, eTime)
	//上周统计信息
	LastWeekAddTaskAllNum, //上周新建任务数
		LastWeekAddLuaNum,                          //lua上周新增爬虫数
		LastWeekAddEffectTaskNum,                   //上周新增有效任务数
		LastWeekAddPythonNum,                       //python上周新增爬虫数
		LastWeekCheckTaskNum,                       //上周核实任务数
		LastWeekFinishLuaNum,                       //lua上周完成爬虫数
		LastWeekFinishEffectTaskNum,                //上周完成有效任务数
		LastWeekFinishPythonNum,                    //python上周完成爬虫数
		LastWeekCodeAllNum,                         //现有爬虫数量
		LastWeekSiteAllNum,                         //现有站点数量
		LastWeekMgoNum,                             //上周mgo数量
		LastWeekEsNum,                              //上周es数量
		LastWeekCompeteMgoNum,                      //上周竞品采集mgo数量
		LastWeekCompeteEsNum := GetLastWeekReport() //上周竞品采集es数量

	//1、任务相关
	//上周新建任务数量
	qu.Debug("上周新建任务数量：", LastWeekAddTaskAllNum)
	//本周新建任务数量
	query := map[string]interface{}{
		"l_comeintime": map[string]interface{}{
			"$gte": sTime,
			"$lt":  eTime,
		},
	}
	ThisWeekAddTaskNum := util.MgoE.Count("task", query)
	qu.Debug("本周新建任务数量：", ThisWeekAddTaskNum)
	//新建任务同比增减
	AddTaskIncDecRatio := float64(0)
	if ThisWeekAddTaskNum != 0 {
		AddTaskIncDecRatio = float64(ThisWeekAddTaskNum-LastWeekAddTaskAllNum) / float64(ThisWeekAddTaskNum)
	}
	AddTaskIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AddTaskIncDecRatio), 64)
	qu.Debug("新建任务同比增减：", AddTaskIncDecRatio)
	//上周核实任务数量
	qu.Debug("上周核实任务数量：", LastWeekCheckTaskNum)
	//本周核实任务数量
	query = map[string]interface{}{
		"l_checktime": map[string]interface{}{
			"$gte": sTime,
			"$lt":  eTime,
		},
		"i_state": map[string]interface{}{
			"$gte": 2, //任务状态：处理中、待审核、审核通过、未通过、关闭
		},
	}
	ThisWeekCheckTaskNum := util.MgoE.Count("task", query)
	qu.Debug("本周核实任务数量：", ThisWeekCheckTaskNum)
	//核实任务同比增减
	CheckTaskIncDecRatio := float64(0)
	if ThisWeekCheckTaskNum != 0 {
		CheckTaskIncDecRatio = float64(ThisWeekCheckTaskNum-LastWeekCheckTaskNum) / float64(ThisWeekCheckTaskNum)
	}
	CheckTaskIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", CheckTaskIncDecRatio), 64)
	qu.Debug("核实任务同比增减：", CheckTaskIncDecRatio)
	//待核实任务总数
	query = map[string]interface{}{
		//"i_state": map[string]interface{}{
		//	"$lte": 1, //任务状态：待确认、待处理
		//},
		"i_state": 1, //待处理
	}
	NeedToCheckTaskAllNum := util.MgoE.Count("task", query)
	qu.Debug("待核实任务总数：", NeedToCheckTaskAllNum)

	//2、lua新增爬虫
	//上周新建爬虫数量
	qu.Debug("lua上周新建爬虫数量：", LastWeekAddLuaNum)
	//本周新建爬虫数量
	query = map[string]interface{}{
		"comeintime": map[string]interface{}{
			"$gte": sTime,
			"$lt":  eTime,
		},
		"platform": "golua平台",
	}
	ThisWeekAddLuaNum := util.MgoEB.Count("luaconfig", query)
	qu.Debug("lua本周新建爬虫数量：", ThisWeekAddLuaNum)
	//lua新建爬虫同比增减
	AddLuaIncDecRatio := float64(0)
	if ThisWeekAddLuaNum != 0 {
		AddLuaIncDecRatio = float64(ThisWeekAddLuaNum-LastWeekAddLuaNum) / float64(ThisWeekAddLuaNum)
	}
	AddLuaIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AddLuaIncDecRatio), 64)
	qu.Debug("lua新建爬虫同比增减：", AddLuaIncDecRatio)
	//上周已完成新建爬虫数量
	qu.Debug("lua上周已完成新建爬虫数量：", LastWeekFinishLuaNum)
	//本周已完成新建爬虫数量
	query = map[string]interface{}{
		"comeintime": map[string]interface{}{
			"$gte": sTime,
			"$lt":  eTime,
		},
	}
	ThisWeekFinishLuaNum := util.MgoE.Count("lua_logs_auditor_new", query)
	qu.Debug("lua本周已完成新建爬虫数量：", ThisWeekFinishLuaNum)
	//lua完成新建爬虫同比增减
	FinishLuaIncDecRatio := float64(0)
	if ThisWeekFinishLuaNum != 0 {
		FinishLuaIncDecRatio = float64(ThisWeekFinishLuaNum-LastWeekFinishLuaNum) / float64(ThisWeekFinishLuaNum)
	}
	FinishLuaIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", FinishLuaIncDecRatio), 64)
	qu.Debug("lua完成新建爬虫同比增减：", FinishLuaIncDecRatio)
	//待完成新建爬虫总数
	query = map[string]interface{}{
		"event":    7000,
		"platform": "golua平台",
		"state": map[string]interface{}{
			"$lte": 2, //待完成、待审核、未通过
		},
	}
	NeedToFinishNewLuaAllNum := CountNeedToFinishNewLuaAllNum(query)
	qu.Debug("lua待完成新建爬虫总数：", NeedToFinishNewLuaAllNum)

	//3、lua历史维护
	//上周新增待维护任务数量
	qu.Debug("上周新增待维护任务数量：", LastWeekAddEffectTaskNum)
	//本周新增待维护任务数量
	query = map[string]interface{}{
		"l_checktime": map[string]interface{}{
			"$gte": sTime,
			"$lt":  eTime,
		},
		"i_state": map[string]interface{}{ //处理中、待审核、审核通过、未通过
			"$gte": 2,
			"$lte": 5,
		},
	}
	ThisWeekAddEffectTaskNum := util.MgoE.Count("task", query)
	qu.Debug("本周新增待维护任务数量：", ThisWeekAddEffectTaskNum)
	//新建待维护任务同比增减
	AddEffectTaskIncDecRatio := float64(0)
	if ThisWeekAddEffectTaskNum != 0 {
		AddEffectTaskIncDecRatio = float64(ThisWeekAddEffectTaskNum-LastWeekAddEffectTaskNum) / float64(ThisWeekAddEffectTaskNum)
	}
	AddEffectTaskIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AddEffectTaskIncDecRatio), 64)
	qu.Debug("新建待维护任务同比增减：", AddEffectTaskIncDecRatio)
	//上周完成待维护任务数量
	qu.Debug("上周完成待维护任务数量：", LastWeekFinishEffectTaskNum)
	//本周完成待维护任务数量
	query = map[string]interface{}{
		"l_uploadtime": map[string]interface{}{
			"$gte": sTime,
			"$lt":  eTime,
		},
		"i_state": 4, //审核通过
	}
	ThisWeekFinishEffectTaskNum := util.MgoE.Count("task", query)
	qu.Debug("本周完成待维护任务数量：", ThisWeekFinishEffectTaskNum)
	//完成待维护任务同比增减
	FinishEffectTaskIncDecRatio := float64(0)
	if ThisWeekFinishEffectTaskNum != 0 {
		FinishEffectTaskIncDecRatio = float64(ThisWeekFinishEffectTaskNum-LastWeekFinishEffectTaskNum) / float64(ThisWeekFinishEffectTaskNum)
	}
	FinishEffectTaskIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", FinishEffectTaskIncDecRatio), 64)
	qu.Debug("完成待维护任务同比增减：", FinishEffectTaskIncDecRatio)
	//待维护任务总数
	query = map[string]interface{}{
		"i_state": map[string]interface{}{
			"$in": []int{2, 3, 5}, //处理中、待审核、未通过
		},
	}
	NeedToFinishEffectTaskAllNum := util.MgoE.Count("task", query)
	qu.Debug("待维护任务总数：", NeedToFinishEffectTaskAllNum)

	//4、python爬虫
	//上周新建爬虫数量
	qu.Debug("python上周新建爬虫数量：", LastWeekAddPythonNum)
	//本周新建爬虫数量
	query = map[string]interface{}{
		"comeintime": map[string]interface{}{
			"$gte": sTime,
			"$lt":  eTime,
		},
		"platform": "python",
	}
	ThisWeekAddPythonNum := util.MgoEB.Count("luaconfig", query)
	qu.Debug("python本周新建爬虫数量：", ThisWeekAddPythonNum)
	//新建爬虫数量同比增减
	AddPythonIncDecRatio := float64(0)
	if ThisWeekAddPythonNum != 0 {
		AddPythonIncDecRatio = float64(ThisWeekAddPythonNum-LastWeekAddPythonNum) / float64(ThisWeekAddPythonNum)
	}
	AddPythonIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AddPythonIncDecRatio), 64)
	qu.Debug("python新建爬虫同比增减：", AddPythonIncDecRatio)
	//上周已完成爬虫数量
	qu.Debug("python上周已完成爬虫数量：", LastWeekFinishPythonNum)
	//本周已完成爬虫数量
	query = map[string]interface{}{
		"l_uploadtime": map[string]interface{}{
			"$gte": sTime,
			"$lt":  eTime,
		},
		"platform": "python",
		"state":    11,
	}
	ThisWeekFinishPythonNum := util.MgoE.Count("luaconfig", query)
	qu.Debug("python本周已完成爬虫数量：", ThisWeekFinishPythonNum)
	//已完成爬虫数量同比增减
	FinishPythonIncDecRatio := float64(0)
	if ThisWeekFinishPythonNum != 0 {
		FinishPythonIncDecRatio = float64(ThisWeekFinishPythonNum-LastWeekFinishPythonNum) / float64(ThisWeekFinishPythonNum)
	}
	FinishPythonIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", FinishPythonIncDecRatio), 64)
	qu.Debug("python已完成爬虫数量同比增减：", FinishPythonIncDecRatio)
	//python待完成爬虫总数
	query = map[string]interface{}{
		"$or": []interface{}{
			map[string]interface{}{
				"platform": "python",
				"state": map[string]interface{}{
					"$nin": []int{4, 5, 7, 10, 11}, //除已作废、已上架、无发布、已删除、已上线状态的爬虫都是待开发的爬虫
				},
			},
			map[string]interface{}{
				"platform": "golua平台",
				"state":    9, //lua平台无法处理（转python）的爬虫算到python待完成的爬虫里
			},
		},
	}
	NeedToFinishPythonAllNum := util.MgoEB.Count("luaconfig", query)
	qu.Debug("python待完成爬虫总数：", NeedToFinishPythonAllNum)

	//5、爬虫和站点信息
	ThisWeekCodeAllNum, ThisWeekSiteAllNum := GetCodeAndSiteInfo()
	qu.Debug("上周统计爬虫总量，站点总量：", LastWeekCodeAllNum, LastWeekSiteAllNum)
	qu.Debug("本周统计爬虫总量，站点总量：", ThisWeekCodeAllNum, ThisWeekSiteAllNum)
	//爬虫总量增长率
	AllCodeIncDecRatio := float64(0)
	if ThisWeekCodeAllNum != 0 {
		AllCodeIncDecRatio = float64(ThisWeekCodeAllNum-LastWeekCodeAllNum) / float64(ThisWeekCodeAllNum)
	}
	AllCodeIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllCodeIncDecRatio), 64)
	qu.Debug("爬虫总量增长率：", AllCodeIncDecRatio)
	//站点总量增长率
	AllSiteIncDecRatio := float64(0)
	if ThisWeekSiteAllNum != 0 {
		AllSiteIncDecRatio = float64(ThisWeekSiteAllNum-LastWeekSiteAllNum) / float64(ThisWeekSiteAllNum)
	}
	AllSiteIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllSiteIncDecRatio), 64)
	qu.Debug("爬虫总量增长率：", AllSiteIncDecRatio)

	//6、数据量统计
	//mgo本周采集量
	query = map[string]interface{}{
		"comeintime": map[string]interface{}{
			"$gte": sTime,
			"$lt":  eTime,
		},
	}
	ThisWeekMgoNum := util.MgoB.Count("bidding", query)
	qu.Debug("上周，本周mgo采集总量：", LastWeekMgoNum, ThisWeekMgoNum)
	//mgo采集量增长率
	AllMgoNumIncDecRatio := float64(0)
	if ThisWeekMgoNum != 0 {
		AllMgoNumIncDecRatio = float64(ThisWeekMgoNum-LastWeekMgoNum) / float64(ThisWeekMgoNum)
	}
	AllMgoNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllMgoNumIncDecRatio), 64)
	qu.Debug("mgo采集量增长率：", AllMgoNumIncDecRatio)

	//本周es索引量
	esQuery := `{"query": {"bool": {"must": [{"range": {"comeintime": {"gte": "` + fmt.Sprint(sTime) + `","lt": "` + fmt.Sprint(eTime) + `"}}}]}}}`
	ThisWeekEsNum := util.Es.Count(util.EsIndex, util.EsType, esQuery)
	qu.Debug("上周，本周es索引总量：", LastWeekEsNum, ThisWeekEsNum)
	//es索引量增长率
	AllEsNumIncDecRatio := float64(0)
	if ThisWeekEsNum != 0 {
		AllEsNumIncDecRatio = (float64(ThisWeekEsNum) - float64(LastWeekEsNum)) / float64(ThisWeekEsNum)
	}
	AllEsNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllEsNumIncDecRatio), 64)
	qu.Debug("es索引量增长率：", AllEsNumIncDecRatio)

	//竞品mgo本周采集量
	ThisWeekCompeteMgoNum := GetThisWeekCompeteMgoNum(sTime, eTime)
	qu.Debug("上周，本周竞品mgo采集总量：", LastWeekCompeteMgoNum, ThisWeekCompeteMgoNum)
	//竞品mgo采集量增长率：
	CompeteMgoNumIncDecRatio := float64(0)
	if ThisWeekEsNum != 0 {
		CompeteMgoNumIncDecRatio = float64(ThisWeekCompeteMgoNum-LastWeekCompeteMgoNum) / float64(ThisWeekCompeteMgoNum)
	}
	CompeteMgoNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", CompeteMgoNumIncDecRatio), 64)
	qu.Debug("竞品mgo采集量增长率：", CompeteMgoNumIncDecRatio)

	//竞品es本周索引量(元博网（采购与招标网）、中国招标与采购网)
	ThisWeekCompeteEsNum := 0
	for _, competeSite := range []string{Compete_ZGZBYCGW, Compete_YBW} {
		esQuery = `{"query": {"bool": {"must": [{"range": {"comeintime": {"gte": "` + fmt.Sprint(sTime) + `","lt": "` + fmt.Sprint(eTime) + `"}}},{"term": {"site": "` + competeSite + `"}}]}}}`
		tmpNum := util.Es.Count(util.EsIndex, util.EsType, esQuery)
		qu.Debug(competeSite, "es统计量：", tmpNum)
		ThisWeekCompeteEsNum += int(tmpNum)
	}
	qu.Debug("上周，本周竞品es索引总量：", LastWeekCompeteEsNum, ThisWeekCompeteEsNum)
	//竞品es索引量增长率
	CompeteEsNumIncDecRatio := float64(0)
	if ThisWeekEsNum != 0 {
		CompeteEsNumIncDecRatio = float64(ThisWeekCompeteEsNum-LastWeekCompeteEsNum) / float64(ThisWeekCompeteEsNum)
	}
	CompeteEsNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", CompeteEsNumIncDecRatio), 64)
	qu.Debug("竞品es索引量增长率：", CompeteEsNumIncDecRatio)

	//保存本周记录
	util.MgoS.Save("spider_weeklyreport", map[string]interface{}{
		"addtasknum":             ThisWeekAddTaskNum,
		"addluanum":              ThisWeekAddLuaNum,
		"addeffecttasknum":       ThisWeekAddEffectTaskNum,
		"addpythonnum":           ThisWeekAddPythonNum,
		"checktasknum":           ThisWeekCheckTaskNum,
		"finishluanum":           ThisWeekFinishLuaNum,
		"finisheffecttasknum":    ThisWeekFinishEffectTaskNum,
		"finishpythonnum":        ThisWeekFinishPythonNum,
		"needchecktaskallnum":    NeedToCheckTaskAllNum,
		"needfinishluaallnum":    NeedToFinishNewLuaAllNum,
		"needfinishtaskallnum":   NeedToFinishEffectTaskAllNum,
		"needfinishpythonallnum": NeedToFinishPythonAllNum,
		"allcodenum":             ThisWeekCodeAllNum,
		"allsitenum":             ThisWeekSiteAllNum,
		"mgonum":                 ThisWeekMgoNum,
		"esnum":                  ThisWeekEsNum,
		"competemgonum":          ThisWeekCompeteMgoNum,
		"competeesnum":           ThisWeekCompeteEsNum,
		"comeintime":             time.Now().Unix(),
	})
	//生成excel
	file, err := xlsx.OpenFile("res/report.xlsx")
	if err != nil {
		qu.Debug("Open Report File Error:", err)
		return
	}
	sheet := file.Sheets[0]
	taskRow := sheet.Rows[1]
	taskRow.Cells[1].SetValue(fmt.Sprint(LastWeekAddTaskAllNum))
	taskRow.Cells[2].SetValue(fmt.Sprint(ThisWeekAddTaskNum))
	taskRow.Cells[3].SetValue(fmt.Sprint(ConvertDataType(AddTaskIncDecRatio)) + "%")
	taskRow.Cells[4].SetValue(fmt.Sprint(LastWeekCheckTaskNum))
	taskRow.Cells[5].SetValue(fmt.Sprint(ThisWeekCheckTaskNum))
	taskRow.Cells[6].SetValue(fmt.Sprint(ConvertDataType(CheckTaskIncDecRatio)) + "%")
	taskRow.Cells[7].SetValue(fmt.Sprint(NeedToCheckTaskAllNum))
	newLuaRow := sheet.Rows[2]
	newLuaRow.Cells[1].SetValue(fmt.Sprint(LastWeekAddLuaNum))
	newLuaRow.Cells[2].SetValue(fmt.Sprint(ThisWeekAddLuaNum))
	newLuaRow.Cells[3].SetValue(fmt.Sprint(ConvertDataType(AddLuaIncDecRatio)) + "%")
	newLuaRow.Cells[4].SetValue(fmt.Sprint(LastWeekFinishLuaNum))
	newLuaRow.Cells[5].SetValue(fmt.Sprint(ThisWeekFinishLuaNum))
	newLuaRow.Cells[6].SetValue(fmt.Sprint(ConvertDataType(FinishLuaIncDecRatio)) + "%")
	newLuaRow.Cells[7].SetValue(fmt.Sprint(NeedToFinishNewLuaAllNum))
	historyLuaRow := sheet.Rows[3]
	historyLuaRow.Cells[1].SetValue(fmt.Sprint(LastWeekAddEffectTaskNum))
	historyLuaRow.Cells[2].SetValue(fmt.Sprint(ThisWeekAddEffectTaskNum))
	historyLuaRow.Cells[3].SetValue(fmt.Sprint(ConvertDataType(AddEffectTaskIncDecRatio)) + "%")
	historyLuaRow.Cells[4].SetValue(fmt.Sprint(LastWeekFinishEffectTaskNum))
	historyLuaRow.Cells[5].SetValue(fmt.Sprint(ThisWeekFinishEffectTaskNum))
	historyLuaRow.Cells[6].SetValue(fmt.Sprint(ConvertDataType(FinishEffectTaskIncDecRatio)) + "%")
	historyLuaRow.Cells[7].SetValue(fmt.Sprint(NeedToFinishEffectTaskAllNum))
	pythonRow := sheet.Rows[4]
	pythonRow.Cells[1].SetValue(fmt.Sprint(LastWeekAddPythonNum))
	pythonRow.Cells[2].SetValue(fmt.Sprint(ThisWeekAddPythonNum))
	pythonRow.Cells[3].SetValue(fmt.Sprint(ConvertDataType(AddPythonIncDecRatio)) + "%")
	pythonRow.Cells[4].SetValue(fmt.Sprint(LastWeekFinishPythonNum))
	pythonRow.Cells[5].SetValue(fmt.Sprint(ThisWeekFinishPythonNum))
	pythonRow.Cells[6].SetValue(fmt.Sprint(ConvertDataType(FinishPythonIncDecRatio)) + "%")
	pythonRow.Cells[7].SetValue(fmt.Sprint(NeedToFinishPythonAllNum))
	thisWeekNumRow := sheet.Rows[6]
	thisWeekNumRow.Cells[1].SetValue(fmt.Sprint(LastWeekCodeAllNum))
	thisWeekNumRow.Cells[2].SetValue(fmt.Sprint(LastWeekSiteAllNum))
	thisWeekNumRow.Cells[3].SetValue(fmt.Sprint(LastWeekMgoNum))
	thisWeekNumRow.Cells[4].SetValue(fmt.Sprint(LastWeekEsNum))
	thisWeekNumRow.Cells[5].SetValue(fmt.Sprint(LastWeekCompeteMgoNum))
	thisWeekNumRow.Cells[6].SetValue(fmt.Sprint(LastWeekCompeteEsNum))
	lastWeekNumRow := sheet.Rows[7]
	lastWeekNumRow.Cells[1].SetValue(fmt.Sprint(ThisWeekCodeAllNum))
	lastWeekNumRow.Cells[2].SetValue(fmt.Sprint(ThisWeekSiteAllNum))
	lastWeekNumRow.Cells[3].SetValue(fmt.Sprint(ThisWeekMgoNum))
	lastWeekNumRow.Cells[4].SetValue(fmt.Sprint(ThisWeekEsNum))
	lastWeekNumRow.Cells[5].SetValue(fmt.Sprint(ThisWeekCompeteMgoNum))
	lastWeekNumRow.Cells[6].SetValue(fmt.Sprint(ThisWeekCompeteEsNum))
	incDecRatioRow := sheet.Rows[8]
	incDecRatioRow.Cells[1].SetValue(fmt.Sprint(ConvertDataType(AllCodeIncDecRatio)) + "%")
	incDecRatioRow.Cells[2].SetValue(fmt.Sprint(ConvertDataType(AllSiteIncDecRatio)) + "%")
	incDecRatioRow.Cells[3].SetValue(fmt.Sprint(ConvertDataType(AllMgoNumIncDecRatio)) + "%")
	incDecRatioRow.Cells[4].SetValue(fmt.Sprint(ConvertDataType(AllEsNumIncDecRatio)) + "%")
	incDecRatioRow.Cells[5].SetValue(fmt.Sprint(ConvertDataType(CompeteMgoNumIncDecRatio)) + "%")
	incDecRatioRow.Cells[6].SetValue(fmt.Sprint(ConvertDataType(CompeteEsNumIncDecRatio)) + "%")
	changeNumrow := sheet.Rows[9]
	changeNumrow.Cells[1].SetValue(fmt.Sprint(ThisWeekCodeAllNum - LastWeekCodeAllNum))
	changeNumrow.Cells[2].SetValue(fmt.Sprint(ThisWeekSiteAllNum - LastWeekSiteAllNum))
	changeNumrow.Cells[3].SetValue(fmt.Sprint(ThisWeekMgoNum - LastWeekMgoNum))
	changeNumrow.Cells[4].SetValue(fmt.Sprint(ThisWeekEsNum - int64(LastWeekEsNum)))
	changeNumrow.Cells[5].SetValue(fmt.Sprint(ThisWeekCompeteMgoNum - LastWeekCompeteMgoNum))
	changeNumrow.Cells[6].SetValue(fmt.Sprint(ThisWeekCompeteEsNum - LastWeekCompeteEsNum))

	mw := &util.MyWrite{
		Byte: &bytes.Buffer{},
	}
	file.Write(mw)
	bt := mw.Byte.Bytes()
	now := time.Now()
	name := qu.FormatDate(&now, qu.Date_Short_Layout) + "爬虫统计周报.xlsx"
	gm.GSendMail_Bq("jy@jianyu360.cn", "zhangjinkun@topnet.net.cn,maxiaoshan@topnet.net.cn", "", "", "爬虫统计周报", "", name, bt, util.GMail)
}

func GetCodeAndSiteInfo() (int, int) {
	defer qu.Catch()
	codeMap := map[string]bool{} //爬虫集合
	siteMap := map[string]bool{} //站点集合
	//1、统计爬虫和站点
	sess := util.MgoEB.GetMgoConn()
	defer util.MgoEB.DestoryMongoConn(sess)
	ch := make(chan bool, 2)
	wg := &sync.WaitGroup{}
	lock := &sync.Mutex{}
	query := map[string]interface{}{
		"state": map[string]interface{}{
			"$nin": []int{4, 10}, //查询非作废和非删除状态的爬虫
		},
	}
	field := map[string]interface{}{
		"code": 1,
		"site": 1,
	}
	it := sess.DB(util.MgoEB.DbName).C("luaconfig").Find(&query).Select(&field).Iter()
	n := 0
	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
		ch <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			code := qu.ObjToString(tmp["code"])
			site := qu.ObjToString(tmp["site"])
			lock.Lock()
			codeMap[code] = true
			siteMap[site] = true
			lock.Unlock()
		}(tmp)
		if n%1000 == 0 {
			qu.Debug("current:", n)
		}
		tmp = map[string]interface{}{}
	}
	wg.Wait()
	codeNum := len(codeMap)
	siteNum := len(siteMap)
	codeMap = map[string]bool{} //爬虫集合
	siteMap = map[string]bool{} //站点集合
	return codeNum, siteNum
}

func GetThisWeekCompeteMgoNum(sTime, eTime int64) (ThisWeekCompeteMgoNum int) {
	defer qu.Catch()
	sess := util.MgoB.GetMgoConn()
	defer util.MgoB.DestoryMongoConn(sess)
	ch := make(chan bool, 2)
	wg := &sync.WaitGroup{}
	lock := &sync.Mutex{}
	query := map[string]interface{}{
		"comeintime": map[string]interface{}{
			"$gte": sTime,
			"$lt":  eTime,
		},
	}
	field := map[string]interface{}{
		"site": 1,
	}
	it := sess.DB(util.MgoB.DbName).C("bidding").Find(&query).Select(&field).Iter()
	n := 0
	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
		ch <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			site := qu.ObjToString(tmp["site"])
			lock.Lock()
			if site == Compete_YBW || site == Compete_ZGZBYCGW {
				ThisWeekCompeteMgoNum++
			}
			lock.Unlock()
		}(tmp)
		if n%1000 == 0 {
			qu.Debug("current:", n)
		}
		tmp = map[string]interface{}{}
	}
	wg.Wait()
	return
}

func GetLastWeekReport() (int, int, int, int, int, int, int, int, int, int, int, int, int, int) {
	defer qu.Catch()
	datas, _ := util.MgoS.Find("spider_weeklyreport", nil, map[string]interface{}{"_id": -1}, nil, true, 0, 1)
	if len(*datas) == 1 {
		data := (*datas)[0]
		return qu.IntAll(data["addtasknum"]),
			qu.IntAll(data["addluanum"]),
			qu.IntAll(data["addeffecttasknum"]),
			qu.IntAll(data["addpythonnum"]),
			qu.IntAll(data["checktasknum"]),
			qu.IntAll(data["finishluanum"]),
			qu.IntAll(data["finisheffecttasknum"]),
			qu.IntAll(data["finishpythonnum"]),
			qu.IntAll(data["allcodenum"]),
			qu.IntAll(data["allsitenum"]),
			qu.IntAll(data["mgonum"]),
			qu.IntAll(data["esnum"]),
			qu.IntAll(data["competemgonum"]),
			qu.IntAll(data["competeesnum"])
	} else {
		qu.Debug("历史周报信息查询失败")
	}
	return -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
}

func CountNeedToFinishNewLuaAllNum(query map[string]interface{}) (result int) {
	defer qu.Catch()
	sess := util.MgoEB.GetMgoConn()
	defer util.MgoEB.DestoryMongoConn(sess)
	ch := make(chan bool, 2)
	wg := &sync.WaitGroup{}
	lock := &sync.Mutex{}
	field := map[string]interface{}{
		"code": 1,
	}
	it := sess.DB(util.MgoEB.DbName).C("luaconfig").Find(&query).Select(&field).Iter()
	n := 0
	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
		ch <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			code := qu.ObjToString(tmp["code"])
			count := util.MgoE.Count("lua_logs_auditor", map[string]interface{}{"code": code, "types": "审核"})
			if count == 0 { //无审核记录表示新爬虫
				lock.Lock()
				result++
				lock.Unlock()
			}
		}(tmp)
		if n%100 == 0 {
			qu.Debug("current:", n)
		}
		tmp = map[string]interface{}{}
	}
	wg.Wait()
	return
}

//解决float64*100后精度丢失问题
func ConvertDataType(ratio float64) float64 {
	ratio, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", ratio*100), 64)
	return ratio
}