package spider

import (
	"encoding/json"
	"fmt"
	"github.com/cron"
	"github.com/donnie4w/go-logger/logger"
	"github.com/yuin/gopher-lua"
	mu "mfw/util"
	"qfw/util"

	//"qfw/util/redis"
	lu "spiderutil"
	"strings"
	"sync/atomic"
	"time"
)

type LogMap struct {
	Code        string //爬虫代码
	Name        string //爬虫名称
	ChannelUrl  string //栏目网址
	Url         string //目标页地址
	Type_web    int    //网站改版
	Type_script int    //脚本
	Type_net    int    //网络
	Type_check  int    //验证
	Type_other  int    //其他
	UploadTime  string //脚本上传时间
	ErrInfo     string //错误信息
	Errtime     int64  //时间

	Round         int //下载轮次
	DownloadCount int //下载量
}

//数据存储批量
func StoreBlak(mode, event int, c, coverAttr string, data []map[string]interface{}) {
	defer mu.Catch()
	for _, v := range data {
		if t, err := time.ParseInLocation(util.Date_Full_Layout, util.ObjToString(v["publishtime"]), time.Local); err == nil {
			v["publishtime"] = t.Unix()
			v["T"] = c
		}
	}
	if mode == 1 {
		b := MgoS.SaveBulk(c, data...)
		if b {
			logger.Debug(c, mode, "保存成功")
		} else {
			logger.Warn(c, mode, "保存失败")
		}
	} else if mode == 2 {
		SaveObjBlak(event, coverAttr, c, data)
	}
}

//数据存储
func Store(mode, event int, c, coverAttr string, data map[string]interface{}, flag bool) {
	defer mu.Catch()
	if t, err := time.ParseInLocation(util.Date_Full_Layout, util.ObjToString(data["publishtime"]), time.Local); err == nil {
		data["publishtime"] = t.Unix()
	}
	if mode == 1 {
		if strings.HasPrefix(c, "ecps") { //公示入库标识值
			data["signvalue"] = int(1)
			if MgoS.Update(c, map[string]interface{}{"coverAttr": fmt.Sprint(data["title"])}, map[string]interface{}{"$set": data}, true, false) {
				logger.Debug(c, data["title"], mode, "保存成功")
			}
		} else {
			id := MgoS.Save(c, data)
			if id != "" {
				logger.Debug(c, mode, "保存成功")
			} else {
				logger.Warn(c, mode, "保存失败", data)
			}
		}
		href := fmt.Sprint(data["href"])
		if len(href) > 5 && flag { //有效数据
			hashHref := HexText(href)
			lu.RedisClusterSet(hashHref, "", -1)
		}
	} else if mode == 2 {
		data["T"] = c
		SaveObj(event, coverAttr, data, flag)
	}
	//公示数据，发送其他服务请求
	if strings.HasPrefix(c, "ecps") {
		SendMsgService(mu.SERVICE_SPIDER_ECPS, []map[string]interface{}{map[string]interface{}{"title": data["title"], "type": "ecps"}})
	}
}

//保存验证错误日志
func saveVerificationLog(code, name, url, content string) {
	defer mu.Catch()
	data := map[string]interface{}{}
	data["code"] = code
	data["name"] = name
	data["url"] = url
	data["content"] = content
	data["comeintime"] = time.Now().Unix()
	data["event"] = lu.Config.Uploadevent
	MgoS.Save("spider_errlog", data)
}

//查找信息是否存在
func findHasExit(c, q string) bool {
	defer mu.Catch()
	ret, _ := MgoS.FindOne(c, q)
	if ret != nil && len(*ret) > 0 {
		return true
	} else {
		return false
	}
}

//获取最后发布时间
var spider_ldtime = map[string]map[string]interface{}{}

func GetLastPubtime(code string) int64 {
	defer mu.Catch()
	if len(spider_ldtime) < 1 {
		list, _ := MgoS.Find("spider_ldtime", nil, nil, nil, false, -1, -1)
		for _, v := range *list {
			spider_ldtime[fmt.Sprint(v["code"])] = v
		}
	}
	if spider_ldtime[code] != nil {
		lastpubtime := spider_ldtime[code]["lastpubtime"]
		return util.Int64All(lastpubtime)
	} else {
		return 0
	}
}

//获取最后执行时间
func GetLastExectime(code string) int64 {
	defer mu.Catch()
	if len(spider_ldtime) < 1 {
		list, _ := MgoS.Find("spider_ldtime", nil, nil, nil, false, -1, -1)
		for _, v := range *list {
			spider_ldtime[fmt.Sprint(v["code"])] = v
		}
	}
	if spider_ldtime[code] != nil {
		lastexectime := spider_ldtime[code]["lastexectime"]
		return util.Int64All(lastexectime)
	} else {
		return 0
	}
}

//获取下载量
var spider_downlog = map[string]map[string]interface{}{}

func GetDownloadLast(code, date string) map[string]interface{} {
	defer mu.Catch()
	if len(spider_downlog) < 1 {
		list, _ := MgoS.Find("spider_downlog", map[string]interface{}{"date": date}, nil, nil, false, -1, -1)
		for _, v := range *list {
			spider_downlog[fmt.Sprint(v["code"])] = v
		}
	}
	if spider_downlog[code] != nil {
		return spider_downlog[code]
	} else {
		return nil
	}
}

//GcCount
func GcCount() {
	t := time.Now()
	if t.Hour() == 0 {
		Allspiders.Range(func(key, value interface{}) bool {
			v := value.(*Spider)
			//生成爬虫数据
			SaveDownCount(v.Code, true, v.TodayDowncount, v.ToDayRequestNum, v.YesterdayDowncount, v.YestoDayRequestNum)
			atomic.StoreInt32(&v.YesterdayDowncount, 0)
			atomic.AddInt32(&v.YesterdayDowncount, v.TodayDowncount)
			atomic.StoreInt32(&v.TodayDowncount, 0)
			atomic.StoreInt32(&v.YestoDayRequestNum, 0)
			atomic.AddInt32(&v.YestoDayRequestNum, v.ToDayRequestNum)
			atomic.StoreInt32(&v.ToDayRequestNum, 0)
			atomic.StoreInt32(&v.LastDowncount, 0)
			atomic.StoreInt32(&v.RoundCount, 0)
			return true
		})
		//删除15天前的数据
		//mgu.Del("data_bak", "spider", "spider", `{"comeintime":{"$lt":`+fmt.Sprint(t.Add(-200*24*time.Hour).Unix())+`}}`)
		MgoS.Del("spider_errlog", map[string]interface{}{"comeintime": map[string]interface{}{"$lt": t.Add(-30 * 24 * time.Hour).Unix()}})
		lu.TimeSleepFunc(1*time.Hour, TimeSleepChan)
	}
	lu.TimeAfterFunc(30*time.Minute, GcCount, TimeChan)
}

//保存错误数据信息,重新下载
func SaveErrorData(modifyuser string, pd map[string]interface{}, err interface{}) {
	defer util.Catch()
	if href := util.ObjToString(pd["href"]); href != "" {
		delete(pd, "_id")
		pd["state"] = 0
		pd["from"] = "lua"
		pd["comeintime"] = time.Now().Unix()
		pd["modifyuser"] = modifyuser
		if luaErr, ok := err.(*lua.ApiError); ok && luaErr != nil {
			pd["error"] = luaErr.Object.String()
		}
		if publishtime, ok := pd["publishtime"].(string); ok {
			pd["publishtime"] = lu.ParseDate2Int64(publishtime)
		}
		if jsondata := util.ObjToString(pd["jsondata"]); jsondata != "" && jsondata != "null" {
			tmp := map[string]interface{}{}
			json.Unmarshal([]byte(jsondata), &tmp)
			pd["jsondata"] = tmp
		}
		//mgu.Save("regatherdata", "spider", "spider", pd)
		query := map[string]interface{}{
			"href": href,
		}
		set := map[string]interface{}{
			"$set": pd,
		}
		update := []map[string]interface{}{}
		update = append(update, query)
		update = append(update, set)
		UpdataMgoCache <- update
		//Mgo.Update("regatherdata", "spider", "spider", query, set, true, false)
	}
}

//保存modal=1模式采集的列表页信息
func SaveHighListPageData(tmp map[string]interface{}, code, hashHref string, num *int) {
	//列表页href判重
	redisCode := lu.RedisClusterGet("list_" + hashHref)
	//此处区分历史节点(7000)和增量节点
	if redisCode != "" {
		if lu.Config.IsHistoryEvent || strings.Contains(redisCode, code) { //列表页数据已采集
			*num++
			return
		} else {
			lu.RedisClusterSet("list_"+hashHref, code+"+"+redisCode, 86400*365*2) //两年
		}
	} else {
		lu.RedisClusterSet("list_"+hashHref, code+"+"+redisCode, 86400*365*2) //两年
	}
	//if redisCode != "" && strings.Contains(redisCode, code) { //相同爬虫采集且href相同，表示重复
	//	*num++
	//	return
	//} else {
	//	lu.RedisClusterSet("list_"+hashHref, code+"+"+redisCode, 86400*365*2) //两年
	//}
	tmp["state"] = 0
	tmp["event"] = lu.Config.Uploadevent
	tmp["comeintime"] = time.Now().Unix()
	if lu.Config.IsHistoryEvent {
		MgoS.Save("spider_historydata", tmp)
	} else {
		MgoS.Save("spider_highlistdata", tmp)
	}
}

//保存7410、7500、7510、7520、7700采集的列表页信息
func SaveListPageData(tmp map[string]interface{}, id *string, isEsRepeat bool) {
	//spider_listdata数据保证href唯一
	if MgoS.Count("spider_listdata", map[string]interface{}{"href": tmp["href"]}) == 0 {
		tmp["event"] = lu.Config.Uploadevent
		tmp["comeintime"] = time.Now().Unix()
		tmp["state"] = 0
		tmp["exist"] = false
		if isEsRepeat { //类竞品数据es判重掉后，更新状态
			tmp["state"] = 1
			tmp["exist"] = true
			tmp["updatetime"] = time.Now().Unix()
		}
		*id = MgoS.Save("spider_listdata", tmp)
	}
}

//获取第day天凌晨的时间戳
func GetTime(day int) int64 {
	nowTime := time.Now().AddDate(0, 0, day)
	timeStr := util.FormatDate(&nowTime, util.Date_Short_Layout)
	t, _ := time.ParseInLocation(util.Date_Short_Layout, timeStr, time.Local)
	return t.Unix()
}

//更新state状态
func UpdateHighListDataByCode(code string) {
	query := map[string]interface{}{
		"spidercode": code,
		"state":      -1,
		"comeintime": map[string]interface{}{
			"$gte": GetTime(-lu.Config.DayNum),
			"$lte": time.Now().Unix(),
		},
	}
	set := map[string]interface{}{
		"$set": map[string]interface{}{
			"state": 0,
			"times": 0,
		},
	}
	MgoS.Update("spider_highlistdata", query, set, false, true)
}

//批量更新错误数据
func UpdateErrDataMgo() {
	fmt.Println("Update Error Data...")
	arru := make([][]map[string]interface{}, 50)
	indexu := 0
	for {
		select {
		case v := <-UpdataMgoCache:
			arru[indexu] = v
			indexu++
			if indexu == 50 {
				SP <- true
				go func(arru [][]map[string]interface{}) {
					defer func() {
						<-SP
					}()
					MgoS.UpSertBulk("regatherdata", arru...)
				}(arru)
				arru = make([][]map[string]interface{}, 50)
				indexu = 0
			}
		case <-time.After(1 * time.Minute):
			if indexu > 0 {
				SP <- true
				go func(arru [][]map[string]interface{}) {
					defer func() {
						<-SP
					}()
					MgoS.UpSertBulk("regatherdata", arru...)
				}(arru[:indexu])
				arru = make([][]map[string]interface{}, 50)
				indexu = 0
			}
		}
	}
}

//批量更新心跳信息
func UpdateHeartInfo() {
	fmt.Println("Update Heart Info...")
	heartarr := make([][]map[string]interface{}, 200)
	indexh := 0
	for {
		select {
		case v := <-UpdataHeartCache:
			heartarr[indexh] = v
			indexh++
			if indexh == 200 {
				SPH <- true
				go func(heartarr [][]map[string]interface{}) {
					defer func() {
						<-SPH
					}()
					MgoS.UpSertBulk("spider_heart", heartarr...)
				}(heartarr)
				heartarr = make([][]map[string]interface{}, 200)
				indexh = 0
			}
		case <-time.After(1 * time.Minute):
			if indexh > 0 {
				SPH <- true
				go func(heartarr [][]map[string]interface{}) {
					defer func() {
						<-SPH
					}()
					MgoS.UpSertBulk("spider_heart", heartarr...)
				}(heartarr[:indexh])
				heartarr = make([][]map[string]interface{}, 200)
				indexh = 0
			}
		}
	}
}

//保存爬虫采集非本站点数据
func SaveOtherSiteData() {
	fmt.Println("Save Other Site Data...")
	savearr := make([]map[string]interface{}, 200)
	indexh := 0
	for {
		select {
		case v := <-SaveMgoCache:
			savearr[indexh] = v
			indexh++
			if indexh == 200 {
				SPS <- true
				go func(savearr []map[string]interface{}) {
					defer func() {
						<-SPS
					}()
					MgoS.SaveBulk("spider_othersite", savearr...)
				}(savearr)
				savearr = make([]map[string]interface{}, 200)
				indexh = 0
			}
		case <-time.After(1 * time.Minute):
			if indexh > 0 {
				SPS <- true
				go func(savearr []map[string]interface{}) {
					defer func() {
						<-SPS
					}()
					MgoS.SaveBulk("spider_othersite", savearr...)
				}(savearr[:indexh])
				savearr = make([]map[string]interface{}, 200)
				indexh = 0
			}
		}
	}
}

//定时任务
func TimeTask() {
	now := time.Now()
	Today = util.FormatDate(&now, util.Date_Short_Layout) //初始化日期
	cr := cron.New()
	cr.Start()
	cr.AddFunc("0 30 0 * * ?", UpdateSpiderFlow) //每天零时提交统计
}

//更新流量信息
func UpdateSpiderFlow() {
	defer util.Catch()
	logger.Info("统计流量信息开始...", Today)
	arr := []map[string]interface{}{}
	SpiderFlowMap.Range(func(key, temp interface{}) bool {
		date := strings.Split(key.(string), "+")
		if len(date) == 2 && date[0] == Today { //统计非当天的
			if sfMap, ok := temp.(*SpiderFlow); ok {
				arr = append(arr, map[string]interface{}{
					"spidercode": date[1],
					"date":       date[0],
					"flow":       sfMap.Flow,
					"site":       sfMap.Site,
					"channel":    sfMap.Channel,
					"modifyuser": sfMap.ModifyUser,
					"comeintime": time.Now().Unix(),
					"event":      lu.Config.Uploadevent,
				})
				SpiderFlowMap.Delete(key) //统计完成后删除非当天数据
			}
		}
		return true
	})
	if len(arr) > 0 {
		MgoS.SaveBulk("spider_flow", arr...)
		arr = []map[string]interface{}{}
	}
	now := time.Now()
	Today = util.FormatDate(&now, util.Date_Short_Layout)
	logger.Info("统计流量信息完成...", Today)
}
