DataMaintenance/src/lua/data.go

package lua

import (
	"mongodb"
	qu "qfw/util"
	"qfw/util/elastic"
	"strings"
	"util"
)

func (l *Lua) DataView() {
	defer qu.Catch()
	coll := l.GetString("coll")
	id := l.GetString("id")
	data := &map[string]interface{}{}
	if coll == "bidding" || coll == "bidding_file" {
		data, _ = util.JYMgo.FindById(coll, id, nil)
	} else {
		if tmp := strings.Split(coll, "-"); len(tmp) == 2 {
			if tmp[0] == "lua" {
				data, _ = util.MgoS.FindById(tmp[1], id, nil)
			} else if tmp[0] == "python" {
				data, _ = util.MgoPy.FindById(tmp[1], id, nil)
			}
		} else {
			data, _ = util.MgoS.FindById(coll, id, nil)
		}
	}
	l.T["data"] = *data
	l.Render("lua/dataview.html", &l.T)
}

func (l *Lua) DataFind() {
	defer qu.Catch()
	l.Render("lua/datafind.html")
}

var Fields = map[string]interface{}{
	"title":       1,
	"publishtime": 1,
	"comeintime":  1,
	"href":        1,
	"extracttype": 1,
	"site":        1,
	"spidercode":  1,
	"s_sha":       1,
	"biddingid":   1,
	"moveok":      1,
	"repeatby":    1,
	"hashref":     1,
	"field":       1,
	"info":        1,
	"level":       1,
	"biddingcoll": 1,
	"sendflag":    1,
	"data":        1,
}

func (l *Lua) SearchData() {
	defer qu.Catch()
	text := l.GetString("text")
	stype := l.GetString("stype")
	data := []map[string]interface{}{}
	query := map[string]interface{}{
		stype: text,
	}
	qu.Debug("query:", query)
	//bidding、索引分析
	if stype == "title" { //根据title找bidding(bidding表有title索引，无href索引)
		query = map[string]interface{}{
			"title": text,
		}
		//1、根据title找bidding
		list_bid, _ := util.JYMgo.Find("bidding", query, nil, Fields, false, -1, -1)
		for _, l := range *list_bid {
			id := mongodb.BsonIdToSId(l["_id"])
			es_query := `{"query":{"bool":{"must":[{"term":{"id":"` + id + `"}}]}}}`
			if util.JyEs.Count(util.EsIndex, util.EsType, es_query) > 0 {
				l["es"] = "已生成"
			} else {
				extracttype := qu.IntAll(l["extracttype"])
				if extracttype == -1 {
					l["es"] = "被判重"
				} else if extracttype == 1 {
					l["es"] = "ES异常"
				} else {
					l["es"] = "未生成"
				}
			}
		}
		AddVal(*list_bid, "bidding", &data)
	} else if stype == "href" { //根据href查索引数据，反推bidding
		es_query := `{"query":{"bool":{"must":[{"term":{"href":"` + text + `"}}]}},"_source":["_id"]}`
		list := elastic.Get(util.EsIndex, util.EsType, es_query)
		for _, l := range *list {
			id := qu.ObjToString(l["_id"])
			qu.Debug(id)
			tmp, _ := util.JYMgo.FindById("bidding", id, Fields)
			if tmp != nil && len(*tmp) > 0 {
				(*tmp)["es"] = "已生成"
				(*tmp)["by"] = "bidding"
				data = append(data, *tmp)
			}
		}
	}
	qu.Debug("bidding 搜索完成...")
	//2、找bidding_file附件信息
	list_file, _ := util.JYMgo.Find("bidding_file", query, `{"_id":-1}`, Fields, false, 0, 20)
	AddVal(*list_file, "bidding_file", &data)
	qu.Debug("bidding_file 搜索完成...", len(*list_file))
	//3、找spider_repeatdata判重数据表
	list_repeat, _ := util.MgoS.Find("spider_repeatdata", query, `{"_id":-1}`, Fields, false, 0, 20)
	AddVal(*list_repeat, "spider_repeatdata", &data)
	qu.Debug("spider_repeatdata 搜索完成...", len(*list_repeat))
	//4、找spider_warn异常数据表
	list_warn, _ := util.MgoS.Find("spider_warn", query, `{"_id":-1}`, Fields, false, 0, 20)
	AddVal(*list_warn, "spider_warn", &data)
	qu.Debug("spider_warn 搜索完成...", len(*list_warn))
	//5、找data_bak表
	list_bak_lua, _ := util.MgoS.Find("data_bak", query, `{"_id":-1}`, Fields, false, 0, 20)
	AddVal(*list_bak_lua, "lua-data_bak", &data)
	qu.Debug("lua-data_bak 搜索完成...", len(*list_bak_lua))
	list_bak_py, _ := util.MgoPy.Find("data_bak", query, `{"_id":-1}`, Fields, false, 0, 20)
	AddVal(*list_bak_py, "python-data_bak", &data)
	qu.Debug("python-data_bak 搜索完成...", len(*list_bak_py))
	//if len(data) == 0 { //以上都没有找到数据，找列表页数据
	//
	//}
	l.ServeJson(map[string]interface{}{
		"data": data,
	})
	data = []map[string]interface{}{}
}

func AddVal(list []map[string]interface{}, by string, data *[]map[string]interface{}) {
	if len(list) > 0 {
		for _, l := range list {
			l["by"] = by
		}
		*data = append(*data, list...)
	}
}