package extract

import (
	"encoding/json"
	"fmt"
	log "github.com/donnie4w/go-logger/logger"
	"jy/clear"
	db "jy/mongodbutil"
	ju "jy/util"
	qu "qfw/util"
	"sort"
	"strings"
	"time"
)

// 分析抽取结果并保存
func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
	qu.Try(func() {
		//(j.Category == "招标" || j.Category == "预告")
		if (j.Category == "招标" || j.Category == "预告") && (len(j.BlockPackage) > 0 || len(j.PackageInfo) > 0 || len(j.Result) > 0) {
			if j.CategorySecond != "单一" {
				delete(j.Result, "winner")
				delete(j.Result, "bidamount")
				for _, v := range j.BlockPackage {
					v.Bidamount = 0
					v.IsTrueBidamount = false
					if v.Winner != "" {
						v.Winner = ""
						if v.SpaceKV != nil {
							delete(v.SpaceKV.KvTags, "中标单位")
						}
						if v.TableKV != nil {
							delete(v.TableKV.KvTags, "中标单位")
						}
						if v.ColonKV != nil {
							delete(v.ColonKV.KvTags, "中标单位")
						}
					}
				}
				for _, v := range j.PackageInfo {
					delete(v, "winner")
					delete(v, "bidamount")
				}
				j.Winnerorder = nil
				if jf != nil && jf.Winnerorder != nil {
					jf.Winnerorder = nil
				}
			}
		}
		//重新取出清理过后的中标候选人重置候选人
		e.ResetWinnerorder(j)
		//打分
		doc, result, _id := funcAnalysis(j, e)
		//标签是否保存
		if ju.IsSaveTag {
			go otherNeedSave(j, result, e)
		}
		//从排序结果中取值
		tmp := map[string]interface{}{} //抽取值
		tmp["spidercode"] = j.SpiderCode
		tmp["site"] = j.Site
		if len(*j.Jsondata) > 0 {
			tmp["jsondata"] = j.Jsondata
		}
		//字段-抽取来源
		fieldSource := make(map[string]interface{}, 0)
		//字段-抽取来源
		for k, val := range result {
			if k == "qualifies" {
				squalifies := make([]interface{}, 0)
				squalifiesMap := make(map[string]*scoreIndex, 0)
				for _, kv := range val {
					skey := kv.RuleText
					if kv.Score > 0 {
						if squalifiesMap[skey] == nil {
							squalifiesMap = map[string]*scoreIndex{
								skey: &scoreIndex{
									Score: kv.Score,
									Index: len(squalifies),
								},
							}
							squalifies = append(squalifies, map[string]interface{}{
								"key":   skey,
								"value": kv.Value,
							})
						} else {
							if squalifiesMap[skey].Score < kv.Score {
								squalifies[squalifiesMap[skey].Index] = map[string]interface{}{
									"key":   skey,
									"value": kv.Value,
								}
							}
						}
					}
				}
				tmp[k] = squalifies
				continue
			}

			//预算-中标金额字段-特殊情况特殊处理
			if k == "bidamount" || k == "budget" {
				b, index := calculateAbnormalMoney(val)
				if b {
					new_v := val[index]
					tmp[new_v.Field] = new_v.Value
					fieldSource[new_v.Field] = map[string]interface{}{
						"ext_type": new_v.Type,
						"ext_from": new_v.ExtFrom,
						"ext_tag":  new_v.BlockTitle,
					}
					tmp["is_dif_ratioMoney"] = true
					continue
				}
			}

			for _, v := range val { //取第一个非负数,项目名称除外//存0是否有效
				if (v.Field == "bidamount" || v.Field == "budget") && v.IsTrue && v.Score > -1 {
					tmp[v.Field] = v.Value
					fieldSource[v.Field] = map[string]interface{}{
						"ext_type": v.Type,
						"ext_from": v.ExtFrom,
						"ext_tag":  v.BlockTitle,
					}
					break
				}
				if v.Score > -1 && (v.Field != "bidamount" && v.Field != "budget") && len(strings.TrimSpace(fmt.Sprint(v.Value))) > 0 {
					tmp[v.Field] = v.Value
					fieldSource[v.Field] = map[string]interface{}{
						"ext_type": v.Type,
						"ext_from": v.ExtFrom,
						"ext_tag":  v.BlockTitle,
					}
					//中标单位~含字母判断~对比企业库
					if (v.Field == "winner" || v.Field == "buyer") && letter_entity.MatchString(qu.ObjToString(v.SourceValue)) {
						qyxy_data := ju.Qyxy_Mgo.FindOne("qyxy_std", map[string]interface{}{
							"company_name": qu.ObjToString(v.SourceValue),
						})
						if qyxy_data != nil && len(qyxy_data) > 0 {
							tmp[v.Field] = v.SourceValue
						}
					}

					break
				}
			}
		}
		tmp["winner"] = strings.ReplaceAll(qu.ObjToString(tmp["winner"]), ",,", ",")
		if len(j.PackageInfo) > 15 {
			for k, v := range j.PackageInfo {
				j.PackageInfo = map[string]map[string]interface{}{}
				j.PackageInfo[k] = v
				break
			}
		}
		if len(j.PackageInfo) > 0 { //分包信息
			tmp["package"] = j.PackageInfo
			//包预算，中标金额合并大于抽取就覆盖
			tmpBidamount, tmpBudget, tmpAgencyfee := qu.Float64All(0), qu.Float64All(0), qu.Float64All(0)
			//s_winner逗号分隔拼接，分包中标人
			var tmpstr, savewinner []string
			//按包排序
			for b, v := range j.PackageInfo {
				if v["winner"] != nil && v["winner"] != "" {
					tmpstr = append(tmpstr, b)
				}
			}
			//包预算，中标金额合并大于抽取就覆盖
			if len(j.PackageInfo) > 1 {
				//包数大于1累加
				for _, v := range j.PackageInfo {
					if v["budget"] != nil {
						tmpBudget = precisionAddFloat(tmpBudget, qu.Float64All(v["budget"]))
					}
					if v["bidamount"] != nil {
						tmpBidamount = precisionAddFloat(tmpBidamount, qu.Float64All(v["bidamount"]))
					}
					if v["agencyfee"] != nil {
						tmpAgencyfee = precisionAddFloat(tmpAgencyfee, qu.Float64All(v["agencyfee"]))
					}
				}
				if qu.Float64All(tmp["budget"]) < tmpBudget {
					fieldSource["budget"] = map[string]interface{}{
						"ext_type": "",
						"ext_from": "package",
						"ext_tag":  "",
					}
					tmp["budget"] = tmpBudget
				}
				if qu.Float64All(tmp["agencyfee"]) < tmpAgencyfee {
					fieldSource["agencyfee"] = map[string]interface{}{
						"ext_type": "",
						"ext_from": "package",
						"ext_tag":  "",
					}
					tmp["agencyfee"] = tmpAgencyfee
				}
				if qu.Float64All(tmp["bidamount"]) < tmpBidamount {
					//特殊爬虫分包金额-不要
					if j.SpiderCode == "sx_sxgzszcgxt_fzb_cjhxrgs_bu" && qu.Float64All(tmp["bidamount"]) > 0.0 {

					} else {
						fieldSource["bidamount"] = map[string]interface{}{
							"ext_type": "",
							"ext_from": "package",
							"ext_tag":  "",
						}
						tmp["bidamount"] = tmpBidamount
					}
				}
				//if qu.Float64All(tmp["bidamount"]) > 0 && qu.Float64All(tmp["budget"]) > 0 && (qu.Float64All(tmp["bidamount"])/10 > qu.Float64All(tmp["budget"])) {
				//	fieldSource["bidamount"] = map[string]interface{}{
				//		"ext_type": "",
				//		"ext_from": "package",
				//	}
				//	tmp["bidamount"] = tmpBidamount
				//} else if qu.Float64All(tmp["bidamount"]) < tmpBidamount {
				//	fieldSource["bidamount"] = map[string]interface{}{
				//		"ext_type": "",
				//		"ext_from": "package",
				//	}
				//	tmp["bidamount"] = tmpBidamount
				//}
			} else {
				//包数等于1,tmp没有值取包里的值
				if tmp["budget"] == nil || tmp["budget"] == 0 {
					for _, v := range j.PackageInfo {
						if v["budget"] != nil {
							fieldSource["budget"] = map[string]interface{}{
								"ext_type": "",
								"ext_from": "package",
								"ext_tag":  "",
							}
							tmp["budget"] = v["budget"]
						}
					}
				}
				if tmp["agencyfee"] == nil || tmp["agencyfee"] == 0 {
					for _, v := range j.PackageInfo {
						if v["agencyfee"] != nil {
							fieldSource["agencyfee"] = map[string]interface{}{
								"ext_type": "",
								"ext_from": "package",
								"ext_tag":  "",
							}
							tmp["agencyfee"] = v["agencyfee"]
						}
					}
				}
				if tmp["bidamount"] == nil || tmp["bidamount"] == 0 {
					for _, v := range j.PackageInfo {
						if v["bidamount"] != nil {
							fieldSource["bidamount"] = map[string]interface{}{
								"ext_type": "",
								"ext_from": "package",
								"ext_tag":  "",
							}
							tmp["bidamount"] = v["bidamount"]
						}
					}
				}
			}
			//s_winner逗号分隔拼接，分包中标人
			sort.Strings(tmpstr)
			for _, v := range tmpstr {
				winner := qu.ObjToString(j.PackageInfo[v]["winner"])
				new_winner := clearWinnerReg.ReplaceAllString(winner, "")
				if new_winner == "" {
					continue
				}
				//名称黑名单
				if unPackageWinnerReg.MatchString(new_winner) {
					continue
				}
				savewinner = append(savewinner, new_winner)
			}
			if (savewinner == nil || len(savewinner) == 0) && tmp["winner"] != nil {
				tmp["s_winner"] = tmp["winner"]
				fieldSource["s_winner"] = fieldSource["winner"]
			} else if savewinner != nil {
				if len(savewinner) == 1 && tmp["winner"] != nil {
					tmp["s_winner"] = tmp["winner"]
					fieldSource["s_winner"] = fieldSource["winner"]
				} else {
					savewinner = RemoveReplicaSliceString(savewinner)
					tmp["s_winner"] = strings.Join(savewinner, ",")
					fieldSource["s_winner"] = map[string]interface{}{
						"ext_type": "",
						"ext_from": "package",
						"ext_tag":  "",
					}
				}
			}
		} else if tmp["winner"] != nil {
			//没有分包取winner
			tmp["s_winner"] = tmp["winner"]
			fieldSource["s_winner"] = fieldSource["winner"]
		}

		if len(j.Winnerorder) > 0 { //候选人信息
			for i, v := range j.Winnerorder {
				if v["price"] != nil {
					tmpPrice := clear.ObjToMoney([]interface{}{v["price"], ""}, j.SpiderCode, j.IsClearnMoney)
					if tmpPrice[len(tmpPrice)-1].(bool) {
						j.Winnerorder[i]["price"] = tmpPrice[0]
					} else {
						delete(j.Winnerorder[i], "price")
					}
				}
			}
			tmp["winnerorder"] = j.Winnerorder
		}
		//处理附件
		var resultf map[string][]*ju.ExtField
		ffield := map[string]interface{}{}
		if jf != nil {
			e.ResetWinnerorder(jf)
			_, resultf, _ = funcAnalysis(jf, e)
			for _, val := range resultf {
				for _, v := range val { //取第一个非负数
					if v.Score > -1 {
						ffield[v.Field] = v.Value
						if tmp[v.Field] == nil || tmp[v.Field] == "" {
							if v.Field == "addressing" {
								break
							}
							if (v.Field == "bidamount" || v.Field == "budget") && v.IsTrue && v.Value.(float64) > 100 && v.Value.(float64) < 50000000000 {
								tmp[v.Field] = v.Value
								fieldSource[v.Field] = map[string]interface{}{
									"ext_type": v.Type,
									"ext_from": "ff",
									"ext_tag":  v.BlockTitle,
								}
								break
							}
							if v.Score > -1 && (v.Field != "bidamount" && v.Field != "budget") && len(strings.TrimSpace(fmt.Sprint(v.Value))) > 0 {
								if v.Field == "winner" && j.Category == "招标" && j.CategorySecond != "单一" {
									break //此方法逻辑上已经不会达到这里 winner
								}
								tmp[v.Field] = v.Value
								fieldSource[v.Field] = map[string]interface{}{
									"ext_type": v.Type,
									"ext_from": "ff",
									"ext_tag":  v.BlockTitle,
								}
								//中标单位~含字母判断~对比企业库
								if (v.Field == "winner" || v.Field == "buyer") && letter_entity.MatchString(qu.ObjToString(v.SourceValue)) {
									qyxy_data := ju.Qyxy_Mgo.FindOne("qyxy_std", map[string]interface{}{
										"company_name": qu.ObjToString(v.SourceValue),
									})
									if qyxy_data != nil && len(qyxy_data) > 0 {
										tmp[v.Field] = v.SourceValue
									}
								}

								break
							}
						}
						break
					}
				}
			}
			if len(jf.PackageInfo) > 0 { //分包信息
				ffield["package"] = jf.PackageInfo
			}
			if len(jf.Winnerorder) > 0 { //候选人信息
				ffield["winnerorder"] = jf.Winnerorder
			}
		}

		//添加字段来源
		//tmp["field_source"] = fieldSource
		//是否为不规则表格字段
		//if j.IsUnRulesTab {
		//	tmp["is_UnRules_Tab"] = j.IsUnRulesTab
		//}
		//补充源表数据的数据
		for k, v := range *doc {
			if tmp[k] == nil && BiddingFields[k] != nil {
				tmp[k] = v
			}
		}
		//质量审核
		if ju.QualityAudit {
			e.QualityAudit(tmp)
		}
		//站点所有单位补充···
		if site := e.SiteCityMap[j.Site]; site != nil && qu.ObjToString(tmp["buyer"]) == "" {
			if site.B != "" {
				tmp["buyer"] = site.B
			}
		}
		//落款等文本识别
		jf_text := ""
		if jf != nil {
			jf_text = jf.ContentClean
		}
		e.inscribeRecognize(&tmp, *j.Data, jf_text)
		//根据正文获取资质要求
		e.getQualifications(&tmp, *j.Data)
		//城市抽取
		if e.IsExtractCity {
			e.ExtractRegionInfo(j, jf, &tmp, false)
			e.ExtractRegionClean(&tmp)
			delete(tmp, "regions_log")
		}
		//品牌抽取
		if ju.IsBrandGoods {
			tmp["checkhas"] = map[string]int{
				"hastable": j.HasTable,
				"hasgoods": j.HasGoods,
				"hasbrand": j.HasBrand,
				"haskey":   j.HasKey,
			}
			if len(j.BrandData) > 0 {
				tmp["tablebrand"] = j.BrandData
			}
		}
		//prince和number抽取
		if ju.IsPriceNumber {
			priceNumberLen := len(j.PriceNumberData)
			if priceNumberLen > 1 { //table数据去重
				tmpPriceNumberData := []map[string]interface{}{}
				tableStrs := map[string]bool{}
				for _, tb := range j.PriceNumberData {
					has := false
					bytes, _ := json.Marshal(tb)
					str := string(bytes)
					if len(tableStrs) > 0 && tableStrs[str] {
						has = true
					} else {
						tableStrs[str] = true
					}
					if !has {
						for _, data := range tb {
							tmpPriceNumberData = append(tmpPriceNumberData, data)
						}
					}
				}
				tmp["pricenumber"] = tmpPriceNumberData
			} else if priceNumberLen == 1 {
				tmp["pricenumber"] = j.PriceNumberData[0]
			}
		}
		//所有kv组成的字符串
		//assembleKVText(j, &tmp)
		//检查字段
		tmp["dataging"] = j.Dataging
		tmp = checkFields(tmp, *j.Data)

		if tmp["projectname"] == nil || tmp["projectname"] == "" {
			tmp["projectname"] = j.Title
		}
		tmp["repeat"] = 0
		if ju.Ffield {
			if len(ffield) > 0 {
				tmp["ffield"] = ffield
			}
		}
		//临时保存指定字段数据
		//new_tmp := map[string]interface{}{}
		//new_tmp["area"] = qu.ObjToString(tmp["area"])
		//new_tmp["city"] = qu.ObjToString(tmp["city"])
		//new_tmp["district"] = qu.ObjToString(tmp["district"])
		//tmp = new_tmp

		if e.TaskInfo.TestColl == "" {
			if len(tmp) > 0 { //保存抽取结果
				delete(tmp, "_id")
				tmparr := []map[string]interface{}{
					map[string]interface{}{
						"_id": qu.StringTOBsonId(_id),
					},
					map[string]interface{}{"$set": tmp},
				}
				e.RWMutex.Lock()
				e.BidArr = append(e.BidArr, tmparr)
				e.BidTotal++
				e.RWMutex.Unlock()
			}
			if ju.SaveResult {
				id := tmp["_id"]
				tmp["result"] = result
				tmp["resultf"] = resultf
				delete(tmp, "_id")
				tmparr := []map[string]interface{}{
					map[string]interface{}{
						"_id": id,
					},
					map[string]interface{}{"$set": tmp},
				}
				e.RWMutex.Lock()
				e.ResultArr = append(e.ResultArr, tmparr)
				e.RWMutex.Unlock()
			}
		} else { //测试结果~结果追踪
			delete(tmp, "_id")
			delete(tmp, "fieldall")
			if len(j.BlockPackage) > 0 { //分包详情
				if len(j.BlockPackage) > 10 {
					tmp["epackage"] = "分包异常"
				} else {
					bs, _ := json.Marshal(j.BlockPackage)
					tmp["epackage"] = string(bs)
				}
			}
			tmp["result"] = result
			//tmp["resultf"] = resultf
			//_,err :=db.Mgo.Get().DB("zhengkun").C("result_data").Upsert(`{"_id":"`+_id+`"}`,map[string]interface{}{"$set": tmp})
			//log.Debug("save:",err)
			b := db.Mgo.Update(e.TaskInfo.TestColl, `{"_id":"`+_id+`"}`, map[string]interface{}{"$set": tmp}, true, false)
			if !b {
				log.Debug(e.TaskInfo.TestColl, _id)
			}
		}
	}, func(err interface{}) {
		log.Debug("AnalysisSaveResult err", err)
	})
}

// 获取抽取结果map[string][]interface{},lua脚本使用
func GetResultMapForLua(j *ju.Job) map[string][]map[string]interface{} {
	defer qu.Catch()
	result := map[string][]map[string]interface{}{}
	for key, val := range j.Result {
		if result[key] == nil {
			result[key] = []map[string]interface{}{}
		}
		for _, v := range val {
			tmp := map[string]interface{}{
				"field":     v.Field,
				"code":      v.Code,
				"ruletext":  v.RuleText,
				"value":     v.Value,
				"type":      v.Type,
				"matchtype": v.MatchType,
				"extfrom":   v.ExtFrom,
			}
			result[key] = append(result[key], tmp)
		}
	}
	return result
}

// 抽取日志
func AddExtLog(ftype, sid string, before interface{}, extinfo interface{}, v *RegLuaInfo, t *TaskInfo) {
	defer qu.Catch()
	if !t.IsEtxLog {
		return
	}
	logdata := map[string]interface{}{
		"code":       qu.If(v.Code == "", "kv", v.Code),
		"name":       v.Name,
		"type":       ftype,
		"ruletext":   v.RuleText,
		"islua":      v.IsLua,
		"field":      v.Field,
		"version":    t.Version,
		"taskname":   t.Name,
		"before":     before,
		"extinfo":    extinfo,
		"sid":        sid,
		"comeintime": time.Now().Unix(),
	}
	lock.Lock()
	ExtLogs[t] = append(ExtLogs[t], logdata)
	lock.Unlock()
}
func BeforeAddClearFnLog(ftype, name, sid, before, matchtype string, ext *ju.ExtField, e *ExtractTask) {
	exts := []map[string]interface{}{}
	exts = append(exts, map[string]interface{}{
		"field":     ext.Field,
		"code":      ext.Code,
		"type":      ftype,
		"matchtype": matchtype,
		"extfrom":   ext.ExtFrom,
		"value":     ext.Value,
	})
	extinfo := map[string]interface{}{
		ext.Field: exts,
	}
	AddClearFnLog(ftype, sid, before, extinfo, ext.Code, name, ext.Field, e.TaskInfo)
}
func AddClearFnLog(ftype, sid string, before interface{}, extinfo interface{}, code, name, field string, t *TaskInfo) {
	defer qu.Catch()
	if !t.IsEtxLog {
		return
	}
	logdata := map[string]interface{}{
		"code":       code,
		"name":       name,
		"type":       ftype,
		"ruletext":   "",
		"islua":      false,
		"field":      field,
		"version":    t.Version,
		"taskname":   t.Name,
		"before":     before,
		"extinfo":    extinfo,
		"sid":        sid,
		"comeintime": time.Now().Unix(),
	}
	lock.Lock()
	ExtLogs[t] = append(ExtLogs[t], logdata)
	lock.Unlock()
}

// 保存抽取日志
func SaveExtLog() {
	defer qu.Catch()
	tmpLogs := map[*TaskInfo][]map[string]interface{}{}
	lock.Lock()
	tmpLogs = ExtLogs
	ExtLogs = map[*TaskInfo][]map[string]interface{}{}
	lock.Unlock()
	for k, v := range tmpLogs {
		if len(v) < saveLimit {
			db.Mgo.SaveBulk(k.TrackColl, v...)
		} else {
			for {
				if len(v) > saveLimit {
					tmp := v[:saveLimit]
					db.Mgo.SaveBulk(k.TrackColl, tmp...)
					v = v[saveLimit:]
				} else {
					db.Mgo.SaveBulk(k.TrackColl, v...)
					break
				}
			}
		}
	}
	time.AfterFunc(10*time.Second, SaveExtLog)
}

// 保存其他
// kv、表格、块上的标签凡是新的标签都入库
// val  type   times   firstid  createtime 判定field
func otherNeedSave(j *ju.Job, result map[string][]*ju.ExtField, e *ExtractTask) {
	now := time.Now().Unix()
	coll := e.TaskInfo.TestColl
	if coll == "" {
		coll = "extract_tag_result"
	} else {
		coll += "_tag"
	}
	datas := []map[string]interface{}{}
	kv := map[string]int{}
	for _, v := range j.Block {
		//
		for _, vv := range []*ju.JobKv{v.ColonKV, v.TableKV, v.SpaceKV} {
			if vv == nil || vv.KvTags == nil {
				continue
			}
			for kkk, vvv := range vv.KvTags {
				for _, vvvv := range vvv {
					if vvvv.IsInvalid {
						kv[kkk] = kv[kkk] + 1
						break
					}
				}
			}
		}
		for _, vv := range v.NotClassifyTitles {
			datas = append(datas, map[string]interface{}{
				"val":        vv,
				"times":      0,
				"type":       "block",
				"firstid":    j.SourceMid,
				"createtime": now,
			})
			if len(datas) == saveLimit {
				db.Mgo.SaveBulk(coll, datas...)
				datas = []map[string]interface{}{}
			}
		}
	}
	for k, v := range kv {
		datas = append(datas, map[string]interface{}{
			"val":        k,
			"times":      v,
			"type":       "kv",
			"firstid":    j.SourceMid,
			"createtime": now,
		})
		if len(datas) == saveLimit {
			db.Mgo.SaveBulk(coll, datas...)
			datas = []map[string]interface{}{}
		}
	}
	if len(datas) > 0 {
		db.Mgo.SaveBulk(coll, datas...)
	}
}

func rangeBlockToJson(j *ju.Block, tmpblock ju.TmpBlock) (b *ju.TmpBlock) {
	if j == nil {
		return nil
	}
	if len(j.Block) > 0 {
		for i, v := range j.Block {
			rangetmp := new(ju.TmpBlock)
			vb, _ := json.Marshal(v)
			json.Unmarshal(vb, &rangetmp)
			tmpblock.Block[i] = rangeBlockToJson(v, *rangetmp)
		}
	}
	if j.ColonKV != nil {
		cb, _ := json.Marshal(j.ColonKV)
		tmpblock.ColonKV = string(cb)
	}
	if j.SpaceKV != nil {
		sb, _ := json.Marshal(j.SpaceKV)
		tmpblock.SpaceKV = string(sb)
	}
	if j.TableKV != nil {
		tb, _ := json.Marshal(j.TableKV)
		tmpblock.TableKV = string(tb)
	}
	return &tmpblock
}
