package task

import (
	//"classification"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"qfw/util"
	"qfw/util/elastic"
	"regexp"
	"strings"
	"time"
	"tools"
	u "util"

	"github.com/donnie4w/go-logger/logger"
	lujson "github.com/yuin/gopher-json"
	"github.com/yuin/gopher-lua"

	//	"gopkg.in/mgo.v2/bson"
	"go.mongodb.org/mongo-driver/bson/primitive"
)

type LuaScript struct {
	L    *lua.LState
	Name string
	File string
}

var s map[string]*LuaScript

// 加载lua
func LoadLua() {
	filepath.Walk("./lua", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			log.Println(err)
			return err
		}
		if info.IsDir() || !strings.HasSuffix(info.Name(), ".lua") {
			return nil
		}
		bs, err := ioutil.ReadFile(path)
		if err != nil {
			log.Println(err)
			return err
		}
		script := NewLuaScript(info.Name(), string(bs)) //初始化腳本
		if script != nil {
			if s == nil {
				s = make(map[string]*LuaScript)
			}
			name := strings.Trim(info.Name(), ".lua")
			s[name] = script
		}
		return nil
	})
}

func NewLuaScript(name, luafile string) (ls *LuaScript) {
	defer util.Catch()
	options := lua.Options{
		RegistrySize:        256 * 20,
		CallStackSize:       256,
		IncludeGoStackTrace: false,
	}
	ls = &LuaScript{}
	ls.L = lua.NewState(options)
	ls.Name = name
	ls.File = luafile
	ls.L.PreloadModule("json", lujson.Loader)
	defer ls.L.Close()
	if err := ls.L.DoString(luafile); err != nil {
		panic("加载" + name + "脚本错误：" + err.Error())
		return nil
	}
	return ls
}

// lua处理数据
func Dealdata(srcript *LuaScript, data map[string]interface{}, field string) map[string]interface{} {
	tab := MapToLuaTable(srcript.L, data)
	if err := srcript.L.CallByParam(lua.P{
		Fn:      srcript.L.GetGlobal("LuaDealdata"),
		NRet:    1,
		Protect: true,
	}, tab, lua.LString(field)); err != nil {
		panic(err)
	}
	ret := srcript.L.Get(-1)
	srcript.L.Pop(1)
	finishdata := LuaTableToMap(ret.(*lua.LTable))
	//	indata := []interface{}{strings.Join(arrdata, "")}
	return finishdata

}

func MapToLuaTable(l *lua.LState, obj map[string]interface{}) *lua.LTable {
	tab := l.NewTable()
	for k, v := range obj {
		if val, ok := v.(string); ok {
			tab.RawSet(lua.LString(k), lua.LString(val))
		} else if val, ok := v.(int64); ok {
			tab.RawSet(lua.LString(k), lua.LNumber(val))
		} else if val, ok := v.(int32); ok {
			tab.RawSet(lua.LString(k), lua.LNumber(val))
		} else if val, ok := v.(float64); ok {
			tab.RawSet(lua.LString(k), lua.LNumber(val))
		} else if val, ok := v.(float32); ok {
			tab.RawSet(lua.LString(k), lua.LNumber(val))
		} else if val, ok := v.(bool); ok {
			tab.RawSet(lua.LString(k), lua.LBool(val))
		} else if val, ok := v.(map[string]interface{}); ok {
			tab.RawSet(lua.LString(k), MapToLuaTable(l, val))
		} else if val, ok := v.([]interface{}); ok {
			bs, _ := json.Marshal(val)
			tab.RawSet(lua.LString(k), lua.LString(string(bs)))
		} else if val, ok := v.(primitive.M); ok {
			tab.RawSet(lua.LString(k), MapToLuaTable(l, val))
		} else if val, ok := v.(primitive.A); ok {
			bs, _ := json.Marshal(val)
			tab.RawSet(lua.LString(k), lua.LString(string(bs)))
		}
	}
	return tab
}
func LuaTableToMap(param *lua.LTable) map[string]interface{} {
	tmp := map[string]interface{}{}
	param.ForEach(func(key, val lua.LValue) {
		k := fmt.Sprint(key)
		if v, ok := val.(lua.LString); ok {
			tmp[k] = string(v)
		} else if v, ok := val.(*lua.LTable); ok {
			tmp[k] = LuaTableToMap(v)
		} else if v, ok := val.(*lua.LBool); ok {
			if v.String() == "true" {
				tmp[k] = true
			} else {
				tmp[k] = false
			}
		} else {
			tmp[k] = v
		}
	})
	return tmp
}
func maptoarr(datamap map[string]interface{}) (dataarr []string) {
	// for i := 0; i < len(datamap); i++ {
	// 	//istr := strconv.Itoa(i + 1)
	// 	istr := fmt.Sprint(i)
	// 	v, _ := datamap[istr].(string)
	// 	dataarr = append(dataarr, v)
	// }
	for _, v := range datamap {
		dataarr = append(dataarr, v.(string))
	}
	return dataarr
}

// NewAnalyTask 初始化任务mgo配置信息,共用加载任务
func NewAnalyTask(_id, s_mgourl, s_mgodb, s_coll string, i_poolsize int) (res bool, task *TTask, msg string) {
	defer tools.Catch()
	task = TaskMap[_id] //从加载好的任务集合中取单个任务
	if task != nil {
		task.WordCount = map[string]map[string]int{}
		task.FlagQuit = make(chan bool, 1) //任务结束控制
		bcon := true
		if s_mgourl == "" {
			s_mgourl = task.S_mgourl
			s_mgodb = task.S_mgodb
			s_coll = task.S_coll
			i_poolsize = task.I_poolsize
		}
		if s_coll == "" {
			msg = "执行规则表不能为空"
			bcon = false
			return
		}
		if s_mgourl != "" {
			if s_mgodb != "" && i_poolsize > 0 {
				task.MgoTask.DbName = s_mgodb
				task.MgoTask.MongodbAddr = s_mgourl
				task.MgoTask.Size = i_poolsize
			} else {
				bcon = false
				msg = "数据库配置错误，不能继续..."
			}
		} else {
			task.MgoTask = tools.MgoClass
		}
		if bcon {
			task.S_collection = s_coll
			task.LastId = task.S_startid
			if s_mgourl != "" {
				//初始化数据库连接
				if task.Dbtype != "" {
					task.MgoTask.UserName = tools.DbInfo[task.Dbtype][0]
					task.MgoTask.Password = tools.DbInfo[task.Dbtype][1]
				}
				task.MgoTask.InitPool()
			}
			res = true
		}
	} else {
		msg = "没有查询到任务.."
	}
	return
}

// NewClassificationRun 共用识别过程
func NewClassificationRun(tt *TTask, tmp map[string]interface{}) *tools.SortMap {
	SMap := tools.NewSortMap()
	//tmpSavefield := map[string]interface{}{}
	fflag := map[string][]string{} //标志父类从属关系
	rulval := map[string]string{}  //存储过滤记录
	class := tt.Class              //获取任务中的多个分类
	if class != nil && len(class) > 0 {
		/**
		一级分类匹配到title，二级标题分类成功匹配到招标或者竞谈，并且detail中含有合同金额等其他字段，
		那么二级分类应该数据合同，一级分类应该数据其他
		*/
		var other bool
		for _, c := range class {
			//预处理lua
			s_fields := c.S_fields                              //识别字段  o_jy.a_key.key  detail,title
			for _, ftmp := range strings.Split(s_fields, ",") { //先将所有o_jy.a_key.key类型字段的值处理到tmp中
				fieldarr := strings.Split(ftmp, ".")
				if len(fieldarr) > 1 {
					key := fieldarr[0] //字段名称
					field := fieldarr[len(fieldarr)-1]
					if tmp[key] == nil { //判断是否有该字段的值
						continue
					}
					rMap := map[string]interface{}{}
					if tMap, ok := tmp[key].(map[string]interface{}); ok {
						rMap = tMap
					} else if sMap, ok := tmp[key].(string); ok {
						json.Unmarshal([]byte(sMap), &rMap)
						logger.Warn("Projectinfo Type Is Wrong:", tmp["_id"])
					}
					if len(rMap) > 0 {
						if s != nil && s[ftmp] != nil {
							newscript := NewLuaScript(s[ftmp].Name, s[ftmp].File) //newlua
							datamap := Dealdata(newscript, rMap, field)           //处理数据
							dataarr := maptoarr(datamap)
							if len(dataarr) == 0 {
								continue
							}
							tmp[ftmp] = dataarr
						}
					}
				}
			}
			cid := c.Cid     //类id
			s_pid := c.S_pid //父类id
			//s_fields := c.S_fields     //识别字段
			savefield := c.S_savefield //保存字段,toptype
			// if tt.I_multiclass == 0 {  //单分类临时记录保存字段
			// 	tmpSavefield[savefield] = ""
			// }
			//同一个字段，例如toptype有值了就跳过
			if savefield != "" && tt.I_multiclass == 0 && SMap.Map[savefield] != nil {
				//同一个字段，只要一个类识别...
				continue
			}
			rule := c.Rule //取规则
			fields := strings.Split(s_fields, ",")
		L:
			for _, f := range fields { //f:每一个识别字段
				val := rulval[f]
				if val == "" { //无过滤记录
					strArr, ok := tmp[f].([]string)
					if ok {
						//strArr := tools.ObjArrToStringArr(strArr)
						val = strings.Join(strArr, "")
					} else if strings.Contains(f, ".") {
						//有.，说明是个二级字段,比如gov_classify.zc_code
						ffields := strings.Split(f, ".")
						ffValue := tmp[ffields[0]]
						if fffValue, ok2 := ffValue.(map[string]interface{}); ok2 {
							key := ffields[1]
							if ffffValue, ok3 := fffValue[key].(string); ok3 {
								val = ffffValue
							}
						}
					} else {
						val = util.ObjToString(tmp[f]) //取识别字段内容
					}
					val = PreFilter(val, tt.Task_PreRule) //任务的前置过滤
					rulval[f] = val                       //整个任务仅过滤一次，将其存储
				}

				if val != "" {
					//1.分类的前置过滤，不符合直接跳过当前循环
					val = PreFilter(val, c.Class_PreRule)
					if val == "" {
						continue
					}
					//2.循环当前类规则
					for _, r := range rule {
						//只要此类的父类不为空
						if s_pid != "" {
							ru_s_pid := r.S_pid                      //当前规则的父规则id,57a02d4fd368081fac39eaf2,57a18dd5d368081d70e185d0,5a4d99b7e138236b380378be
							class_s_pid := strings.Split(s_pid, ",") //当前规则所属类的父类集合
							bcontinue := true
						L1:
							for _, c_s_pid := range class_s_pid {
								r_ids := fflag[c_s_pid]
								if r_ids != nil {
									for _, ids1 := range r_ids {
										if strings.Index(ru_s_pid, ids1) > -1 {
											bcontinue = false
											break L1
										}
									}
								}
							}
							if bcontinue { //没有找到父类的识别，跳过此规则
								continue
							}
						}
						ruval := val //避免不同class的rule串行过滤
						if len(r.Rule_PreRule) > 0 {
							ruval = PreFilter(ruval, r.Rule_PreRule) //rule的前置过滤
						}
						if ruval == "" {
							continue
						}
						//开始识别
						i_rule := r.Reg //toptype=结果 的所有规则
						//util.Debug("text---", ruval, "rule---", i_rule)
						b, rulearr := DFAAnalyRules(ruval, i_rule)
						//util.Debug("===============", b, f, rulearr, r.S_name)
						if b {
							//title、channel二级分类中标处理
							if r.S_name == "中选" && len(r.DetailReg) > 0 {
								detail := util.ObjToString(tmp["detail"])
								if len(r.NotReg) > 0 { //排除规则
									dnrb, _ := DFAAnalyRules(detail, r.NotReg)
									if dnrb { //排除规则匹配成功，匹配下一条
										continue
									}
								}
								//util.Debug("detail---", detail)
								drb, _ := DFAAnalyRules(detail, r.DetailReg)
								//util.Debug("-----", drb)
								if drb {
									SMap.AddKey(savefield, "中标")
								} else {
									SMap.AddKey(savefield, "成交")
								}
								break L
							}

							//一级分类是招标，二级分类匹配到招标
							if f == "title" && other && savefield == "subtype" {
								if (r.S_name == "招标" || r.S_name == "竞谈") && len(r.DetailReg) > 0 { //title、channel二级分类中标处理
									detail := util.ObjToString(tmp["detail"])
									if len(r.NotReg) > 0 { //排除规则
										dnrb, _ := DFAAnalyRules(detail, r.NotReg)
										if dnrb { //排除规则匹配成功，匹配下一条
											continue
										}
									}
									//util.Debug("detail---", detail)
									drb, _ := DFAAnalyRules(detail, r.DetailReg)
									//util.Debug("-----", drb)
									if drb {
										SMap.AddKey(savefield, "合同")
										//SMap.RemoveKey("toptype")
										SMap.AddKey("toptype", "其它")
									} else {
										SMap.AddKey(savefield, r.S_name)
									}

									break L
								}
							}

							r_id := r.Rid
							if fflag[cid] != nil {
								arr1 := fflag[cid]
								arr1 = append(arr1, r_id)
								fflag[cid] = arr1
							} else {
								fflag[cid] = []string{r_id} //第一个规则中的_id
							}
							//57a02bc0d368081fac39eaef =[57a02cdcd368081fac39eaf1]
							s_name := r.S_name
							//词频统计
							if tt.I_wordcount == 1 && len(rulearr) > 0 {
								tt.WcLock.Lock()
								map1 := tt.WordCount[s_name]
								if map1 == nil {
									map1 = map[string]int{}
								}
								for _, rulekey := range rulearr {
									map1[rulekey]++
								}
								tt.WordCount[s_name] = map1
								tt.WcLock.Unlock()
							}
							if savefield != "" {
								s_code := r.S_code
								if tt.I_multiclass == 0 { //单分类
									if tt.I_savetype == 1 { //存储属性
										SMap.AddKey(savefield, s_name)
										//通过一级分类title分类出，toptype= "招标"时，
										//再次针对二级分类招标的title匹配配置的detail字段，出现合同金额应该属于其他
										if savefield == "toptype" && f == "title" && s_name == "招标" {
											other = true
										}
									} else {
										SMap.AddKey(savefield, s_code)
									}
									break L //停止其他字段识别
								} else { //多分类
									sf := []string{}
									b := false
									if SMap.Map[savefield] != nil {
										b = true
										sf = SMap.Map[savefield].([]string)
									}
									if tt.I_savetype == 1 { //
										sf = append(sf, s_name)
									} else {
										sf = append(sf, s_code)
									}
									if b {
										SMap.Map[savefield] = sf
									} else {
										SMap.AddKey(savefield, sf)
									}
								}
							}
						}
					}
				}
			}
			//设置默认值
			//log.Println("smp", SMap.Map, SMap.Keys)
			if savefield != "" && SMap.Map[savefield] == nil && c.S_default != "" {
				if tt.I_multiclass == 0 { //单分类
					//if savefield == "buyerclass" && util.ObjToString(tmp["buyer"]) == "" { //buyer不存在时,无buyerclass字段
					//	return SMap
					//}
					SMap.AddKey(savefield, c.S_default)
				} else { //多分类
					SMap.AddKey(savefield, []string{c.S_default})
				}
			}
		}
	}

	//多分类去重
	if tt.I_multiclass != 0 {
		for _, k := range SMap.Keys {
			repeatMap := make(map[string]string)
			tmpArr := make([]string, 0)
			if arr, ok := SMap.Map[k].([]string); ok {
				for _, v := range arr {
					if repeatMap[v] != "has" {
						tmpArr = append(tmpArr, v)
						repeatMap[v] = "has"
					}
				}
				SMap.Map[k] = tmpArr
			}
		}
	} /* else { //单分类
		for k, v := range tmpSavefield {
			if k != "" && SMap.Map[k] == nil {
				SMap.Map[k] = v
			}
		}
	}*/
	return SMap
}

func ReSub(tt *TTask, tmp map[string]interface{}, top string) *tools.SortMap {
	SMap := tools.NewSortMap()
	//tmpSavefield := map[string]interface{}{}
	//fflag := map[string][]string{} //标志父类从属关系
	rulval := map[string]string{} //存储过滤记录
	class := tt.Class             //获取任务中的多个分类

	var top_class_ids []string //一级分类的规则ID
	if len(class) > 0 {
		for _, cla := range class {
			rules := cla.Rule
			for _, rule := range rules {
				if rule.S_name == top {
					top_class_ids = append(top_class_ids, rule.Rid)
				}
			}

		}
	}

	if class != nil && len(class) > 0 {
		for _, c := range class {
			//预处理lua
			s_fields := c.S_fields                              //识别字段  o_jy.a_key.key  detail,title
			for _, ftmp := range strings.Split(s_fields, ",") { //先将所有o_jy.a_key.key类型字段的值处理到tmp中
				fieldarr := strings.Split(ftmp, ".")
				if len(fieldarr) > 1 {
					key := fieldarr[0] //字段名称
					field := fieldarr[len(fieldarr)-1]
					if tmp[key] == nil { //判断是否有该字段的值
						continue
					}
					rMap := map[string]interface{}{}
					if tMap, ok := tmp[key].(map[string]interface{}); ok {
						rMap = tMap
					} else if sMap, ok := tmp[key].(string); ok {
						json.Unmarshal([]byte(sMap), &rMap)
						logger.Warn("Projectinfo Type Is Wrong:", tmp["_id"])
					}
					if len(rMap) > 0 {
						if s != nil && s[ftmp] != nil {
							newscript := NewLuaScript(s[ftmp].Name, s[ftmp].File) //newlua
							datamap := Dealdata(newscript, rMap, field)           //处理数据
							dataarr := maptoarr(datamap)
							if len(dataarr) == 0 {
								continue
							}
							tmp[ftmp] = dataarr
						}
					}
				}
			}
			//cid := c.Cid     //类id
			s_pid := c.S_pid //父类id

			if s_pid == "" {
				continue
			}
			savefield := c.S_savefield //保存字段,toptype
			// if tt.I_multiclass == 0 {  //单分类临时记录保存字段
			// 	tmpSavefield[savefield] = ""
			// }
			//同一个字段，例如toptype有值了就跳过
			if savefield != "" && tt.I_multiclass == 0 && SMap.Map[savefield] != nil {
				//同一个字段，只要一个类识别...
				continue
			}
			rule := c.Rule //取规则
			fields := strings.Split(s_fields, ",")
		L:
			for _, f := range fields { //f:每一个识别字段
				val := rulval[f]
				if val == "" { //无过滤记录
					strArr, ok := tmp[f].([]string)
					if ok {
						//strArr := tools.ObjArrToStringArr(strArr)
						val = strings.Join(strArr, "")
					} else {
						val = util.ObjToString(tmp[f]) //取识别内容
					}
					val = PreFilter(val, tt.Task_PreRule) //任务的前置过滤
					rulval[f] = val                       //整个任务仅过滤一次，将其存储
				}

				if val != "" {
					//1.分类的前置过滤，不符合直接跳过当前循环
					val = PreFilter(val, c.Class_PreRule)
					if val == "" {
						continue
					}
					//2.循环当前类规则
					for _, r := range rule {
						//只要此类的父类不为空
						if s_pid != "" {
							ru_s_pid := r.S_pid //当前规则的父规则id,57a02d4fd368081fac39eaf2,57a18dd5d368081d70e185d0,5a4d99b7e138236b380378be
							//class_s_pid := strings.Split(s_pid, ",") //当前规则所属类的父类集合
							bcontinue := true
						L1:
							for _, ids1 := range top_class_ids {
								if strings.Index(ru_s_pid, ids1) > -1 {
									bcontinue = false
									break L1
								}
							}
							if bcontinue { //没有找到父类的识别，跳过此规则
								continue
							}
						}
						ruval := val //避免不同class的rule串行过滤
						if len(r.Rule_PreRule) > 0 {
							ruval = PreFilter(ruval, r.Rule_PreRule) //rule的前置过滤
						}
						if ruval == "" {
							continue
						}
						//开始识别
						i_rule := r.Reg //toptype=结果 的所有规则
						//util.Debug("text---", ruval, "rule---", i_rule)
						b, rulearr := DFAAnalyRules(ruval, i_rule)
						//util.Debug("===============", b, f, rulearr, r.S_name)
						if b {
							detail := util.ObjToString(tmp["detail"])
							if len(r.NotReg) > 0 { //排除规则
								dnrb, _ := DFAAnalyRules(detail, r.NotReg)
								if dnrb { //排除规则匹配成功，匹配下一条
									continue
								}
							}
						}

						if b {
							//title、channel二级分类中标处理
							if r.S_name == "中选" && len(r.DetailReg) > 0 {
								detail := util.ObjToString(tmp["detail"])
								if len(r.NotReg) > 0 { //排除规则
									dnrb, _ := DFAAnalyRules(detail, r.NotReg)
									if dnrb { //排除规则匹配成功，匹配下一条
										continue
									}
								}
								//util.Debug("detail---", detail)
								drb, _ := DFAAnalyRules(detail, r.DetailReg)
								//util.Debug("-----", drb)
								if drb {
									SMap.AddKey(savefield, "中标")
								} else {
									SMap.AddKey(savefield, "成交")
								}
								break L
							}

							//57a02bc0d368081fac39eaef =[57a02cdcd368081fac39eaf1]
							s_name := r.S_name
							//词频统计
							if tt.I_wordcount == 1 && len(rulearr) > 0 {
								tt.WcLock.Lock()
								map1 := tt.WordCount[s_name]
								if map1 == nil {
									map1 = map[string]int{}
								}
								for _, rulekey := range rulearr {
									map1[rulekey]++
								}
								tt.WordCount[s_name] = map1
								tt.WcLock.Unlock()
							}
							if savefield != "" {
								s_code := r.S_code
								if tt.I_multiclass == 0 { //单分类
									if tt.I_savetype == 1 { //存储属性
										SMap.AddKey(savefield, s_name)
									} else {
										SMap.AddKey(savefield, s_code)
									}
									break L //停止其他字段识别
								} else { //多分类
									sf := []string{}
									b := false
									if SMap.Map[savefield] != nil {
										b = true
										sf = SMap.Map[savefield].([]string)
									}
									if tt.I_savetype == 1 { //
										sf = append(sf, s_name)
									} else {
										sf = append(sf, s_code)
									}
									if b {
										SMap.Map[savefield] = sf
									} else {
										SMap.AddKey(savefield, sf)
									}
								}
							}
						}
					}
				}
			}
			//设置默认值
			//log.Println("smp", SMap.Map, SMap.Keys)
			if savefield != "" && SMap.Map[savefield] == nil && c.S_default != "" {
				if tt.I_multiclass == 0 { //单分类
					//if savefield == "buyerclass" && util.ObjToString(tmp["buyer"]) == "" { //buyer不存在时,无buyerclass字段
					//	return SMap
					//}
					SMap.AddKey(savefield, c.S_default)
				} else { //多分类
					SMap.AddKey(savefield, []string{c.S_default})
				}
			}
		}
	}

	//多分类去重
	if tt.I_multiclass != 0 {
		for _, k := range SMap.Keys {
			repeatMap := make(map[string]string)
			tmpArr := make([]string, 0)
			if arr, ok := SMap.Map[k].([]string); ok {
				for _, v := range arr {
					if repeatMap[v] != "has" {
						tmpArr = append(tmpArr, v)
						repeatMap[v] = "has"
					}
				}
				SMap.Map[k] = tmpArr
			}
		}
	} /* else { //单分类
		for k, v := range tmpSavefield {
			if k != "" && SMap.Map[k] == nil {
				SMap.Map[k] = v
			}
		}
	}*/
	return SMap
}

// 标签识别过程
func TagClassificationRun(tt *TTask, tmp map[string]interface{}) *tools.SortMap {
	SMap := tools.NewSortMap()
	rulval := map[string]string{} //存储过滤记录
	resultArr := []string{}
	resultMap := map[string]bool{}
	savefield := ""   //标签任务的特殊性，指定字段保存，不能通过不同分类，指定不同保存的字段
	class := tt.Class //获取任务中的多个分类
	if class != nil && len(class) > 0 {
		for _, c := range class {
			s_fields := c.S_fields    //识别字段  o_jy.a_key.key  detail,title
			savefield = c.S_savefield //保存字段
			rule := c.Rule            //取规则
			fields := strings.Split(s_fields, ",")
			for _, f := range fields { //f:每一个识别字段
				val := rulval[f]
				if val == "" { //无过滤记录
					strArr, ok := tmp[f].([]string)
					if ok {
						//strArr := tools.ObjArrToStringArr(strArr)
						val = strings.Join(strArr, "")
					} else {
						val = util.ObjToString(tmp[f]) //取识别内容
					}
					val = PreFilter(val, tt.Task_PreRule) //任务的前置过滤
					rulval[f] = val                       //整个任务仅过滤一次，将其存储
				}
				if val != "" {
					val = PreFilter(val, c.Class_PreRule) //分类的前置过滤
					if val == "" {
						continue
					}
					for _, r := range rule {
						if len(r.Rule_PreRule) > 0 {
							val = PreFilter(val, r.Rule_PreRule) //rule的前置过滤
						}
						if val == "" {
							continue
						}
						//开始识别
						//util.Debug("file---", f, "text---", val, "rule---", r.Reg)
						rulearr := TagDFAAnalyRules(val, r.Reg)
						//util.Debug("===============", rulearr, r.S_name)
						for _, rule := range rulearr {
							if !resultMap[rule] {
								resultArr = append(resultArr, rule)
								resultMap[rule] = true
							}
						}
					}
				}
			}
		}
	}
	if len(resultArr) > 0 {
		SMap.AddKey(savefield, strings.Join(resultArr, ","))
	}
	return SMap
}

// 附件识别过程
func FileClassificationRun(tt *TTask, tmp map[string]interface{}) *tools.SortMap {
	SMap := tools.NewSortMap()
	if tmp["projectinfo"] == nil { //没有附件字段直接return
		return SMap
	}
	projectinfo, _ := tmp["projectinfo"].(map[string]interface{})
	attachments, _ := projectinfo["attachments"].(map[string]interface{})
	class := tt.Class              //获取任务中的多个分类
	fflag := map[string][]string{} //标志父类从属关系
	if class != nil && len(class) > 0 {
		for _, c := range class {
			s_fields := c.S_fields     //识别字段 content,filename
			cid := c.Cid               //类id
			s_pid := c.S_pid           //父类id
			savefield := c.S_savefield //保存字段

			rule := c.Rule //取规则
			fields := strings.Split(s_fields, ",")
			for n, files := range attachments {
				tmpsavefield := savefield + "_" + n
				if tmpsavefield != "" && tt.I_multiclass == 0 && SMap.Map[tmpsavefield] != nil {
					//同一个字段，只要一个类识别...
					continue
				}
				file, ok := files.(map[string]interface{})
				if !ok {
					log.Println("附件数据错误")
					continue
				}
			L:
				for _, f := range fields { //f:每一个识别字段
					val, _ := file[f].(string)
					val = PreFilter(val, tt.Task_PreRule) //任务的前置过滤
					if val != "" {
						val = PreFilter(val, c.Class_PreRule) //分类的前置过滤
						if val == "" {
							continue
						}
						for _, r := range rule {
							if s_pid != "" { //只要此类的父类不为空
								ru_s_pid := r.S_pid //规则的父类id
								class_s_pid := strings.Split(s_pid, ",")
								bcontinue := true
							L1:
								for _, c_s_pid := range class_s_pid {
									r_ids := fflag[c_s_pid]
									if r_ids != nil {
										for _, ids1 := range r_ids {
											if strings.Index(ru_s_pid, ids1) > -1 {
												bcontinue = false
												break L1
											}
										}
									}
								}
								if bcontinue { //没有找到父类的识别，跳过此规则
									continue
								}
							}
							ruval := val //避免不同class的rule串行过滤
							if len(r.Rule_PreRule) > 0 {
								ruval = PreFilter(ruval, r.Rule_PreRule) //rule的前置过滤
							}
							if ruval == "" {
								continue
							}
							//开始识别
							i_rule := r.Reg
							//log.Println("text---", ruval, "rule---", i_rule)
							b, rulearr := DFAAnalyRules(ruval, i_rule)
							//log.Println("===============", b, rulearr, r.S_name)
							if b {
								r_id := r.Rid
								if fflag[cid] != nil {
									arr1 := fflag[cid]
									arr1 = append(arr1, r_id)
									fflag[cid] = arr1
								} else {
									fflag[cid] = []string{r_id} //第一个规则中的_id
								}
								s_name := r.S_name
								if tt.I_wordcount == 1 && len(rulearr) > 0 {
									tt.WcLock.Lock()
									map1 := tt.WordCount[s_name]
									if map1 == nil {
										map1 = map[string]int{}
									}
									for _, rulekey := range rulearr {
										map1[rulekey]++
									}
									tt.WordCount[s_name] = map1
									tt.WcLock.Unlock()
								}
								if tmpsavefield != "" {
									s_code := r.S_code
									if tt.I_multiclass == 0 {
										if tt.I_savetype == 1 {
											SMap.AddKey(tmpsavefield, s_name)
										} else {
											SMap.AddKey(tmpsavefield, s_code)
										}
										break L //停止其他字段识别
									} else { //多分类
										sf := []string{}
										b := false
										if SMap.Map[tmpsavefield] != nil {
											b = true
											sf = SMap.Map[tmpsavefield].([]string)
										}
										if tt.I_savetype == 1 { //
											sf = append(sf, s_name)
										} else {
											sf = append(sf, s_code)
										}
										if b {
											SMap.Map[tmpsavefield] = sf
										} else {
											SMap.AddKey(tmpsavefield, sf)
										}
									}
								}
							}
						}
					}
				}
				//设置默认值
				//log.Println("smp", SMap.Map, SMap.Keys)
				if tmpsavefield != "" && SMap.Map[tmpsavefield] == nil && c.S_default != "" {
					if tt.I_multiclass == 0 {
						SMap.AddKey(tmpsavefield, c.S_default)
					} else {
						SMap.AddKey(tmpsavefield, []string{c.S_default})
					}
				}
			}
		}
	}
	//多分类去重
	if tt.I_multiclass != 0 {
		for _, k := range SMap.Keys {
			repeatMap := make(map[string]string)
			tmpArr := make([]string, 0)
			if arr, ok := SMap.Map[k].([]string); ok {
				for _, v := range arr {
					if repeatMap[v] != "has" {
						tmpArr = append(tmpArr, v)
						repeatMap[v] = "has"
					}
				}
				SMap.Map[k] = tmpArr
			}
		}
	}
	//修改projectinfo字段
	for n, files := range attachments {
		file, _ := files.(map[string]interface{})
		for _, key := range SMap.Keys {
			save_num := strings.Split(key, "_")
			if len(save_num) == 2 && save_num[1] == n {
				file[save_num[0]] = SMap.Map[key]
				SMap.RemoveKey(key) //只在附件信息内部保存分类信息
			}
		}
	}
	return SMap
}

var rc *regexp.Regexp

// 前置过滤
func PreFilter(text, taskPrerule string) string {
	defer tools.Catch()
	if len(taskPrerule) > 0 && text != "" {
		taskPreruleArr := strings.Split(taskPrerule, "\n")
		for _, r := range taskPreruleArr {
			rClearArr := strings.Split(r, "__")
			if len(rClearArr) == 2 {
				rc = regexp.MustCompile(rClearArr[0])
				text = rc.ReplaceAllString(text, rClearArr[1])
			}
		}
	}
	return text
}

// 存放测试的数据
var UPDATE = &TestList{
	Count: map[string][]int{},
}

func LoadUpdateTask(_id, s_mgourl, s_mgodb, s_coll, i_poolsize, s_esurl, s_esdb, s_escoll, s_startid, s_endid, s_query string, bcompare bool) (bs bool) {
	defer tools.Catch()
	//bres, tt, msg := AnalyTask(_id, s_mgourl, s_mgodb, s_coll, tools.IntAllDef(i_poolsize, 5))
	bres, tt, msg := NewAnalyTask(_id, s_mgourl, s_mgodb, s_coll, tools.IntAllDef(i_poolsize, 5))
	log.Println(msg)
	if bres && tt != nil {
		bs = true
		espool := make(chan bool, 4)
		go tools.Try(func() {
			var es *elastic.Elastic
			if s_esurl != "" && s_esdb != "" && s_escoll != "" {
				es = &elastic.Elastic{
					S_esurl: s_esurl,
					I_size:  15,
				}
				es.InitElasticSize()
			}
			defer func() {
				go func() {
					time.AfterFunc(30*time.Second, func() {
						UPDATE.Del(_id)
					})
				}()
				if es != nil {
					es.Close()
				}
			}()
			defer tools.Catch()
			//开始识别
			sess := tt.MgoTask.GetMgoConn()
			defer tt.MgoTask.DestoryMongoConn(sess)
			q := map[string]interface{}{}
			if s_query != "" {
				json.Unmarshal([]byte(strings.Replace(s_query, "'", "\"", -1)), &q)
			}
			if s_startid != "" {
				q["_id"] = map[string]interface{}{
					"$gte": u.StringTOBsonId(s_startid),
				}
			}
			if s_endid != "" {
				if q["_id"] != nil {
					q["_id"].(map[string]interface{})["$lte"] = u.StringTOBsonId(s_endid)
				} else {
					q["_id"] = map[string]interface{}{
						"$lte": u.StringTOBsonId(s_endid),
					}
				}
			}
			//log.Println(q)
			count := tt.MgoTask.Count(tt.S_collection, &q)
			if count == 0 {
				return
			}
			//放入数组池中，方便进度查询
			UPDATE.Put(_id, 0, count, 0)
			query := sess.DB(tt.S_mgodb).C(tt.S_collection).Find(&q).Iter()
			arr := [][]map[string]interface{}{}
			updateEs := []map[string]interface{}{}
			i := 0
			for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
				//按顺序识别
				var SMap = tools.NewSortMap()
				if util.IntAll(tmp["infoformat"]) == 2 { //此处增加特例
					SMap.AddKey("toptype", "拟建")
					SMap.AddKey("subtype", "拟建")
				} else if util.IntAll(tmp["infoformat"]) == 3 {
					SMap.AddKey("toptype", "产权")
					SMap.AddKey("subtype", "产权")
				} else {
					//SMap = ClassificationRun(tt, tmp)
					SMap = NewClassificationRun(tt, tmp)
					//一级分类时，符合结果中成交规则时
					if SMap.Map["toptype"] == "招标" && SMap.Map["subtype"] != "单一" {
						if _, ok := tmp["detail"]; ok {
							if u.ChargeDetailResult(util.ObjToString(tmp["detail"])) {
								SMap.Map["toptype"] = "结果"
								resa := ReSub(tt, tmp, "结果")
								subtype := resa.Map["subtype"]
								delete(SMap.Map, "subtype")
								SMap.Map["subtype"] = subtype
							}
						}
					}
				}
				bupdate := false
				//对比，是否保存
				if bcompare {
					for _, k := range SMap.Keys {
						if tmp[k] != nil {
							if util.ObjToString(SMap.Map[k]) != util.ObjToString(tmp[k]) {
								bupdate = true
								break
							}
						} else {
							bupdate = true
							break
						}
					}
				} else {
					bupdate = true
				}

				if bupdate {
					update := []map[string]interface{}{}
					tid := tmp["_id"]
					update = append(update, map[string]interface{}{
						"_id": tid,
					})
					res := SMap.Map
					if tt.S_attr != "" {
						res[tt.S_attr] = tt.AttrVal
					}
					update = append(update, map[string]interface{}{
						"$set": res,
					})
					if es != nil {
						esObj := tmp
						for rk, rv := range res {
							esObj[rk] = rv
						}
						updateEs = append(updateEs, esObj)
					}
					//保存到mgo库
					arr = append(arr, update)
					if len(arr) >= NN {
						tt.MgoTask.UpdateBulk(tt.S_collection, arr...)
						arr = [][]map[string]interface{}{}
						//应该更新到es库
						if es != nil {
							espool <- true
							tmpArr := updateEs
							go func() {
								defer func() {
									<-espool
								}()
								es.BulkSave(s_esdb, s_escoll, &tmpArr, false)
							}()
							updateEs = []map[string]interface{}{}
						}
					}
				}
				if i%100 == 0 {
					log.Println("进度", i, count)
				}
				UPDATE.Put(_id, i+1)
				tmp = make(map[string]interface{})
			}

			if len(arr) > 0 {
				tt.MgoTask.UpdateBulk(tt.S_collection, arr...)
				arr = nil
				//应该更新到es库
				if es != nil {
					espool <- true
					go func() {
						defer func() {
							<-espool
						}()
						es.BulkSave(s_esdb, s_escoll, &updateEs, false)
					}()
				}
			}
			log.Println("运行更新", tt.S_name, "OVER")
		})
	}
	return
}

// DFA识别规则
func DFAAnalyRules(text string, rules []interface{}) (bool, []string) {
	var arr []string
	//log.Println("len===", len(rules))
	for _, r := range rules {
		//log.Println("i--------------", i)
		ruleReg, ok := r.(*regexp.Regexp)
		if ok { //正则
			//log.Println("正则===", ruleReg)
			textArr := ruleReg.FindAllString(text, -1)
			if len(textArr) > 0 {
				regStr := []string{ruleReg.String()}
				return true, regStr
			}
		} else {
			rDFA, b := r.(*RuleDFA)
			//log.Println(j, "规则===", b, rDFA.Match, rDFA.MatchNum, rDFA.MisMatch, rDFA.MisMatchNum)
			if b { //规则DFA
				//b1, b2 := false, false
				b1, b2 := false, true
				var res []string
				//log.Println("res========", res, len(rDFA.MatchNum) == len(rDFA.Match), len(rDFA.MatchNum))
				if len(rDFA.MatchNum) == len(rDFA.Match) {
					for i, matchnum := range rDFA.MatchNum {
						if matchnum >= 1 {
							btmp, restmp := rDFA.Match[i].CheckSensitiveWord(text, matchnum)
							//log.Println("btmp====", btmp, restmp)
							if !btmp { //逗号隔开的每条规则不匹配，继续匹配下一条
								//log.Println("继续匹配")
								b2 = false
								break
							}
							res = append(res, restmp...)
						}
					}
				}
				if !b2 {
					continue
				}
				//走到这一步证明需要匹配的词正确个数满足要求，下面判断不需要匹配的词的情况
				mismatchnum := rDFA.MisMatchNum
				if mismatchnum >= 1 { //有排除词,排除词不应该出现在匹配的文本中
					b1, _ = rDFA.MisMatch.CheckSensitiveWord(text, mismatchnum)
				} else {
					b1 = false
				}
				if !b1 { //不要匹配的词满足情况，跳出
					return true, res
				} else {
					continue
				}
			}
		}
	}
	return false, arr
}

// 单独的标签识别规则
func TagDFAAnalyRules(text string, rules []interface{}) (res []string) {
	defer util.Catch()
	//util.Debug("len===", len(rules))
	for _, r := range rules {
		//util.Debug("j--------------", j)
		rDFA, b := r.(*RuleDFA)
		//util.Debug(j, "规则===", b, rDFA.Match, rDFA.MatchNum, rDFA.MisMatch, rDFA.MisMatchNum)
		if b { //规则DFA
			//util.Debug("res========", res, len(rDFA.MatchNum) == len(rDFA.Match), len(rDFA.MatchNum))
			if len(rDFA.MatchNum) == len(rDFA.Match) {
				for i, matchnum := range rDFA.MatchNum {
					if matchnum >= 1 {
						btmp, restmp := rDFA.Match[i].CheckSensitiveWord(text, matchnum)
						//util.Debug("btmp====", btmp, restmp)
						if !btmp { //逗号隔开的每条规则不匹配，继续匹配下一条
							//log.Println("继续匹配")
							break
						}
						res = append(res, restmp...)
					}
				}
			}
		}
	}
	return
}

// 满足所有情况的DFA识别规则
func DFAAnalyRules_All(text string, rules []interface{}, multiple bool) (bool, []string) {
	var res []string
	success := false
	util.Debug("len===", len(rules))
	for j, r := range rules {
		util.Debug("j--------------", j)
		ruleReg, ok := r.(*regexp.Regexp)
		if ok { //正则
			//log.Println("正则===", ruleReg)
			textArr := ruleReg.FindAllString(text, -1)
			if len(textArr) > 0 {
				regStr := []string{ruleReg.String()}
				return true, regStr
			}
		} else {
			rDFA, b := r.(*RuleDFA)
			util.Debug(j, "规则===", b, rDFA.Match, rDFA.MatchNum, rDFA.MisMatch, rDFA.MisMatchNum)
			if b { //规则DFA
				//b1, b2 := false, false
				b1, b2 := false, true
				//var res []string
				util.Debug("res========", res, len(rDFA.MatchNum) == len(rDFA.Match), len(rDFA.MatchNum))
				if len(rDFA.MatchNum) == len(rDFA.Match) {
					for i, matchnum := range rDFA.MatchNum {
						if matchnum >= 1 {
							btmp, restmp := rDFA.Match[i].CheckSensitiveWord(text, matchnum)
							util.Debug("btmp====", btmp, restmp)
							if !btmp { //逗号隔开的每条规则不匹配，继续匹配下一条
								//log.Println("继续匹配")
								b2 = false
								break
							}
							res = append(res, restmp...)
						}
					}
				}
				if !b2 {
					continue
				}
				//走到这一步证明需要匹配的词正确个数满足要求，下面判断不需要匹配的词的情况
				mismatchnum := rDFA.MisMatchNum
				if mismatchnum >= 1 { //有排除词,排除词不应该出现在匹配的文本中
					b1, _ = rDFA.MisMatch.CheckSensitiveWord(text, mismatchnum)
				} else {
					b1 = false
				}
				if !b1 { //到此，规则匹配成功
					if !multiple { //单个规则匹配
						success = true
						break
					} else if multiple && len(rules) == j+1 { //1、匹配成功且匹配单个；2、匹配多个到结束(multiple的实质作用目前仅限于标签任务)
						success = true
						break
					}
				} else {
					if !multiple { //非多规则匹配清res
						res = []string{}
					}
					continue
				}
			}
		}
	}
	return success, res
}

// 规则测试，只匹配一个，匹配一个成功后返回
func DFAAnalyRulesTest(text string, rules []interface{}) (bool, bool, [][]string, int) {
	arr := make([][]string, len(rules))
	for j, r := range rules {
		ruleReg, ok := r.(*regexp.Regexp)
		if ok { //正则
			textArr := ruleReg.FindAllString(text, -1)
			if len(textArr) > 0 {
				arr[j] = []string{ruleReg.String()}
				return true, true, arr, j //匹配一个结束
			} else {
				arr[j] = []string{}
			}

		} else {
			rDFA, b := r.(*RuleDFA)
			//util.Debug(j, "规则===", b, rDFA.Match, rDFA.MatchNum, rDFA.MisMatch, rDFA.MisMatchNum)
			if b { //规则DFA
				//1.先匹配排除规则
				flag := false
				if rDFA.MisMatchNum == 0 { //没有排除规则
					flag = true
				} else { //有排除规则
					btmp, restmp := rDFA.MisMatch.CheckSensitiveWord(text, rDFA.MisMatchNum)
					if btmp { //排除规则匹配成功,不在匹配规则
						arr[j] = restmp
						//arr = append(arr, restmp)
						continue
					} else {
						flag = true
					}
				}
				//2.再匹配规则
				if flag {
					isMatch := true
					res := []string{}
					if len(rDFA.MatchNum) == len(rDFA.Match) {
						for i, matchnum := range rDFA.MatchNum {
							if matchnum >= 1 {
								btmp, restmp := rDFA.Match[i].CheckSensitiveWord(text, matchnum)
								if !btmp {
									isMatch = false
								}
								res = append(res, restmp...)
							}
						}
					}
					arr[j] = res
					//arr = append(arr, res)
					if isMatch { //匹配成功
						return true, false, arr, j
					}
				}
			}
		}
	}
	return false, false, arr, -1
}

// 规则测试，返回所有匹配到的规则
func DFAAnalyRulesTest2(text string, rules []interface{}) (bool, bool, [][]string, int) {
	isok := false
	arr := make([][]string, len(rules))
	ok := false
	pos := -1
	for j, r := range rules {
		ruleReg, ok1 := r.(*regexp.Regexp)
		if ok1 { //正则
			textArr := ruleReg.FindAllString(text, -1)
			if len(textArr) > 0 {
				arr[j] = []string{ruleReg.String()}
				return true, true, arr, j //匹配一个结束
			} else {
				arr[j] = []string{}
			}
		} else {
			rDFA, b := r.(*RuleDFA)
			if b { //规则DFA
				//1.先匹配排除规则
				flag := false
				misMatchArr := []string{}
				if rDFA.MisMatchNum == 0 { //没有排除规则
					flag = true
				} else { //有排除规则
					btmp, restmp := rDFA.MisMatch.CheckSensitiveWord(text, rDFA.MisMatchNum)
					if btmp { //排除规则匹配成功,不在匹配规则
						misMatchArr = append(misMatchArr, restmp...)
					} else {
						flag = true
					}
				}
				//2.再匹配规则
				isMatch := true
				res := []string{}
				if len(rDFA.MatchNum) == len(rDFA.Match) {
					for i, matchnum := range rDFA.MatchNum {
						if matchnum >= 1 {
							btmp, restmp := rDFA.Match[i].CheckSensitiveWordTest(text, matchnum)
							if !btmp {
								isMatch = false
							}
							res = append(res, restmp...)
						}
					}
				}
				res = append(res, misMatchArr...)
				arr[j] = res
				if isMatch && flag && !isok { //匹配成功
					ok = true
					pos = j
					isok = true
					//return true, false, arr, j
				}
			}
		}
	}
	return ok, false, arr, pos
}
