package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	log "github.com/donnie4w/go-logger/logger"
	"github.com/tealeg/xlsx"
	"io"
	"os"
	es_elastic "qfw/common/src/gopkg.in/olivere/elastic.v1"
	qu "qfw/util"
	"qfw/util/elastic"
	"regexp"
	"strings"
	"sync"
	"unicode/utf8"
)

var tidyArr = []string{
	"365410",
	"365409",
	"40130401",
	"40130402",
	"62020105",
	"365411",
	"36540903",
	"400101",
	"40010101",
	"40010103",
	"4008030102",
	"400203",
	"400599",
	"40059901",
	"40059902",
	"4008049903",
	"4008049902",
	"4008022201",
	"40090101",
	"4009010101",
	"4009010102",
	"40090103",
	"40090104",
	"401001",
	"40100301",
	"40100302",
	"40100305",
	"40100401",
	"401002",
	"40100203",
	"401101",
	"40110206",
	"40110103",
	"40110212",
	"401104",
	"40110402",
	"40110405",
	"4011040401",
	"4011040402",
	"40110503",
	"40110504",
	"40120101",
	"401202",
	"40120201",
	"40120202",
	"40129901",
	"40130201",
	"401303",
	"40130301",
	"40130302",
	"40130303",
	"40130304",
	"401304",
	"4016120101",
	"40190603",
	"402002",
	"390904",
	"39090203",
	"391001",
	"39100101",
	"391002",
	"620101",
	"620201",
	"620202",
	"620303",
	"62030301",
	"62030302",
	"620401",
	"610102",
	"600103",
	"60010303",
	"60010304",
	"76020205",
	"400803",
	"402201",
	"480102",
	"4022020103",
}

var LdxArr_1 = []string{"365410", "365409", "40130401", "40130402", "62020105", "365411", "36540903", "4008030102", "4008049903", "4008049902", "40110206", "40110212", "40130301", "76020205", "4022020103", "480102", "480202", "40080301"}
var LdxArr_2 = []string{"4701", "4013", "61", "62", "8301", "40", "4127", "6101", "6202", "83", "4899", "8405", "7417"}
var LdxReg_2 *regexp.Regexp = regexp.MustCompile("(安防|监控|校园安全|视频监控|校园安防|校园监控|监控系统|安防监控|学校监控|标准化考场|标准化考点|网络安全)")
var LdxArr_3 = []string{"400101", "401906", "40010101", "402201", "40010103", "400203", "400599", "40059901", "40059902", "4008022201", "40090101", "4009010101", "4009010102", "40090103", "40090104", "401001", "40100301", "40100302", "40100305", "40100401", "401002", "40100203", "401101", "40110103", "401104", "40110402", "40110405", "4011040401", "4011040402", "40110503", "40110504", "40120101", "401202", "40120201", "40120202", "40129901", "40130201", "401303", "40130302", "40130303", "40130304", "401304", "4016120101", "40190603", "402002", "390904", "39090203", "391001", "39100101", "391002", "620101", "620201", "620202", "620303", "62030301", "62030302", "620401", "610102", "600103", "60010303", "60010304"}
var LdxReg_3 *regexp.Regexp = regexp.MustCompile("(明厨亮灶|安防|监控|校园安全|视频监控|校园安防|校园监控|监控系统|安防监控|学校监控|标准化考场|摄录一体机|标准化考点|平安校园|摄像头|摄像机)")

var LdxMark1 *regexp.Regexp = regexp.MustCompile("(校园安全|校园安防|校园监控|安防监控|平安校园|智慧校园|监控改造|监控升级|标准化考场|明厨亮灶|监控设备)")
var LdxMark2 *regexp.Regexp = regexp.MustCompile("(校园监控安防监控|平安校园|智慧校园|监控改造|监控升级|标准化考场|明厨亮灶|人脸识别|摄像设备安装|一键报警器|视频监控设备|人脸闸机|监控设备|门禁一体机|监控摄像头|视频监控|安防监控系统|监控可视化|平安校园建设|智能测温人脸|摄像头|考勤门禁)")

//更新新标记
func updateNewLdxTag() {
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{}, 0
	isok := 0
	it := sess.DB(data_mgo.DbName).C(SourceColl + "_new").Find(&q).Sort("_id").Select(map[string]interface{}{
		"class_code": 1,
		"title":      1,
		"detail":     1,
	}).Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("cur index ", total, "~", isok)
		}
		tmpid := qu.ObjToString(tmp["_id"])
		code := qu.ObjToString(tmp["class_code"])
		codeArr := strings.Split(code, "~")
		title := qu.ObjToString(tmp["title"])
		detail := qu.ObjToString(tmp["detail"])
		if len(codeArr) > 0 {
			isUpdate := false
			isLdx := 0
			isLdx_code := ""
			if b1, code1 := firstLdxInfo(codeArr); b1 {
				isUpdate = true
				isLdx = 1
				isLdx_code = code1
			} else {
				if b2, code2 := secondLdxInfo(code, title); b2 {
					isUpdate = true
					isLdx = 2
					isLdx_code = code2
				} else {
					if b3, code3 := thirdLdxInfo(codeArr, title, detail); b3 {
						isUpdate = true
						isLdx = 3
						isLdx_code = code3
					}
				}
			}
			if isUpdate {
				isok++
				data_mgo.Save(SourceColl+"_test", map[string]interface{}{
					"_id":        tmpid,
					"isLdx":      isLdx,
					"isLdx_code": isLdx_code,
				})
				//data_mgo.UpdateStrId(SourceColl+"_new", tmpid, map[string]interface{}{
				//	"$set": map[string]interface{}{
				//		"isLdx":      isLdx,
				//		"isLdx_code": isLdx_code,
				//	},
				//})
			}
		}
		tmp = make(map[string]interface{})
	}
	log.Debug("is over~", total, "~", isok)
}

//第一步校验
func firstLdxInfo(codeArr []string) (bool, string) {
	for _, code := range codeArr {
		for _, v := range LdxArr_1 {
			if v == code {
				return true, v
			}
		}
	}
	return false, ""
}

//第二步校验
func secondLdxInfo(code string, title string) (bool, string) {
	for _, v := range LdxArr_2 {
		if v == code {
			if LdxReg_2.MatchString(title) {
				return true, code
			}
		}
	}
	return false, ""
}

//第三步校验
func thirdLdxInfo(codeArr []string, title string, detail string) (bool, string) {
	for _, code := range codeArr {
		for _, v := range LdxArr_3 {
			if v == code {
				//if LdxReg_3.MatchString(title) {
				//	return true, v
				//}
				//if LdxReg_3.MatchString(detail) {
				//	return true, v
				//}
				return true, v
			}
		}
	}
	return false, ""
}

//打匹配上的标记-source isMatch
func updateLdxSourceMatchInfo() {
	dataArr, _ := data_mgo.Find(SourceColl+"_source", nil, nil, map[string]interface{}{"id": 1})
	ok := 0
	for k, v := range dataArr {
		if k%1000 == 0 {
			log.Debug("cur index ", k, "~", ok)
		}
		tmpid := qu.ObjToString(v["id"])
		data := data_mgo.FindByStrId(SourceColl, tmpid)
		if data != nil && len(data) > 0 {
			ok++
			data_mgo.UpdateById(SourceColl+"_source", BsonTOStringId(v["_id"]), map[string]interface{}{
				"$set": map[string]interface{}{
					"isMatch": 1,
				},
			})
		}
	}
	log.Debug(len(dataArr), "~", ok)
}

//根据关键词~更新不同的标记
func updateLdxMark0410() {
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{
		"isLdx_1": 1,
	}, 0
	it := sess.DB(data_mgo.DbName).C(SourceColl + "_new").Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("cur index ", total)
		}
		tmpid := qu.ObjToString(tmp["_id"])
		data := data_mgo.FindOne(SourceColl, map[string]interface{}{
			"_id": tmpid,
		})
		if data != nil {
			title := qu.ObjToString(data["title"])
			detail := qu.ObjToString(data["detail"])
			if LdxMark1.MatchString(title) {
				data_mgo.UpdateStrId(SourceColl+"_new", tmpid, map[string]interface{}{
					"$set": map[string]interface{}{"Ldx_mark": 1},
				})
			} else {
				if LdxMark2.MatchString(detail) {
					data_mgo.UpdateStrId(SourceColl+"_new", tmpid, map[string]interface{}{
						"$set": map[string]interface{}{"Ldx_mark": 1},
					})
				}
			}
		}
		tmp = make(map[string]interface{})
	}
	log.Debug("is over~", total)
}

//更新~五级类别分组
func updateFiveClassInfo() {
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{}, 0
	pool := make(chan bool, 6)
	wg := &sync.WaitGroup{}
	it := sess.DB(data_mgo.DbName).C(SourceColl + "_new").Find(&q).Sort("_id").Select(map[string]interface{}{
		"class_code": 1,
	}).Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("cur index ", total)
		}
		pool <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-pool
				wg.Done()
			}()
			tmpid := qu.ObjToString(tmp["_id"])
			codeArr := strings.Split(qu.ObjToString(tmp["class_code"]), "~")
			update := splitFiveInfo(codeArr, tmpid)
			if tmpid != "" && update != nil {
				//data_mgo.UpdateStrId(SourceColl+"_new", tmpid, map[string]interface{}{
				//	"$set": update,
				//})
			}
		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg.Wait()
	log.Debug("is over ", total)
}

func splitFiveInfo(codeArr []string, tmpid string) map[string]interface{} {
	arr1, arr2, arr3, arr4, arr5 := []string{}, []string{}, []string{}, []string{}, []string{}
	for _, v := range codeArr {
		if v == "" {
			continue
		}
		l := utf8.RuneCountInString(v)
		switch l {
		case 2:
			arr1 = append(arr1, v)
		case 4:
			arr2 = append(arr2, v)
		case 6:
			arr3 = append(arr3, v)
		case 8:
			arr4 = append(arr4, v)
		case 10:
			arr5 = append(arr5, v)
		default:
			log.Debug("异常~", tmpid, "~", v)
		}
	}
	update := map[string]interface{}{
		"class_1": strings.Join(arr1, "~"),
		"class_2": strings.Join(arr2, "~"),
		"class_3": strings.Join(arr3, "~"),
		"class_4": strings.Join(arr4, "~"),
		"class_5": strings.Join(arr5, "~"),
	}
	return update
}

//更新字段~buyerclass
func updateBuyerClassInfo() {
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{}, 0
	pool := make(chan bool, 6)
	wg := &sync.WaitGroup{}
	it := sess.DB(data_mgo.DbName).C(SourceColl).Find(&q).Sort("_id").Select(map[string]interface{}{
		"buyerclass": 1,
	}).Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("cur index ", total)
		}
		pool <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-pool
				wg.Done()
			}()
			tmpid := qu.ObjToString(tmp["_id"])
			data_mgo.UpdateStrId(SourceColl+"_new", tmpid, map[string]interface{}{
				"$set": map[string]interface{}{
					"buyerclass": qu.ObjToString(tmp["buyerclass"]),
				},
			})
		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg.Wait()
	log.Debug("is over ", total)
}

//分析数据~打匹配的数据
func tidyMactchTheLdxInfo() {
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{}, 0
	temp := map[string]string{}
	it := sess.DB(data_mgo.DbName).C(SourceColl + "_new").Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("cur index ", total, "~", len(temp))
		}
		tmpid := qu.ObjToString(tmp["_id"])
		temp[tmpid] = tmpid
		tmp = make(map[string]interface{})
	}
	log.Debug("符合条件加载完毕~", total, "~", len(temp))

	dataArr, _ := data_mgo.Find("zktest_lidaxin_info_source", nil, nil, map[string]interface{}{"id": 1})
	ok := 0
	for k, v := range dataArr {
		if k%1000 == 0 {
			log.Debug("cur index ", k, "~", ok)
		}
		tmpid := qu.ObjToString(v["id"])
		if temp[tmpid] != "" {
			ok++
			data_mgo.UpdateById(SourceColl+"_source", BsonTOStringId(v["_id"]), map[string]interface{}{
				"$set": map[string]interface{}{
					"isMatch": 1,
				},
			})
		}
	}
	log.Debug(len(dataArr), "~", ok)
}

//分析数据~打疑似立达信标记
func tidyMatchTheLdxData() {
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{}, 0
	isok := 0
	it := sess.DB(data_mgo.DbName).C(SourceColl + "_new").Find(&q).Sort("_id").Select(map[string]interface{}{
		"class_code": 1,
	}).Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("cur index ", total, "~", isok)
		}
		class_code := qu.ObjToString(tmp["class_code"])
		if class_code != "" {
			codeArr := strings.Split(class_code, "~")
			b, ldx_code := isTidyInfo(codeArr)
			if b {
				isok++
				data_mgo.UpdateStrId(SourceColl+"_new", qu.ObjToString(tmp["_id"]), map[string]interface{}{
					"$set": map[string]interface{}{
						"isLdx_1":      1,
						"isLdx_code_1": ldx_code,
					},
				})
			}
		}
		tmp = make(map[string]interface{})
	}
	log.Debug("is over~", total, "~", isok)
}
func isTidyInfo(codeArr []string) (bool, string) {
	for _, code := range codeArr {
		for _, v := range tidyArr {
			if v == code {
				return true, v
			}
		}
	}
	return false, ""
}

//导出指定数据
func exportLdxInfo() {
	log.Debug("准备~标讯信息~~~")
	elastic.InitElasticSize("http://172.17.4.184:19800", 10)
	//elastic.InitElasticSize("http://127.0.0.1:13002", 10)
	log.Debug("开始...遍历索引...")
	total := int64(0)
	esclient := elastic.GetEsConn()
	defer elastic.DestoryEsConn(esclient)
	if esclient == nil {
		log.Debug("连接池异常")
	}
	taskQuery := [][]string{}
	taskQuery = append(taskQuery, []string{"成交", "学校"})
	taskQuery = append(taskQuery, []string{"成交", "教育"})
	taskQuery = append(taskQuery, []string{"中标", "学校"})
	taskQuery = append(taskQuery, []string{"中标", "教育"})

	for _, v := range taskQuery {
		q_1, q_2 := v[0], v[1]
		query1 := es_elastic.NewRangeQuery("publishtime").Gte(1609430400).Lt(1672502400)
		query2 := es_elastic.NewTermQuery("subtype", q_1)
		query3 := es_elastic.NewTermQuery("buyerclass", q_2)
		cursor, err := esclient.Scan("bidding").Query(es_elastic.NewBoolQuery().Must(query1, query2, query3)).
			Size(200).Do()
		if err != nil {
			log.Debug("cursor", err)
		}
		if cursor.Results == nil {
			log.Debug("results != nil; got nil")
		}
		if cursor.Results.Hits == nil {
			log.Debug("expected results.Hits != nil; got nil")
		}
		total += cursor.TotalHits()
		log.Debug("当前查询正常数：", cursor.TotalHits(), "总数：", total)
		numDocs := 0
		//多线程 - 处理数据
		pool_es := make(chan bool, 10)
		wg_es := &sync.WaitGroup{}
		for {
			searchResult, err := cursor.Next()
			if err != nil {
				if err.Error() == "EOS" {
					break
				} else {
					log.Debug("cursor searchResult", err)
				}
			}
			for _, hit := range searchResult.Hits.Hits {
				tmp := make(map[string]interface{})
				err := json.Unmarshal(*hit.Source, &tmp)
				if err != nil {
					log.Debug("json Unmarshal error")
					continue
				}
				if numDocs%1000 == 0 {
					log.Debug("当前条数：", numDocs, "Es数据：", tmp["_id"])
				}
				numDocs++
				pool_es <- true
				wg_es.Add(1)
				go func(tmp map[string]interface{}) {
					defer func() {
						<-pool_es
						wg_es.Done()
					}()
					//p_list := IsMarkInterfaceMap(tmp["purchasinglist"])
					//if len(p_list) > 0 {
					//	data_mgo.Save("zktest_lidaxin_info", tmp)
					//}
					data_mgo.Save("zktest_lidaxin_info", tmp)
				}(tmp)
			}
		}
		wg_es.Wait()
		log.Debug("遍历完毕...", total, numDocs)
	}
	log.Debug("最后真结束了~~~~~~~~~~~~~")
}
func exportSimilarText() {
	r, err := os.Open("res/similar.txt")
	if err != nil {

	}
	defer r.Close()
	rd := bufio.NewReader(r)
	total, isok := 0, 0

	os.Remove("1.xlsx")
	f := xlsx.NewFile()
	sheet, _ := f.AddSheet("相似")
	f_row := sheet.AddRow()
	f_row.AddCell().Value = "匹配状态"
	f_row.AddCell().Value = "标准代码"
	f_row.AddCell().Value = "一级名称"
	f_row.AddCell().Value = "二级名称"
	f_row.AddCell().Value = "三级名称"
	f_row.AddCell().Value = "匹配词"

	for {
		total++
		if total%1000 == 0 {
			log.Debug("cur index ", total)
		}
		line, err := rd.ReadString('\n') //以'\n'为结束符读入一行
		if err != nil || io.EOF == err {
			break
		}
		if line == "\n" {
			continue
		}
		line = strings.ReplaceAll(line, "\n", "")
		strArr := strings.Split(line, " ")
		strArr = strArr[1:]
		if len(strArr) > 0 {
			b, code, str1, str2, str3, match := CiLinInfo(strArr)
			if b {
				row := sheet.AddRow()
				row.AddCell().Value = "完全"
				row.AddCell().Value = code
				row.AddCell().Value = str1
				row.AddCell().Value = str2
				row.AddCell().Value = str3
				row.AddCell().Value = match
				row.AddCell().Value = ""
				row.AddCell().Value = ""
				for _, v := range strArr {
					row.AddCell().Value = v
				}
			}
		}
	}
	log.Debug("总计~", total, "~", isok)

	e := f.Save("1.xlsx")
	if e != nil {
		log.Debug("保存xlsx失败:", err)
		return
	} else {
		log.Debug("保存xlsx成功:", err)
	}
}

//导出特殊校验的数据
func exportSpecLdxInfo() {
	//elastic.InitElasticSize("http://172.17.4.184:19800", 10)
	elastic.InitElasticSize("http://127.0.0.1:13002", 10)
	query := `{"query":{"filtered":{"filter":{"bool":{"must":[{"terms":{"subtype":["成交","中标"]}},{"range":{"publishtime":{"gte":1609498796,"lt":1672502400}}}],"must_not":[{"term":{"bidding.buyerclass":"学校"}},{"term":{"bidding.buyerclass":"教育"}}]}},"query":{"bool":{"should":[{"multi_match":{"query":"学校","type":"phrase","fields":["title"]}},{"multi_match":{"query":"幼儿园","type":"phrase","fields":["title"]}},{"multi_match":{"query":"小学","type":"phrase","fields":["title"]}},{"multi_match":{"query":"中学","type":"phrase","fields":["title"]}},{"multi_match":{"query":"高中","type":"phrase","fields":["title"]}},{"multi_match":{"query":"大学","type":"phrase","fields":["title"]}},{"multi_match":{"query":"职业技术学院","type":"phrase","fields":["title"]}}]}}}},"_source":["_id","title"],"sort":{"comeintime":{"order":"asc"}}}`
	ccc := elastic.Count("bidding", "bidding", query)
	log.Debug(ccc)
	return
	size, total := 200, 0
	pool_es := make(chan bool, 7)
	wg_es := &sync.WaitGroup{}
	for i := 0; i < 2030; i++ {
		if total%5000 == 0 {
			log.Debug("cur index ", i, "~", total)
		}
		pool_es <- true
		wg_es.Add(1)
		go func(i int) {
			defer func() {
				<-pool_es
				wg_es.Done()
			}()
			from := fmt.Sprintf("%d", i*size)
			q1 := `{"query":{"filtered":{"filter":{"bool":{"must":[{"terms":{"subtype":["成交","中标"]}},{"range":{"publishtime":{"gte":1609430400,"lt":1672502400}}}],"must_not":[{"term":{"bidding.buyerclass":"学校"}},{"term":{"bidding.buyerclass":"教育"}}]}},"query":{"bool":{"should":[{"multi_match":{"query":"学校","type":"phrase","fields":["title"]}},{"multi_match":{"query":"幼儿园","type":"phrase","fields":["title"]}},{"multi_match":{"query":"小学","type":"phrase","fields":["title"]}},{"multi_match":{"query":"中学","type":"phrase","fields":["title"]}},{"multi_match":{"query":"高中","type":"phrase","fields":["title"]}},{"multi_match":{"query":"大学","type":"phrase","fields":["title"]}},{"multi_match":{"query":"职业技术学院","type":"phrase","fields":["title"]}}]}}}},`
			q2 := `"sort":{"comeintime":{"order":"asc"}},"from":"` + from + `","size":"200"}`
			dataArr := *elastic.Get("bidding", "bidding", q1+q2)
			numLock.Lock()
			total += len(dataArr)
			numLock.Unlock()
			for _, v := range dataArr {
				data_mgo.Save("zktest_lidaxin_info", v)
			}
		}(i)
	}
	wg_es.Wait()
	log.Debug("is over ", total)
}

//词林信息
func CiLinInfo(strArr []string) (bool, string, string, string, string, string) {
	code, code_1, code_2, code_3 := "", "", "", ""
	for _, v := range strArr {
		if utf8.RuneCountInString(v) <= 1 {
			continue
		}
		for k1, v1 := range PurClassInfo {
			if utf8.RuneCountInString(k1) <= 1 {
				continue
			}
			if v == k1 {
				code = v1.code
				if utf8.RuneCountInString(code) == 7 {
					code_1 = code[:2]
					code_2 = code[:4]
					code_3 = code[:7]
				} else if utf8.RuneCountInString(code) == 4 {
					code_1 = code[:2]
					code_2 = code[:4]
					code_3 = ""
				} else if utf8.RuneCountInString(code) == 2 {
					code_1 = code[:2]
				} else {

				}
				return true, v1.code, PurCodeInfo[code_1], PurCodeInfo[code_2], PurCodeInfo[code_3], k1
			}
		}
	}
	return false, "", "", "", "", ""
}

func exportXXJS() {
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{
		"class_code": "",
	}, 0
	isok := 0
	it := sess.DB(data_mgo.DbName).C(SourceColl + "_new").Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("cur index ", total, "~", isok)
		}
		num := 0
		s_subscopeclass := qu.ObjToString(tmp["s_subscopeclass"])
		for _, v := range strings.Split(s_subscopeclass, ",") {
			if strings.Contains(v, "信息技术") {
				num++
			}
		}
		if num > 1 {
			isok++
			data_mgo.Save("zktest_bidding_info_jg_new_new", tmp)
		}
		tmp = make(map[string]interface{})
	}
	log.Debug("is over~", total, "~", isok)
}

func exportXXJSInfo() {
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{}, 0
	it := sess.DB(data_mgo.DbName).C(SourceColl).Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("cur index ", total)
		}
		s_topscopeclass := qu.ObjToString(tmp["s_topscopeclass"])
		if strings.Contains(s_topscopeclass, "信息技术") {
			data_mgo.Save("zktest_xinxijishu_info_zb", tmp)
		}
		tmp = make(map[string]interface{})
	}
	log.Debug("is over~", total)
}

//准备标讯~源数据
func exportBidSourceInfo() {
	log.Debug("准备~标讯信息~~~")
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{
		"_id": map[string]interface{}{
			"$gte": StringTOBsonId("63765a800000000000000000"),
			"$lt":  StringTOBsonId("63c6c6000000000000000000"),
		},
	}, 0

	isok1, isok2 := 0, 0
	pool := make(chan bool, 4)
	wg := &sync.WaitGroup{}
	it := sess.DB(data_mgo.DbName).C("bidding").Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("curent index ", total, "~", isok1, "~", isok2)
		}
		pool <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-pool
				wg.Done()
			}()
			toptype := qu.ObjToString(tmp["toptype"])
			if (toptype == "招标" || toptype == "结果") && qu.IntAll(tmp["extracttype"]) == 1 {
				p_list := IsMarkInterfaceMap(tmp["purchasinglist"])
				if len(p_list) > 0 {
					if toptype == "招标" {
						data_mgo.Save("zktest_bidding_info_zb", tmp)
					} else if toptype == "结果" {
						data_mgo.Save("zktest_bidding_info_jg", tmp)
					} else {

					}
				}
			}
		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg.Wait()
	log.Debug("is over ", total, "~", isok1, "~", isok2)
}

//全量招标~结果数据准备
func exportPurSourceInfo() {
	log.Debug("导出标的物信息数据~~~")
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{
		"_id": map[string]interface{}{
			"$lt": StringTOBsonId("63d93b000000000000000000"),
		},
	}, 0
	ZhaoBiaoInfos := map[string]map[string]interface{}{}
	JieGuoInfos := map[string]map[string]interface{}{}

	it := sess.DB(data_mgo.DbName).C("bidding").Find(&q).Sort("_id").Select(map[string]interface{}{
		"s_topscopeclass": 1,
		"purchasinglist":  1,
		"toptype":         1,
	}).Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("curent index ", total, "~", len(ZhaoBiaoInfos), "~", len(JieGuoInfos))
		}
		toptype := qu.ObjToString(tmp["toptype"])
		if toptype == "招标" || toptype == "结果" {
			p_list := IsMarkInterfaceMap(tmp["purchasinglist"])
			if len(p_list) > 0 {
				s_topscopeclass := qu.ObjToString(tmp["s_topscopeclass"])
				classArr := []string{}
				if s_topscopeclass != "" {
					classArr = strings.Split(s_topscopeclass, ",")
				}
				for _, v := range p_list {
					itemname := qu.ObjToString(v["itemname"])
					if itemname == "" || utf8.RuneCountInString(itemname) > PurLength {
						continue
					}
					if toptype == "招标" {
						info := ZhaoBiaoInfos[itemname]
						if info == nil { //新增
							ZhaoBiaoInfos[itemname] = map[string]interface{}{
								"total": 1,
								"name":  itemname,
								"class": s_topscopeclass,
							}
						} else { //补充类别
							new_info := dealWithPurInfo(info, classArr)
							ZhaoBiaoInfos[itemname] = new_info
						}
					} else if toptype == "结果" {
						info := JieGuoInfos[itemname]
						if info == nil { //新增
							JieGuoInfos[itemname] = map[string]interface{}{
								"total": 1,
								"name":  itemname,
								"class": s_topscopeclass,
							}
						} else { //补充类别
							new_info := dealWithPurInfo(info, classArr)
							JieGuoInfos[itemname] = new_info
						}
					} else {

					}

				}
			}

		}

		tmp = make(map[string]interface{})
	}
	log.Debug("标的物数据准备完毕~~ ", total, "~", len(ZhaoBiaoInfos), "~", len(JieGuoInfos))
	log.Debug("~~~~~~")
	log.Debug("开始保存~标的物数据~")

	ok1 := 0
	for k, v := range ZhaoBiaoInfos {
		if ok1%10000 == 0 {
			log.Debug("is save index ", ok1, "~", k)
		}
		data_mgo.Save("zktest_purchasing_info_zb", v)
		ok1++
	}
	log.Debug("is save zb over ", len(ZhaoBiaoInfos), "~", ok1)

	ok2 := 0
	for k, v := range JieGuoInfos {
		if ok2%10000 == 0 {
			log.Debug("is save index ", ok2, "~", k)
		}
		data_mgo.Save("zktest_purchasing_info_jg", v)
		ok2++
	}
	log.Debug("is save jg over ", len(JieGuoInfos), "~", ok2)

}

//准备按照行业准备标的物数据
func exportIndustryPurchasingInfo() {
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{}, 0
	temp := map[string]string{}
	it := sess.DB(data_mgo.DbName).C(SourceColl + "_new").Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		class_code := qu.ObjToString(tmp["class_code"])
		tmpid := BsonTOStringId(tmp["_id"])
		if class_code == "" {
			temp[tmpid] = tmpid
		}
		tmp = make(map[string]interface{})
	}
	log.Debug("待导出数据~", len(temp))

	infos := map[string]int{}
	total = 0
	isok := 0
	it = sess.DB(data_mgo.DbName).C(SourceColl).Find(&q).Sort("_id").Select(map[string]interface{}{
		"purchasinglist":  1,
		"s_topscopeclass": 1,
	}).Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("cur index ", total)
		}
		s_topscopeclass := qu.ObjToString(tmp["s_topscopeclass"])
		tmpid := BsonTOStringId(tmp["_id"])
		if strings.Contains(s_topscopeclass, "信息技术") {
			if temp[tmpid] != "" {
				isok++
				p_list := IsMarkInterfaceMap(tmp["purchasinglist"])
				for _, v := range p_list {
					itemname := qu.ObjToString(v["itemname"])
					if itemname != "" && utf8.RuneCountInString(itemname) <= PurLength {
						num := qu.IntAll(infos[itemname])
						infos[itemname] = num + 1
					}
				}
			}
		}
		tmp = make(map[string]interface{})
	}
	log.Debug("is over~", isok, "~", len(infos))

	for k, v := range infos {
		data_mgo.Save("zktest_industry_info_jg", map[string]interface{}{
			"name":  k,
			"total": v,
		})
	}
	log.Debug("is over ", len(infos))
}

//准备空标的物~词
func exportEmptyClassInfo() {
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{}, 0
	temp := map[string]interface{}{}
	it := sess.DB(data_mgo.DbName).C(SourceColl + "_new").Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		class_code := qu.ObjToString(tmp["class_code"])
		tmpid := BsonTOStringId(tmp["_id"])
		if class_code == "" {
			temp[tmpid] = qu.ObjToString(tmp["basicClass"])
		}
		tmp = make(map[string]interface{})
	}
	log.Debug("待导出数据~", len(temp))
	total = 0
	infos := map[string]int{}
	basic := map[string]string{}
	it = sess.DB(data_mgo.DbName).C(SourceColl).Find(&q).Sort("_id").Select(map[string]interface{}{
		"purchasinglist": 1,
	}).Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Debug("cur index ", total)
		}
		tmpid := BsonTOStringId(tmp["_id"])
		if temp[tmpid] != nil {
			basiclass := qu.ObjToString(temp[tmpid])
			p_list := IsMarkInterfaceMap(tmp["purchasinglist"])
			for _, v := range p_list {
				itemname := qu.ObjToString(v["itemname"])
				if itemname != "" && utf8.RuneCountInString(itemname) <= PurLength {
					num := qu.IntAll(infos[itemname])
					infos[itemname] = num + 1
					bas := qu.ObjToString(basic[itemname])
					if basiclass != "" {
						if bas == "" {
							bas = basiclass
						} else {
							if !strings.Contains(bas, basiclass) {
								bas = bas + "~" + basiclass
							}
						}
					}
					basic[itemname] = bas
				}
			}
		}

		tmp = make(map[string]interface{})
	}
	log.Debug("is over~", "~", len(infos))

	for k, v := range infos {
		data_mgo.Save("zktest_bidding_info_zb_empty", map[string]interface{}{
			"name":  k,
			"total": v,
			"basic": basic[k],
		})
	}
	log.Debug("is over ", len(infos))
}

//更新三大类
func updateBasicClassInfo() {
	sess := data_mgo.GetMgoConn()
	defer data_mgo.DestoryMongoConn(sess)
	q, total := map[string]interface{}{}, 0
	it := sess.DB(data_mgo.DbName).C(SourceColl).Find(&q).Sort("_id").Select(map[string]interface{}{
		"basicClass": 1,
	}).Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		basicClass := qu.ObjToString(tmp["basicClass"])
		tmpid := BsonTOStringId(tmp["_id"])
		data_mgo.UpdateById(SourceColl+"_new", tmpid, map[string]interface{}{
			"$set": map[string]interface{}{
				"basicClass": basicClass,
			},
		})
		tmp = make(map[string]interface{})
	}
	log.Debug("is over~", total)
}

//导出统计出现频次的数据
func integrateInfoTest() {
	dataArr, _ := data_mgo.Find("11111", nil, nil, nil)
	log.Debug(len(dataArr))
	temp := map[string]int{}
	for _, info := range dataArr {
		code := qu.ObjToString(info["class_code"])
		if code == "" {
			continue
		}
		arr := strings.Split(code, "~")
		for _, v := range arr {
			num := temp[v]
			num++
			temp[v] = num
		}
	}
	for k, v := range temp {
		data_mgo.Save("22222", map[string]interface{}{
			"code": k,
			"num":  v,
		})
	}
	log.Debug("is over ~")
}

//测试调试
func testEsQuery() {
	//elastic.InitElasticSize("http://172.17.4.184:19800", 10)
	elastic.InitElasticSize("http://127.0.0.1:13002", 10)
	total := int64(0)
	esclient := elastic.GetEsConn()
	defer elastic.DestoryEsConn(esclient)
	if esclient == nil {
		log.Debug("连接池异常")
	}
	query1 := es_elastic.NewRangeQuery("publishtime").Gte(1681833600).Lt(1681920000)
	query2 := es_elastic.NewTermQuery("subtype", "成交")
	query3 := es_elastic.NewMissingFilter("bidamount")
	query4 := es_elastic.NewMissingFilter("winner")
	cursor, err := esclient.Scan("bidding").Query(es_elastic.NewBoolQuery().Must(query1, query2, query3, query4)).Size(200).Do()
	if err != nil {
		log.Debug("cursor", err)
	}
	if cursor.Results == nil {
		log.Debug("results != nil; got nil")
	}
	if cursor.Results.Hits == nil {
		log.Debug("expected results.Hits != nil; got nil")
	}
	total += cursor.TotalHits()
	log.Debug("当前查询正常数：", cursor.TotalHits(), "总数：", total)

	numDocs, isok := 0, 0
	var packageReg *regexp.Regexp = regexp.MustCompile("(标段|分包)")
	//多线程 - 处理数据
	pool_es := make(chan bool, 1)
	wg_es := &sync.WaitGroup{}
	for {
		searchResult, err := cursor.Next()
		if err != nil {
			if err.Error() == "EOS" {
				break
			} else {
				log.Debug("cursor searchResult", err)
			}
		}
		for _, hit := range searchResult.Hits.Hits {
			tmp := make(map[string]interface{})
			err := json.Unmarshal(*hit.Source, &tmp)
			if err != nil {
				log.Debug("json Unmarshal error")
				continue
			}
			if numDocs%1000 == 0 {
				log.Debug("当前条数：", numDocs, " 符合条数：", isok)
			}
			numDocs++
			pool_es <- true
			wg_es.Add(1)
			go func(tmp map[string]interface{}) {
				defer func() {
					<-pool_es
					wg_es.Done()
				}()
				tmpid := qu.ObjToString(tmp["_id"])
				detail := qu.ObjToString(tmp["detail"])
				if packageReg.MatchString(detail) {
					isok++
					tmp["_id"] = StringTOBsonId(tmpid)
					data_mgo.Save("zktest_chatglm_info", tmp)
				}
			}(tmp)
		}
	}
	wg_es.Wait()
	log.Debug("遍历完毕...", total, numDocs, isok)
}
