package main
import (
	"flag"
	"fmt"
	"log"
	"os"
	qu "qfw/util"
	"regexp"
	"strings"
	//"time"
)

var zhb_key_list = []string{"budget", "buyer", "agency", "s_winner", "bidamount", "projectcode", "contractcode"}
var packreg *regexp.Regexp
var Mgo *MongodbSim
var listSource []*dataSource

type dataSource struct {
	_id, id, title                         string
	projectname, projectcode, contractcode string
	buyer, agency, s_winner                string
	budget, bidamount                      float64
	isrepeat                               bool
	repeat_id_source                       string
	repeat_id                              map[string]string
	repeatText                             string
	publishtime							   int64
}

//var addr, dbname, table, startTime, endTime, sortType *string
var addr, dbname, table,  sortType *string

func init() {
	addr = flag.String("addr", "172.17.4.87:27080", "数据库名称")
	dbname = flag.String("dbname", "py_spider", "数据库名称")
	table = flag.String("table", "zktest_mysql_bidding_repeat", "表名称")
	sortType = flag.String("sort", "1", "sort--请输入排序方式，1正序、-1倒序")
	flag.Parse()
	Mgo = &MongodbSim{
		MongodbAddr: *addr,
		Size:        3,
		DbName:      *dbname,
	}
	Mgo.InitPool()
	packreg, _ = regexp.Compile(`([a-zA-Z0-9①②ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ一二三四五六七八九十](包|标|段)[：:]?)`)
	//packreg, _ = regexp.MustCompile("([a-zA-Z0-9①②ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ一二三四五六七八九十](包|标|段)[：:]?)")
}

func repeatMedicalTest() {
	log.Printf("表名：%s,排序方式：%s", *table, *sortType)
	if *addr == "" || *dbname == "" || *table == "" ||  *sortType == "" {
		log.Println("参数输入有误")
		fmt.Printf("数据库地址：%s\n数据库名称：%s\n表名：%s\n排序方式：%s\n", *addr, *dbname, *table, *sortType)
		os.Exit(0)
	}
	//stime, _ := time.Parse(qu.Date_Short_Layout, *startTime)
	//etime, _ := time.Parse(qu.Date_Short_Layout, *endTime)
	//query := map[string]interface{}{}
	//query["$and"] = []interface{}{
	//	map[string]interface{}{
	//		"publishtime":map[string]interface{}{
	//			"$gte":stime.Unix(),
	//		},
	//	},
	//	map[string]interface{}{
	//		"publishtime":map[string]interface{}{
	//			"$lte":etime.Unix(),
	//		},
	//	},
	//	//bson.M{"publishtime": bson.M{"$gte": stime.Unix()}},
	//	//bson.M{"publishtime": bson.M{"$lte": etime.Unix()}},
	//}

	sort := "publishtime"
	if *sortType == "-1" {
		sort = "-publishtime"
	}
	//log.Println(sort)
	sess := Mgo.GetMgoConn()
	defer Mgo.DestoryMongoConn(sess)
	//it := sess.DB(Mgo.DbName).C(*table).Find(query).Sort(sort).Iter()
	it := sess.DB(Mgo.DbName).C(*table).Find(map[string]interface{}{}).Sort(sort).Iter()
	//对标题、项目名称等中英文符号、空格等进行处理
	var filterReg = regexp.MustCompile("[`~!@#$^&*()=|{}':;',\\[\\].<>/?~！@#￥……&*（）——|{}【】‘；：”“'。，、？%+_]")
	index := 0
	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
		d := &dataSource{
			_id:          BsonTOStringId(tmp["_id"]),
			id:           qu.ObjToString(tmp["id"]),
			title:        filterReg.ReplaceAllString(strings.ToLower(qu.ObjToString(tmp["title"])), ""),
			projectname:  filterReg.ReplaceAllString(strings.ToLower(qu.ObjToString(tmp["projectname"])), ""),
			projectcode:  filterReg.ReplaceAllString(strings.ToLower(qu.ObjToString(tmp["projectcode"])), ""),
			contractcode: filterReg.ReplaceAllString(strings.ToLower(qu.ObjToString(tmp["contractcode"])), ""),
			buyer:        filterReg.ReplaceAllString(strings.ToLower(qu.ObjToString(tmp["buyer"])), ""),
			agency:       filterReg.ReplaceAllString(strings.ToLower(qu.ObjToString(tmp["agency"])), ""),
			s_winner:     filterReg.ReplaceAllString(strings.ToLower(qu.ObjToString(tmp["s_winner"])), ""),
			budget:       qu.Float64All(tmp["budget"]),
			bidamount:    qu.Float64All(tmp["bidamount"]),
			publishtime:  qu.Int64All(tmp["publishtime"]),
			repeat_id:    map[string]string{},
		}
		//log.Println(tmp["_id"], d.id)
		if index%10000 == 0 {
			log.Println("加载数据：", index,tmp["_id"])
		}
		listSource = append(listSource, d)
		tmp = map[string]interface{}{}
	}
	log.Println("数据加载完成",len(listSource))

	dataItem()
	dd := 0
	for i := 0; i < len(listSource); i++ {
		a := listSource[i]
		if a.isrepeat {
			dd++
		}
		//更新数据
		if len(a.repeat_id) ==0{
			Mgo.UpdateById(*table, a._id,
				map[string]interface{}{"$set": map[string]interface{}{
					//重复数据看repeatid
					"repeatid":     a.repeat_id_source, //和哪条数据重复id
					"repeat":       a.isrepeat,         //本条数据是否重复数据
					"repeattext":   a.repeatText,       //本数据被判重的原因
				}})
		}else {
			if len(a.repeat_id) > 0{
				arr:=[]string{}
				for k,_:=range a.repeat_id{
					arr = append(arr,k)
				}
				Mgo.UpdateById(*table, a._id,
					map[string]interface{}{"$set": map[string]interface{}{
						//原始数据看repeatid_ids_str
						"repeatid":     a.repeat_id_source, //和哪条数据重复id
						"repeat":       a.isrepeat,         //本条数据是否重复数据
						//"repeatid_ids": a.repeat_id,        //和我重复的数据都有哪些
						"repeatid_ids_str": strings.Join(arr,","),
						"repeattext":   a.repeatText,       //本数据被判重的原因
					}})}
		}
		if i%1000 == 0 {
			log.Println("已更新", i)
		}
	}
	log.Println(dd)
}

var listSize = 20000

func dataItem() {
	for i := 0; i < len(listSource); i++ {
		a := listSource[i]
		// if a.isrepeat {
		// 	continue
		// }
		b := &dataSource{}
		for j := i + 1; j < len(listSource); j++ {
			b = listSource[j]
			if *sortType == "1" {
				if publishtime_b_a(*a,*b){
					// if b.isrepeat {
					// 	continue
					// }
					a, b = panchong(*a, *b)
					listSource[j] = b
					listSource[i] = a
					// if b.isrepeat {
					// 	log.Println("sss", a.id, b.isrepeat, b.repeat_id)
					// }
				}
			}else{
				if publishtime_a_b(*a,*b){
					// if b.isrepeat {
					// 	continue
					// }
					a, b = panchong(*a, *b)
					listSource[j] = b
					listSource[i] = a
					// if b.isrepeat {
					// 	log.Println("sss", a.id, b.isrepeat, b.repeat_id)
					// }
				}
			}
		}
		if i%500 == 0 {
			log.Println("已处理：", i)
		}
	}
}

func panchong(a, b dataSource) (c, d *dataSource) {
	switch {
	case a.title == b.title: //标题相等
		if pankong(a.contractcode) && pankong(b.contractcode) && a.contractcode != b.contractcode {

		} else if a.budget == b.budget && (a.budget >= 0 || b.budget >= 0) {
			if pankong(a.buyer) && pankong(b.buyer) && a.buyer == b.buyer {
				if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner {
					if a.budget == b.budget && (a.budget >= 0 || b.budget >= 0) {
						b.repeat_id_source = a.id
						a.repeat_id[b.id] = ""
						b.isrepeat = true
						b.repeatText = "标题相等 && buyer && s_winner && bidamount"
						//log.Println("1111", a.id, b.id, b.isrepeat)
					}
				}
			} else {
				r := key_list(a, b)
				if r {
					b.repeat_id_source = a.id
					a.repeat_id[b.id] = ""
					b.isrepeat = true
					b.repeatText = "标题相等 && budget && key_list"
				}
			}
		} else if pankong(a.projectcode) && pankong(b.projectcode) && a.projectcode == b.projectcode {
			r := key_list(a, b)
			if r {
				b.repeat_id_source = a.id
				a.repeat_id[b.id] = ""
				b.isrepeat = true
				b.repeatText = "标题相等 && projectcode && key_list"
			}
		} else if a.bidamount == b.bidamount && (a.bidamount >= 0 || b.bidamount >= 0) {
			if pankong(a.buyer) && pankong(b.buyer) && a.buyer == b.buyer {
				if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner {
					b.repeat_id_source = a.id
					a.repeat_id[b.id] = ""
					b.isrepeat = true
					b.repeatText = "标题相等 && bidamount && buyer && s_winner"
					//log.Println("1111", a.id, b.id, b.isrepeat)
				}
			} else {
				r := key_list(a, b)
				if r {
					b.repeat_id_source = a.id
					a.repeat_id[b.id] = ""
					b.isrepeat = true
					b.repeatText = "标题相等 && bidamount && key_list"
				}
			}
		} else {
			//
		}
	case a.title != b.title: //标题不相等
		//项目名称包含及相等
		if strings.Contains(a.projectname, b.projectname) || strings.Contains(b.projectname, a.projectname) {
			isp := packreg.MatchString(a.title)
			//有分包
			if isp {
				//项目名称相等
				if a.projectname == b.projectname {
					if pankong(a.contractcode) && pankong(b.contractcode) && a.contractcode != b.contractcode {
						//
					} else if a.bidamount == b.bidamount && (a.bidamount > 0 || b.bidamount > 0) {
						b.repeat_id_source = a.id
						a.repeat_id[b.id] = ""
						b.isrepeat = true
						b.repeatText = "标题不相等-->有分包 && projectname && bidamount"
					} else if a.bidamount != b.bidamount {
						//
					} else {
						if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner && a.budget == b.budget && (a.budget >= 0 || b.budget >= 0) {
							b.repeat_id_source = a.id
							a.repeat_id[b.id] = ""
							b.isrepeat = true
							b.repeatText = "标题不相等-->有分包 && projectname && s_winner && budget"
						}
					}
				} else { //项目名称包含
					if pankong(a.contractcode) && pankong(b.contractcode) && a.contractcode != b.contractcode {
						//
					} else if a.bidamount == b.bidamount && (a.bidamount >= 0 || b.bidamount >= 0) {
						if pankong(a.projectcode) && pankong(b.projectcode) && a.projectcode == b.projectcode {
							b.repeat_id_source = a.id
							a.repeat_id[b.id] = ""
							b.isrepeat = true
							b.repeatText = "标题不相等-->有分包 && projectname包含 && bidamount && projectcode"
						} else if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner {
							b.repeat_id_source = a.id
							a.repeat_id[b.id] = ""
							b.isrepeat = true
							b.repeatText = "标题不相等-->有分包 && projectname包含 && bidamount && s_winner"
						} else if a.budget == b.budget && (a.budget >= 0 || b.budget >= 0) {
							if pankong(a.buyer) && pankong(b.buyer) && a.buyer == b.buyer {
								b.repeat_id_source = a.id
								a.repeat_id[b.id] = ""
								b.isrepeat = true
								b.repeatText = "标题不相等-->有分包 && projectname包含 && bidamount && budget && buyer"
							} else if pankong(a.agency) && pankong(b.agency) && a.agency == b.agency {
								b.repeat_id_source = a.id
								a.repeat_id[b.id] = ""
								b.isrepeat = true
								b.repeatText = "标题不相等-->有分包 && projectname包含 && bidamount && budget && agency"
							} else {
								//
							}
						}
					} else if a.bidamount != b.bidamount {
						//
					} else {
						if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner && a.budget == b.budget && (a.budget > 0 || b.budget > 0) {
							b.repeat_id_source = a.id
							a.repeat_id[b.id] = ""
							b.isrepeat = true
							b.repeatText = "标题不相等-->有分包 && projectname包含 && s_winner && budget"
						} else {
							//
						}
					}
				}
			} else { //无分包
				//项目名称相等
				if a.projectname == b.projectname {
					if pankong(a.contractcode) && pankong(b.contractcode) && a.contractcode != b.contractcode {
						//
					} else if a.bidamount == b.bidamount && (a.bidamount >= 0 || b.bidamount >= 0) {
						b.repeat_id_source = a.id
						a.repeat_id[b.id] = ""
						b.isrepeat = true
						b.repeatText = "标题不相等-->无分包 && projectname && bidamount"
					} else if a.bidamount != b.bidamount {
						//
					} else {
						if pankong(a.projectcode) && pankong(b.projectcode) && a.projectcode == b.projectcode {
							b.repeat_id_source = a.id
							a.repeat_id[b.id] = ""
							b.isrepeat = true
							b.repeatText = "标题不相等-->无分包 && projectname && projectcode"
						} else if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner {
							b.repeat_id_source = a.id
							a.repeat_id[b.id] = ""
							b.isrepeat = true
							b.repeatText = "标题不相等-->无分包 && projectname && s_winner"
						} else if a.budget == b.budget && (a.budget >= 0 || b.budget >= 0) {
							if pankong(a.buyer) && pankong(b.buyer) && a.buyer == b.buyer {
								b.repeat_id_source = a.id
								a.repeat_id[b.id] = ""
								b.isrepeat = true
								b.repeatText = "标题不相等-->无分包 && projectname && budget && buyer"
							} else if pankong(a.agency) && pankong(b.agency) && a.agency == b.agency {
								b.repeat_id_source = a.id
								a.repeat_id[b.id] = ""
								b.isrepeat = true
								b.repeatText = "标题不相等-->无分包 && projectname && budget && agency"
							} else {
								//
							}
						}
					}
				} else { //项目名称包含
					if pankong(a.contractcode) && pankong(b.contractcode) && a.contractcode != b.contractcode {
						//
					} else if a.bidamount == b.bidamount && (a.bidamount >= 0 || b.bidamount >= 0) {
						if pankong(a.projectcode) && pankong(b.projectcode) && a.projectcode == b.projectcode {
							b.repeat_id_source = a.id
							a.repeat_id[b.id] = ""
							b.isrepeat = true
							b.repeatText = "标题不相等-->无分包 && projectname包含 && bidamount"
						} else if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner {
							b.repeat_id_source = a.id
							a.repeat_id[b.id] = ""
							b.isrepeat = true
							b.repeatText = "标题不相等-->无分包 && projectname包含 && s_winner"
						} else if a.budget == b.budget && (a.budget >= 0 || b.budget >= 0) {
							if pankong(a.buyer) && pankong(b.buyer) && a.buyer == b.buyer {
								b.repeat_id_source = a.id
								a.repeat_id[b.id] = ""
								b.isrepeat = true
								b.repeatText = "标题不相等-->无分包 && projectname包含 && budget && buyer"
							} else if pankong(a.agency) && pankong(b.agency) && a.agency == b.agency {
								b.repeat_id_source = a.id
								a.repeat_id[b.id] = ""
								b.isrepeat = true
								b.repeatText = "标题不相等-->无分包 && projectname包含 && budget && agency"
							} else {
								//
							}
						} else {
							//
						}
					} else if a.bidamount != b.bidamount {
						//
					} else {
						if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner && a.budget == a.budget && (a.budget > 0 || b.budget > 0) {
							b.repeat_id_source = a.id
							a.repeat_id[b.id] = ""
							b.isrepeat = true
							b.repeatText = "标题不相等-->无分包 && projectname包含 && s_winner && budget"
						}
					}
				}
			}
		}
	default:

	}
	return &a, &b
}

//zhb_key_list 判断
//"budget", "buyer", "agency", "s_winner", "bidamount", "projectcode", "contractcode"
func key_list(a, b dataSource) bool {
	for i := 0; i < len(zhb_key_list); i++ {
		key := zhb_key_list[i]
		switch key {
		case "budget":
			if a.budget == b.budget && (a.budget > 0 || b.budget > 0) {
				continue
			} else {
				return false
			}
		case "buyer":
			if a.buyer == b.buyer && pankong(a.buyer) && pankong(b.buyer) {
				continue
			} else {
				return false
			}
		case "agency":
			if a.agency == b.agency && pankong(a.agency) && pankong(b.agency) {
				continue
			} else {
				return false
			}
		case "s_winner":
			if a.s_winner == b.s_winner && pankong(a.s_winner) && pankong(b.s_winner) {
				continue
			} else {
				return false
			}
		case "bidamount":
			if a.bidamount == b.bidamount && (a.bidamount > 0 || b.bidamount > 0) {
				continue
			} else {
				return false
			}
		case "projectcode":
			if a.projectcode == b.projectcode && pankong(a.projectcode) && pankong(b.projectcode) {
				continue
			} else {
				return false
			}
		case "contractcode":
			if a.contractcode == b.contractcode && pankong(a.contractcode) && pankong(b.contractcode) {
				continue
			} else {
				return false
			}
		}
	}
	return true
}


//发布时间判断
//正序
func publishtime_b_a(a,b dataSource) bool{
	return b.publishtime-a.publishtime < 86400 * 31 * 9
}
//倒序
func publishtime_a_b(a,b dataSource) bool {
	return a.publishtime-b.publishtime < 86400 * 31 * 9
}


//
func pankong(a string) bool {
	if a != "" {
		return true
	} else {
		return false
	}
}
