no message

.gitignore

@@ -0,0 +1,8 @@
+pkg
+bin
+*/pkg
+*.exe
+*.log
+*/src/src
+*/bin
+createindex_linux

README.md

@@ -0,0 +1,7 @@
+## 数据处理项目-索引数据处理
+
+
+### common_utils 公共包
+
+### udpcreateindex 索引处理程序-v1.0
+

common_utils/clearHtml.go

@@ -0,0 +1,165 @@
+package util
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+)
+
+//
+type Cut struct {
+	annotate      *regexp.Regexp
+	tag           *regexp.Regexp
+	scripttag     *regexp.Regexp
+	inputag       *regexp.Regexp
+	isborder      *regexp.Regexp
+	hiddentag     *regexp.Regexp
+	styletag      *regexp.Regexp
+	colstag       *regexp.Regexp
+	rowstag       *regexp.Regexp
+	display       *regexp.Regexp
+	multiCR       *regexp.Regexp
+	replBlankLine *regexp.Regexp
+	replStartWrap *regexp.Regexp
+	replTags2CR   []string
+	retainTags2CR []string
+}
+
+//
+func NewCut() *Cut {
+	t, _ := regexp.Compile("<[^>]+>")
+	m, _ := regexp.Compile("([\r\n][\u3000\u2003\u00a0\\s]*)+|[\r\n]+")
+	//sc, _ := regexp.Compile("\\<script[^\\>]*\\>*[^\\>]+\\</script\\>")
+	//ss, _ := regexp.Compile("\\<style[^\\>]*\\>*[^\\>]+\\</style\\>")
+	scs := regexp.MustCompile("(?s)<(script|style)[^>]*>.+?</(script|style)>")
+	at := regexp.MustCompile("(?s)<(!%-%-|!--).*?(%-%-|--)>") //注释 css
+	hiddentag := regexp.MustCompile(`<\s*input[^<]*type=("|')hidden("|')[^<]*>`)
+	input := regexp.MustCompile(`<\s*input[^<]*value=("|')([^>"']*)[^<]*>`)
+	cols, _ := regexp.Compile(`colspan="\d+"`)
+	rows, _ := regexp.Compile(`rowspan="\d+"`)
+	border, _ := regexp.Compile(`(border="(\d+)")|(cellpadding="(\d+)")|(cellspacing="(\d+)")`)
+	dis, _ := regexp.Compile(`display:none`)
+	return &Cut{
+		annotate:      at,
+		tag:           t,
+		scripttag:     scs,
+		hiddentag:     hiddentag,
+		inputag:       input,
+		colstag:       cols,
+		isborder:      border,
+		rowstag:       rows,
+		display:       dis,
+		multiCR:       m,
+		replBlankLine: regexp.MustCompile("\\s+[\r\n]"),
+		replStartWrap: regexp.MustCompile("^[\u3000\u2003\u00a0\\s]+|[\u3000\u2003\u00a0\\s]+$"),
+		replTags2CR:   []string{"div", "p", "br", "h1", "h2", "h3", "h4", "h5"},
+		retainTags2CR: []string{"table", "thead", "tfoot", "tbody", "th", "td", "tr"},
+	}
+}
+
+//清理HTML标签
+func (c *Cut) ClearHtml(src string) string {
+	src = c.replBlankLine.ReplaceAllString(src, "")
+	src = strings.Replace(src, ">\n", ">", -1)
+	src = strings.Replace(src, " ", "", -1)
+	//标签全转小写
+	src = c.tag.ReplaceAllStringFunc(src, strings.ToLower)
+	//清script,style
+	src = c.scripttag.ReplaceAllString(src, "")
+	//清理注释文本
+	src = c.annotate.ReplaceAllString(src, "")
+	//清理input
+	src = c.hiddentag.ReplaceAllString(src, "")
+	src = c.inputag.ReplaceAllString(src, "$2")
+	document, err := goquery.NewDocumentFromReader(strings.NewReader(src))
+	if err == nil {
+		if tmpstr, err := document.Each(func(i int, sel *goquery.Selection) {
+			sel.Find("td").Each(func(i int, selection *goquery.Selection) {
+				val, b := selection.Attr("title")
+				if b && strings.Trim(val, " ") != "" {
+					tmpstr := strings.TrimFunc(selection.Text(), func(r rune) bool {
+						return r == 9 || r == 32
+					})
+					if utf8.RuneCountInString(strings.Trim(tmpstr, " ")) < utf8.RuneCountInString(strings.Trim(val, " ")) {
+						selection.SetText(strings.Trim(val, " "))
+					}
+				}
+			})
+		}).Html(); err == nil {
+			src = tmpstr
+		}
+	}
+	//换结束标签
+	src = c.tag.ReplaceAllStringFunc(src, func(tmp string) string {
+		tmp = strings.Replace(tmp, " ", "", -1)
+		//保留这些标签
+		for _, v := range c.retainTags2CR {
+			if "<"+v+">" == tmp || "</"+v+">" == tmp {
+				if tmp == "</table>" {
+					return tmp + "\n"
+				}
+				return tmp
+			}
+			if strings.HasPrefix(tmp, "<"+v) {
+				dispstrs := c.display.FindAllString(tmp, -1)
+				rowstrs := c.rowstag.FindAllString(tmp, -1)
+				colstrs := c.colstag.FindAllString(tmp, -1)
+				con := "<" + v
+				if con == "<table" {
+					if isHasBoder(tmp, c.isborder) {
+						con = con + ` border="1"`
+					}
+				}
+				if len(colstrs) > 0 { //处理多列合并
+					con += " " + colstrs[0]
+				}
+				if len(rowstrs) > 0 { //处理多行合并
+					con += " " + rowstrs[0]
+				}
+				if len(dispstrs) > 0 {
+					con += " style=\"" + dispstrs[0] + "\""
+				}
+				return con + ">"
+
+			}
+		}
+		if tmp == "<br>" || tmp == "<br/>" || tmp == "<center>" || tmp == "</center>" || tmp == "<ul>" || tmp == "</ul>" {
+			return "\n"
+		}
+		if tmp[1] != 47 { //开始标签
+			for _, v := range c.replTags2CR {
+				if v == tmp[1:len(tmp)-1] {
+					return "\n"
+				}
+			}
+			return ""
+		}
+		for _, v := range c.replTags2CR {
+			if v == tmp[2:len(tmp)-1] {
+				return "\n"
+			}
+		}
+		return ""
+	})
+	src = c.replStartWrap.ReplaceAllString(src, "")
+	src = c.replBlankLine.ReplaceAllString(src, "\n")
+	//清除多余换行
+	c.multiCR.ReplaceAllString(src, "\n")
+	return strings.Replace(src, "\n", "<br/>", -1)
+}
+
+//判断table是否加表格线
+func isHasBoder(con string, reg *regexp.Regexp) bool {
+	res := reg.FindAllStringSubmatch(con, -1)
+	hasBorder := false
+	for _, v := range res {
+		for k, val := range v {
+			if k > 0 && k%2 == 0 && IntAll(val) > 0 {
+				hasBorder = true
+				break
+			}
+		}
+	}
+	return hasBorder
+}

common_utils/common.go

@@ -0,0 +1,696 @@
+package util
+
+import (
+	"crypto/md5"
+	cryptoRand "crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	bson "gopkg.in/mgo.v2/bson"
+	"io"
+	"log"
+	"math"
+	"math/big"
+	mathRand "math/rand"
+	"net/url"
+	"reflect"
+	"regexp"
+	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/dchest/captcha"
+)
+
+const (
+	tmp = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ12345678900"
+)
+
+//短地址加密
+func EncodeArticleId(keys ...string) string {
+	kstr := strings.Join(keys, ",")
+	return SE.EncodeString(kstr)
+}
+
+//短地址解密
+func DecodeArticleId(id string) []string {
+	return strings.Split(SE.DecodeString(id), ",")
+}
+
+//短地址加密,二次加密带校验和
+func EncodeArticleId2ByCheck(keys ...string) string {
+	kstr := strings.Join(keys, ",")
+	kstr = SE.EncodeStringByCheck(kstr)
+	return url.QueryEscape("ABC" + SE2.EncodeStringByCheck(kstr))
+}
+
+//短地址解密,二次解密带校验和
+func DecodeArticleId2ByCheck(id string) []string {
+	if !strings.Contains(id, "+") { //新加密算法解密
+		id, _ = url.QueryUnescape(id)
+	}
+	if id[:3] == "ABC" { //前三位为ABC是新加密数据
+		kstr := SE2.DecodeStringByCheck(id[3:])
+		return strings.Split(SE.DecodeStringByCheck(kstr), ",")
+	} else { //历史数据
+		rep := DecodeArticleId(id)
+		oldpushid := "58f87a9561a0721f157bc74d" //剑鱼1.9发版前最后一次推送信息id
+		if rep[0] > oldpushid {
+			return []string{""}
+		} else {
+			return rep
+		}
+	}
+}
+
+func Uuid(length int) string {
+	ret := []string{}
+	r := mathRand.New(mathRand.NewSource(time.Now().UnixNano()))
+	for i := 0; i < length; i++ {
+		index := r.Intn(62)
+		ret = append(ret, tmp[index:index+1])
+	}
+	return strings.Join(ret, "")
+}
+
+//计算字符串和值
+func Sumstring(code string) (sum int) {
+	tmp := []rune(code)
+	for _, v := range tmp {
+		sum = sum + int(v)
+	}
+	return
+}
+
+//获取随机数
+func GetRandom(n int) string {
+	var idChars = []byte("0123456789")
+	b := captcha.RandomDigits(n)
+	for i, c := range b {
+		b[i] = idChars[c]
+	}
+	return string(b)
+}
+
+//获取复杂的随机数
+func GetLetterRandom(length int, flag ...bool) string {
+	var idChars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")
+	var mod byte = 52
+	if len(flag) > 0 && flag[0] {
+		idChars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+		mod = 26
+	}
+	b := make([]byte, length)
+	maxrb := byte(256 - (256 % int(mod)))
+	i := 0
+EXIT:
+	for {
+		r := make([]byte, length+(length/4))
+		if _, err := io.ReadFull(cryptoRand.Reader, r); err != nil {
+			panic("captcha: error reading random source: " + err.Error())
+		}
+		for _, c := range r {
+			if c > maxrb {
+				continue
+			}
+			b[i] = c % mod
+			i++
+			if i == length {
+				break EXIT
+			}
+		}
+	}
+	for i, c := range b {
+		b[i] = idChars[c]
+	}
+	return string(b)
+}
+
+/*获取复杂的随机数,数字和字母的组合
+ * c > 2 数字的个数和字母的个数随机分配
+ * n 数字的个数
+ * l 字母的个数
+ */
+func GetComplexRandom(c, n, l int) string {
+	if c < 2 && (n < 1 || l < 1) {
+		return "--"
+	}
+	r := mathRand.New(mathRand.NewSource(time.Now().UnixNano()))
+	myCommonMethod := func(flag bool) int {
+		if flag {
+			return r.Intn(c-1) + 1
+		} else {
+			return r.Intn(c)
+		}
+	}
+	if c >= 2 {
+		n = myCommonMethod(true)
+		l = c - n
+	} else {
+		c = l + n
+	}
+	value := GetRandom(n) + GetLetterRandom(l)
+	var array = strings.Split(value, "")
+	for i := 0; i < c/2; i++ {
+		r1 := myCommonMethod(false)
+		r2 := myCommonMethod(false)
+		o := array[r1]
+		array[r1] = array[r2]
+		array[r2] = o
+	}
+	return strings.Join(array, "")
+}
+
+/*隐藏部分账号
+ *返回手机号：150...70765 邮箱：...shenjun@vip.qq.com
+ */
+func EncryCode(value string) string {
+	if len(value) == 0 {
+		return value
+	} else if strings.Contains(value, "@") {
+		start := strings.Index(value, "@") / 2
+		if start == 0 {
+			start++
+		}
+		value = "...." + string(value[start:])
+	} else {
+		value = string(value[0:3]) + "..." + string(value[len(value)-4:])
+	}
+	return value
+}
+
+//生成32位md5字串
+func GetMd5String(s string) string {
+	h := md5.New()
+	h.Write([]byte(s))
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+//obj(string,M)转M,查询用到
+func ObjToMap(obj interface{}) *map[string]interface{} {
+	data := make(map[string]interface{})
+	if s, ok := obj.(string); ok {
+		json.Unmarshal([]byte(strings.Replace(s, "'", "\"", -1)), &data)
+	} else if s1, ok1 := obj.(map[string]interface{}); ok1 {
+		data = s1
+	} else if s1, ok1 := obj.(*map[string]interface{}); ok1 {
+		return s1
+	} else {
+		data = nil
+	}
+	return &data
+}
+
+/*UTC类型时间转字符串
+ *flag==true,日期格式yyyy-mm-dd hh:mm:ss
+ *flag==false,日期格式yyyy-mm-dd
+ */
+func LongToDate(date interface{}, flag bool) string {
+	var int64Date int64
+	if l1, ok1 := date.(float64); ok1 {
+		int64Date = int64(l1)
+	} else if l2, ok2 := date.(int64); ok2 {
+		int64Date = l2
+	} else if l3, ok3 := date.(int); ok3 {
+		int64Date = int64(l3)
+	}
+	t := time.Unix(int64Date, 0)
+	if flag {
+		return t.Format("2006-01-02 15:04:05")
+	} else {
+		return t.Format("2006-01-02")
+	}
+}
+
+func IntAll(num interface{}) int {
+	return IntAllDef(num, 0)
+}
+
+func Int64All(num interface{}) int64 {
+	if i, ok := num.(int64); ok {
+		return int64(i)
+	} else if i0, ok0 := num.(int32); ok0 {
+		return int64(i0)
+	} else if i1, ok1 := num.(float64); ok1 {
+		return int64(i1)
+	} else if i2, ok2 := num.(int); ok2 {
+		return int64(i2)
+	} else if i3, ok3 := num.(float32); ok3 {
+		return int64(i3)
+	} else if i4, ok4 := num.(string); ok4 {
+		i64, _ := strconv.ParseInt(i4, 10, 64)
+		//in, _ := strconv.Atoi(i4)
+		return i64
+	} else if i5, ok5 := num.(int16); ok5 {
+		return int64(i5)
+	} else if i6, ok6 := num.(int8); ok6 {
+		return int64(i6)
+	} else if i7, ok7 := num.(*big.Int); ok7 {
+		in, _ := strconv.ParseInt(fmt.Sprint(i7), 10, 64)
+		return int64(in)
+	} else if i8, ok8 := num.(*big.Float); ok8 {
+		in, _ := strconv.ParseInt(fmt.Sprint(i8), 10, 64)
+		return int64(in)
+	} else {
+		return 0
+	}
+}
+
+func Float64All(num interface{}) float64 {
+	if i, ok := num.(float64); ok {
+		return float64(i)
+	} else if i0, ok0 := num.(int32); ok0 {
+		return float64(i0)
+	} else if i1, ok1 := num.(int64); ok1 {
+		return float64(i1)
+	} else if i2, ok2 := num.(int); ok2 {
+		return float64(i2)
+	} else if i3, ok3 := num.(float32); ok3 {
+		return float64(i3)
+	} else if i4, ok4 := num.(string); ok4 {
+		in, _ := strconv.ParseFloat(i4, 64)
+		return in
+	} else if i5, ok5 := num.(int16); ok5 {
+		return float64(i5)
+	} else if i6, ok6 := num.(int8); ok6 {
+		return float64(i6)
+	} else if i6, ok6 := num.(uint); ok6 {
+		return float64(i6)
+	} else if i6, ok6 := num.(uint8); ok6 {
+		return float64(i6)
+	} else if i6, ok6 := num.(uint16); ok6 {
+		return float64(i6)
+	} else if i6, ok6 := num.(uint32); ok6 {
+		return float64(i6)
+	} else if i6, ok6 := num.(uint64); ok6 {
+		return float64(i6)
+	} else if i7, ok7 := num.(*big.Float); ok7 {
+		in, _ := strconv.ParseFloat(fmt.Sprint(i7), 64)
+		return float64(in)
+	} else if i8, ok8 := num.(*big.Int); ok8 {
+		in, _ := strconv.ParseFloat(fmt.Sprint(i8), 64)
+		return float64(in)
+	} else {
+		return 0
+	}
+}
+
+func IntAllDef(num interface{}, defaultNum int) int {
+	if i, ok := num.(int); ok {
+		return int(i)
+	} else if i0, ok0 := num.(int32); ok0 {
+		return int(i0)
+	} else if i1, ok1 := num.(float64); ok1 {
+		return int(i1)
+	} else if i2, ok2 := num.(int64); ok2 {
+		return int(i2)
+	} else if i3, ok3 := num.(float32); ok3 {
+		return int(i3)
+	} else if i4, ok4 := num.(string); ok4 {
+		in, _ := strconv.Atoi(i4)
+		return int(in)
+	} else if i5, ok5 := num.(int16); ok5 {
+		return int(i5)
+	} else if i6, ok6 := num.(int8); ok6 {
+		return int(i6)
+	} else if i7, ok7 := num.(*big.Int); ok7 {
+		in, _ := strconv.Atoi(fmt.Sprint(i7))
+		return int(in)
+	} else if i8, ok8 := num.(*big.Float); ok8 {
+		in, _ := strconv.Atoi(fmt.Sprint(i8))
+		return int(in)
+	} else {
+		return defaultNum
+	}
+}
+
+func ObjToString(old interface{}) string {
+	if nil == old {
+		return ""
+	} else {
+		r, _ := old.(string)
+		return r
+	}
+}
+
+func ObjToStringDef(old interface{}, defaultstr string) string {
+	if nil == old {
+		return defaultstr
+	} else {
+		r, _ := old.(string)
+		if r == "" {
+			return defaultstr
+		}
+		return r
+	}
+}
+
+//对象数组转成string数组
+func ObjArrToStringArr(old []interface{}) []string {
+	if old != nil {
+		new := make([]string, len(old))
+		for i, v := range old {
+			new[i] = v.(string)
+		}
+		return new
+	} else {
+		return nil
+	}
+}
+
+//对象数组转成map数组
+func ObjArrToMapArr(old []interface{}) []map[string]interface{} {
+	if old != nil {
+		new := make([]map[string]interface{}, len(old))
+		for i, v := range old {
+			new[i] = v.(map[string]interface{})
+		}
+		return new
+	} else {
+		return nil
+	}
+}
+
+//map数组转成对象数组
+func MapArrToObjArr(old []map[string]interface{}) []interface{} {
+	if old != nil {
+		new := make([]interface{}, len(old))
+		for i, v := range old {
+			new[i] = v
+		}
+		return new
+	} else {
+		return nil
+	}
+}
+
+func SubstrByByte(str string, length int) string {
+	bs := []byte(str)[:length]
+	bl := 0
+	for i := len(bs) - 1; i >= 0; i-- {
+		switch {
+		case bs[i] >= 0 && bs[i] <= 127:
+			return string(bs[:i+1])
+		case bs[i] >= 128 && bs[i] <= 191:
+			bl++
+		case bs[i] >= 192 && bs[i] <= 253:
+			cl := 0
+			switch {
+			case bs[i]&252 == 252:
+				cl = 6
+			case bs[i]&248 == 248:
+				cl = 5
+			case bs[i]&240 == 240:
+				cl = 4
+			case bs[i]&224 == 224:
+				cl = 3
+			default:
+				cl = 2
+			}
+			if bl+1 == cl {
+				return string(bs[:i+cl])
+			}
+			return string(bs[:i])
+		}
+	}
+	return ""
+}
+
+func SubString(str string, begin, length int) (substr string) {
+	// 将字符串的转换成[]rune
+	rs := []rune(str)
+	lth := len(rs)
+	// 简单的越界判断
+	if begin < 0 {
+		begin = 0
+	}
+	if begin >= lth {
+		begin = lth
+	}
+	end := begin + length
+	if end > lth {
+		end = lth
+	}
+
+	// 返回子串
+	return string(rs[begin:end])
+}
+
+//捕获异常
+func Try(fun func(), handler func(interface{})) {
+	defer func() {
+		if err := recover(); err != nil {
+			for skip := 1; ; skip++ {
+				_, file, line, ok := runtime.Caller(skip)
+				if !ok {
+					break
+				}
+				go log.Printf("%v,%v\n", file, line)
+			}
+			handler(err)
+		}
+	}()
+	fun()
+}
+
+//3目运算
+func If(b bool, to, fo interface{}) interface{} {
+	if b {
+		return to
+	} else {
+		return fo
+	}
+}
+
+//HashCode值
+func HashCode(uid string) int {
+	var h uint32 = 0
+	rs := []rune(uid)
+	for i := 0; i < len(rs); i++ {
+		h = 31*h + uint32(rs[i])
+	}
+	return int(h)
+}
+
+//获取离n天的秒差
+func GetDayStartSecond(n int) int64 {
+	now := time.Now()
+	tom := time.Date(now.Year(), now.Month(), now.Day()+n, 0, 0, 0, 0, time.Local)
+	return tom.Unix()
+}
+
+func InterfaceArrTointArr(arr []interface{}) []int {
+	tmp := make([]int, 0)
+	for _, v := range arr {
+		tmp = append(tmp, int(v.(float64)))
+	}
+	return tmp
+}
+func InterfaceArrToint64Arr(arr []interface{}) []int64 {
+	tmp := make([]int64, 0)
+	for _, v := range arr {
+		tmp = append(tmp, int64(v.(float64)))
+	}
+	return tmp
+}
+
+//根据bsonID转string
+func BsonIdToSId(uid interface{}) string {
+	if uid == nil {
+		return ""
+	} else if u, ok := uid.(string); ok {
+		return u
+	} else {
+		return fmt.Sprintf("%x", string(uid.(bson.ObjectId)))
+	}
+}
+
+func StringTOBsonId(id string) (bid bson.ObjectId) {
+	defer Catch()
+	if id != "" {
+		bid = bson.ObjectIdHex(id)
+	}
+	return
+}
+
+func GetSubDay(t1 int64) int {
+	tt1 := time.Unix(t1, 0)
+	tt2 := time.Now()
+	nt1 := time.Date(tt1.Year(), tt1.Month(), tt1.Day(), 0, 0, 0, 0, time.Local)
+	nt2 := time.Date(tt2.Year(), tt2.Month(), tt2.Day(), 0, 0, 0, 0, time.Local)
+	return int((nt1.Unix() - nt2.Unix()) / 86400)
+}
+func StartWith(value, str string) bool {
+	ok, _ := regexp.MatchString("^"+str, value)
+	return ok
+}
+func EndWith(value, str string) bool {
+	ok, _ := regexp.MatchString(str+"$", value)
+	return ok
+}
+
+//出错拦截
+func Catch() {
+	if r := recover(); r != nil {
+		log.Println(r)
+		for skip := 0; ; skip++ {
+			_, file, line, ok := runtime.Caller(skip)
+			if !ok {
+				break
+			}
+			go log.Printf("%v,%v\n", file, line)
+		}
+	}
+}
+
+func ConvertFileSize(s int) string {
+	size := float64(s)
+	var kb float64 = 1024
+	var mb float64 = kb * 1024
+	var gb float64 = mb * 1024
+	if size >= gb {
+		return fmt.Sprintf("%.1f GB", float64(size/gb))
+	} else if size >= mb {
+		f := float64(size / mb)
+		if f > 100 {
+			return fmt.Sprintf("%.0f MB", f)
+		}
+		return fmt.Sprintf("%.1f MB", f)
+	} else if size >= kb {
+		f := float64(size / kb)
+		if f > 100 {
+			return fmt.Sprintf("%.0f KB", f)
+		}
+		return fmt.Sprintf("%.1f KB", f)
+	}
+	return fmt.Sprintf("%d B", s)
+}
+
+//MD5签名
+func WxSign(format string, param ...interface{}) string {
+	data := fmt.Sprintf(format, param...)
+	h := md5.New()
+	h.Write([]byte(data))
+	sign := strings.ToUpper(hex.EncodeToString(h.Sum(nil)))
+	return sign
+}
+
+//计算时差
+func TimeDiff(date time.Time) string {
+	var date1 = date                        //开始时间
+	var date2 = time.Now()                  //结束时间
+	var date3 = date2.Unix() - date1.Unix() //时间差的毫秒数
+	//计算出相差天数
+	var days = math.Floor(float64(date3 / (24 * 3600)))
+	//计算出小时数
+	var leave1 = date3 % (24 * 3600) //计算天数后剩余的毫秒数
+	var hours = math.Floor(float64(leave1 / (3600)))
+	//计算相差分钟数
+	var leave2 = leave1 % (3600) //计算小时数后剩余的毫秒数
+	var minutes = math.Floor(float64(leave2 / (60)))
+	//计算相差秒数
+	var td = "30秒前"
+	if days > 0 {
+		if days > 10 {
+			if date1.Year() < date2.Year() {
+				td = FormatDate(&date, Date_Short_Layout)
+			} else {
+				td = FormatDate(&date, Date_Small_Layout)
+			}
+		} else {
+			td = fmt.Sprint(days) + "天前"
+		}
+	} else if hours > 0 {
+		td = fmt.Sprint(hours) + "小时前"
+	} else if minutes > 0 {
+		td = fmt.Sprint(minutes) + "分钟前"
+	}
+	return td
+}
+
+func FloatFormat(tmp float64, n int) float64 {
+	fs := fmt.Sprintf("%."+fmt.Sprint(n)+"f", tmp)
+	f, _ := strconv.ParseFloat(fs, 64)
+	return f
+}
+
+//生成微信支付的签名
+func CreateWxSign(afterStr string, obj interface{}, filter ...string) string {
+	filter = append(filter, "sign", "xml")
+	keys := []string{}
+	m := make(map[string]string)
+	t := reflect.TypeOf(obj)
+	v := reflect.ValueOf(obj)
+	k := t.Kind()
+	if t.Kind() == reflect.Ptr {
+		t = t.Elem()
+		k = t.Kind()
+		v = v.Elem()
+	}
+	if k == reflect.Map {
+		for _, key := range v.MapKeys() {
+			keys = append(keys, key.String())
+			m[key.String()] = fmt.Sprint(v.MapIndex(key).Interface())
+		}
+	} else if k == reflect.Struct {
+		for n := 0; n < t.NumField(); n++ {
+			tagName := t.Field(n).Tag.Get("xml")
+			if tagName == "" {
+				tagName = t.Field(n).Tag.Get("json")
+			}
+			if tagName == "" {
+				tagName = t.Field(n).Name
+			}
+			keys = append(keys, tagName)
+			m[tagName] = fmt.Sprint(v.Field(n))
+		}
+	}
+	sort.Strings(keys)
+	vs := []string{}
+L:
+	for _, v := range keys {
+		for _, f := range filter {
+			if f == v {
+				continue L
+			}
+		}
+		if strings.TrimSpace(m[v]) == "" {
+			continue
+		}
+		vs = append(vs, fmt.Sprintf("%s=%s", v, m[v]))
+	}
+	return WxSign(strings.Join(vs, "&") + afterStr)
+}
+
+//简单的xml转map，只有一个层级，没有多层嵌套
+func XmlToMap(input string) map[string]string {
+	var t xml.Token
+	var err error
+	inputReader := strings.NewReader(input)
+	decoder := xml.NewDecoder(inputReader)
+	isStart := false
+	nodeName := ""
+	m := make(map[string]string)
+	for t, err = decoder.Token(); err == nil; t, err = decoder.Token() {
+		switch token := t.(type) {
+		// 处理元素开始（标签）
+		case xml.StartElement:
+			isStart = true
+			nodeName = token.Name.Local
+		// 处理元素结束（标签）
+		case xml.EndElement:
+			isStart = false
+		// 处理字符数据（这里就是元素的文本）
+		case xml.CharData:
+			if isStart {
+				m[nodeName] = string([]byte(token))
+			}
+		default:
+			// ...
+		}
+	}
+	return m
+}

common_utils/config.go

@@ -0,0 +1,99 @@
+package util
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+
+	"os"
+	"strings"
+	"sync"
+)
+
+var lock sync.Mutex
+
+//读取配置文件,可以有2个参数，
+//第一个参数是配置文件路径，如果只有一个参数时，默认配置文件路径为./config.json
+//第二个参数是要注入的对象，可以是结构体，或者是interface对象
+func ReadConfig(config ...interface{}) {
+	var r *os.File
+	if len(config) > 1 {
+		filepath, _ := config[0].(string)
+		println(filepath)
+		r, _ = os.Open(filepath)
+		defer r.Close()
+		bs, _ := ioutil.ReadAll(r)
+		json.Unmarshal(bs, config[1])
+	} else {
+		r, _ = os.Open("./config.json")
+		defer r.Close()
+		bs, _ := ioutil.ReadAll(r)
+		json.Unmarshal(bs, config[0])
+	}
+}
+
+//程序修改SysConfig配置表后，调用写入配置文件
+func WriteSysConfig(config ...interface{}) {
+	var r *os.File
+	var configobj interface{}
+	if len(config) > 1 {
+		filepath, _ := config[0].(string)
+		r, _ = os.OpenFile(filepath, os.O_WRONLY|os.O_TRUNC, 0x644)
+		configobj = config[1]
+	} else {
+		r, _ = os.OpenFile("./config.json", os.O_WRONLY|os.O_TRUNC, 0x644)
+		configobj = config[0]
+	}
+	defer r.Close()
+	if s, ok := configobj.(string); ok {
+		r.Write([]byte(s))
+
+	} else {
+		bs, _ := json.Marshal(configobj)
+		r.Write(bs)
+	}
+}
+
+//按路径查map配置
+//TODO 暂未处理查询路径中是数组的情况
+func GetPropertie(qpath /*map的查询路径*/ string, config map[string]interface{}) (ret interface{}) {
+	//tmp := new(map[string]interface{})
+	tmp := config
+	qps := strings.Split(qpath, ".")
+	length := len(qps)
+	for i, v := range qps {
+		if v1, ok := tmp[v]; ok {
+			//log.Println("类型:", reflect.TypeOf(v1))
+			if v2, ok2 := v1.(map[string]interface{}); ok2 {
+				tmp = v2
+			} else if i == length-1 {
+				//map断了,没有下一层了
+				ret = v1
+			}
+		} else {
+			return nil
+		}
+	}
+	return
+}
+
+//设置值
+//TODO 未处理数组，仅处理map递归型
+func SetPropertie(qpath string, value interface{}, config map[string]interface{}) (err error) {
+	tmp := config
+	qps := strings.Split(qpath, ".")
+	length := len(qps)
+	for i := 0; i < length-1; i++ {
+		if v1, ok1 := (tmp[qps[i]]).(map[string]interface{}); ok1 {
+			tmp = v1
+		} else {
+			err = errors.New("路径查找失败")
+			break
+		}
+
+	}
+	if err == nil {
+		tmp[qps[length-1]] = value
+	}
+	return
+}

common_utils/date.go

@@ -0,0 +1,79 @@
+//日期处理工具类
+package util
+
+import (
+	"fmt"
+	"time"
+)
+
+const (
+	Date_Full_Layout    = "2006-01-02 15:04:05"
+	Date_Short_Layout   = "2006-01-02"
+	Date_Small_Layout   = "01-02"
+	Date_Time_Layout    = "15:04"
+	Date_yyyyMMdd       = "20060102"
+	Date_yyyyMMdd_Point = "2006.01.02"
+)
+
+//当前日期格式化
+func NowFormat(layout string) string {
+	return time.Now().Local().Format(layout)
+}
+
+//日期格式化
+func FormatDate(src *time.Time, layout string) string {
+	return (*src).Local().Format(layout)
+}
+
+//兼容从Java转换过来的数据，java生成的时间戳是按微秒算的，Go中只能按毫秒或纳秒算
+func formatDateWithInt64(src *int64, layout string) string {
+	var tmp int64
+	if *src > 0 {
+		if len(fmt.Sprint(*src)) >= 12 {
+			tmp = (*src) / 1000
+		} else {
+			tmp = (*src)
+		}
+	} else {
+		if len(fmt.Sprint(*src)) >= 13 {
+			tmp = (*src) / 1000
+		} else {
+			tmp = (*src)
+		}
+	}
+	date := time.Unix(tmp, 0)
+	return FormatDate(&date, layout)
+}
+
+func FormatDateByInt64(src *int64, layout string) string {
+	var tmp int64
+	if *src > 0 {
+		if len(fmt.Sprint(*src)) >= 12 {
+			tmp = (*src) / 1000
+		} else {
+			tmp = (*src)
+		}
+	} else {
+		if len(fmt.Sprint(*src)) >= 13 {
+			tmp = (*src) / 1000
+		} else {
+			tmp = (*src)
+		}
+	}
+	date := time.Unix(tmp, 0)
+	return FormatDate(&date, layout)
+}
+
+//支持源端多种格式
+func FormatDateWithObj(src *interface{}, layout string) string {
+	if tmp, ok := (*src).(int64); ok {
+		return formatDateWithInt64(&tmp, layout)
+	} else if tmp2, ok2 := (*src).(float64); ok2 {
+		tmpe := int64(tmp2)
+		return formatDateWithInt64(&tmpe, layout)
+	} else if tmp3, ok3 := (*src).(*time.Time); ok3 {
+		return tmp3.Format(layout)
+	} else {
+		return ""
+	}
+}

common_utils/elastic/elasticSim.go

@@ -0,0 +1,357 @@
+package elastic
+
+import (
+	"encoding/json"
+	"fmt"
+	es "gopkg.in/olivere/elastic.v2"
+	"log"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+)
+
+type Elastic struct {
+	S_esurl      string
+	I_size       int
+	Addrs        []string
+	Pool         chan *es.Client
+	lastTime     int64
+	lastTimeLock sync.Mutex
+	ntimeout     int
+}
+
+func (e *Elastic) InitElasticSize() {
+	e.Pool = make(chan *es.Client, e.I_size)
+	for _, s := range strings.Split(e.S_esurl, ",") {
+		e.Addrs = append(e.Addrs, s)
+	}
+	for i := 0; i < e.I_size; i++ {
+		client, _ := es.NewClient(es.SetURL(e.Addrs...), es.SetMaxRetries(2), es.SetSniff(false))
+		e.Pool <- client
+	}
+}
+
+//关闭连接
+func (e *Elastic) DestoryEsConn(client *es.Client) {
+	select {
+	case e.Pool <- client:
+		break
+	case <-time.After(time.Second * 1):
+		if client != nil {
+			client.Stop()
+		}
+		client = nil
+	}
+}
+
+func (e *Elastic) GetEsConn() *es.Client {
+	select {
+	case c := <-e.Pool:
+		if c == nil || !c.IsRunning() {
+			log.Println("new esclient.", len(e.Pool))
+			client, err := es.NewClient(es.SetURL(e.Addrs...),
+				es.SetMaxRetries(2), es.SetSniff(false))
+			if err == nil && client.IsRunning() {
+				return client
+			}
+		}
+		return c
+	case <-time.After(time.Second * 4):
+		//超时
+		e.ntimeout++
+		e.lastTimeLock.Lock()
+		defer e.lastTimeLock.Unlock()
+		//12秒后允许创建链接
+		c := time.Now().Unix() - e.lastTime
+		if c > 12 {
+			e.lastTime = time.Now().Unix()
+			log.Println("add client..", len(e.Pool))
+			c, _ := es.NewClient(es.SetURL(e.Addrs...), es.SetMaxRetries(2), es.SetSniff(false))
+			go func() {
+				for i := 0; i < 2; i++ {
+					client, _ := es.NewClient(es.SetURL(e.Addrs...), es.SetMaxRetries(2), es.SetSniff(false))
+					e.Pool <- client
+				}
+			}()
+			return c
+		}
+		return nil
+	}
+}
+
+func (e *Elastic) Get(index, itype, query string) *[]map[string]interface{} {
+	client := e.GetEsConn()
+	defer func() {
+		go e.DestoryEsConn(client)
+	}()
+	var res []map[string]interface{}
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
+		if err != nil {
+			log.Println("从ES查询出错", err.Error())
+			return nil
+		}
+		if searchResult.Hits != nil {
+			resNum := len(searchResult.Hits.Hits)
+			if resNum < 5000 {
+				res = make([]map[string]interface{}, resNum)
+				for i, hit := range searchResult.Hits.Hits {
+					parseErr := json.Unmarshal(*hit.Source, &res[i])
+					if parseErr == nil && hit.Highlight != nil && res[i] != nil {
+						res[i]["highlight"] = map[string][]string(hit.Highlight)
+					}
+				}
+			} else {
+				log.Println("查询结果太多，查询到：", resNum, "条")
+			}
+		}
+	}
+	return &res
+}
+
+//关闭elastic
+func (e *Elastic) Close() {
+	for i := 0; i < e.I_size; i++ {
+		cli := <-e.Pool
+		cli.Stop()
+		cli = nil
+	}
+	e.Pool = nil
+	e = nil
+}
+
+//获取连接
+//func (e *Elastic) GetEsConn() (c *es.Client) {
+//	defer util.Catch()
+//	select {
+//	case c = <-e.Pool:
+//		if c == nil || !c.IsRunning() {
+//			client, err := es.NewClient(es.SetURL(addrs...),
+//				es.SetMaxRetries(2), es.SetSniff(false))
+//			if err == nil && client.IsRunning() {
+//				return client
+//			}
+//			return nil
+//		}
+//		return
+//	case <-time.After(time.Second * 7):
+//		//超时
+//		ntimeout++
+//		log.Println("timeout times:", ntimeout)
+//		return nil
+//	}
+//}
+
+func (e *Elastic) BulkSave(index, itype string, obj *[]map[string]interface{}, isDelBefore bool) {
+	client := e.GetEsConn()
+	defer e.DestoryEsConn(client)
+	if client != nil {
+		req := client.Bulk()
+		for _, v := range *obj {
+			if isDelBefore {
+				req = req.Add(es.NewBulkDeleteRequest().Index(index).Type(itype).Id(fmt.Sprintf("%v", v["_id"])))
+			}
+			req = req.Add(es.NewBulkIndexRequest().Index(index).Type(itype).Doc(v))
+		}
+		_, err := req.Do()
+		if err != nil {
+			log.Println("批量保存到ES出错", err.Error())
+		}
+	}
+}
+
+//根据id删除索引对象
+func (e *Elastic) DelById(index, itype, id string) bool {
+	client := e.GetEsConn()
+	defer e.DestoryEsConn(client)
+	b := false
+	if client != nil {
+		var err error
+		_, err = client.Delete().Index(index).Type(itype).Id(id).Do()
+		if err != nil {
+			log.Println("更新检索出错:", err.Error())
+		} else {
+			b = true
+		}
+	}
+	return b
+}
+func (e *Elastic) GetNoLimit(index, itype, query string) *[]map[string]interface{} {
+	//log.Println("query  -- ", query)
+	client := e.GetEsConn()
+	defer e.DestoryEsConn(client)
+	var res []map[string]interface{}
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
+		if err != nil {
+			log.Println("从ES查询出错", err.Error())
+			return nil
+		}
+
+		if searchResult.Hits != nil {
+			resNum := len(searchResult.Hits.Hits)
+			res = make([]map[string]interface{}, resNum)
+			for i, hit := range searchResult.Hits.Hits {
+				json.Unmarshal(*hit.Source, &res[i])
+			}
+		}
+	}
+	return &res
+}
+func (e *Elastic) GetByIdField(index, itype, id, fields string) *map[string]interface{} {
+	client := e.GetEsConn()
+	defer e.DestoryEsConn(client)
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		query := `{"query":{"term":{"_id":"` + id + `"}}`
+		if len(fields) > 0 {
+			query = query + `,"_source":[` + fields + `]`
+		}
+		query = query + "}"
+		searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
+		if err != nil {
+			log.Println("从ES查询出错", err.Error())
+			return nil
+		}
+		var res map[string]interface{}
+		if searchResult.Hits != nil {
+			resNum := len(searchResult.Hits.Hits)
+			if resNum == 1 {
+				res = make(map[string]interface{})
+				for _, hit := range searchResult.Hits.Hits {
+					json.Unmarshal(*hit.Source, &res)
+				}
+				return &res
+			}
+		}
+	}
+	return nil
+}
+
+func (e *Elastic) Count(index, itype string, query interface{}) int64 {
+	client := e.GetEsConn()
+	defer e.DestoryEsConn(client)
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		var qq es.Query
+		if qi, ok2 := query.(es.Query); ok2 {
+			qq = qi
+		}
+		n, err := client.Count(index).Type(itype).Query(qq).Do()
+		if err != nil {
+			log.Println("统计出错", err.Error())
+		}
+
+		return n
+	}
+	return 0
+}
+
+//更新一个字段
+func (e *Elastic) BulkUpdateArr(index, itype string, update []map[string]string) {
+	client := e.GetEsConn()
+	defer e.DestoryEsConn(client)
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		for _, data := range update {
+			id := data["id"]
+			updateStr := data["updateStr"]
+			if id != "" && updateStr != "" {
+				_, err := client.Update().Index(index).Type(itype).Id(id).Script(updateStr).ScriptLang("groovy").Do()
+				if err != nil {
+					log.Println("更新检索出错:", err.Error())
+				}
+			} else {
+				log.Println("数据错误")
+			}
+		}
+	}
+}
+
+//更新多个字段
+func (e *Elastic) BulkUpdateMultipleFields(index, itype string, arrs [][]map[string]interface{}) {
+	client := e.GetEsConn()
+	defer e.DestoryEsConn(client)
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		for _, arr := range arrs {
+			id := arr[0]["id"].(string)
+			update := arr[1]["update"].([]string)
+			for _, str := range update {
+				_, err := client.Update().Index(index).Type(itype).Id(id).Script(str).ScriptLang("groovy").Do()
+				if err != nil {
+					log.Println("更新检索出错:", err.Error())
+				}
+			}
+		}
+	}
+}

common_utils/elastic/elasticutil.go

@@ -0,0 +1,1169 @@
+package elastic
+
+import (
+	"encoding/json"
+	"fmt"
+	es "gopkg.in/olivere/elastic.v2"
+	"log"
+	"net/url"
+	"reflect"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+//检索库服务地址
+var addrs []string
+var LocCity = map[string]string{}
+var SIZE = 30
+
+const (
+	QStr = `{"query":{"bool":{"must":[$and],"must_not":[],
+	"should":[$or],"minimum_should_match" : 1}}}`
+)
+
+var pool chan *es.Client
+var ntimeout int
+
+var syncPool sync.Pool
+
+//自定义HttpClient
+/**
+var httpclient = &http.Client{Transport: &http.Transport{
+	Dial: func(netw, addr string) (net.Conn, error) {
+		deadline := time.Now().Add(5000 * time.Millisecond)
+		c, err := net.DialTimeout(netw, addr, 10000*time.Millisecond)
+		if err != nil {
+			return nil, err
+		}
+		tcp_conn := c.(*net.TCPConn)
+		tcp_conn.SetKeepAlive(false)
+		tcp_conn.SetDeadline(deadline)
+		return tcp_conn, nil
+	},
+	DisableKeepAlives: true, //不保持，这样才能释放
+}}
+**/
+//var op = es.SetHttpClient(httpclient)
+var poolsize = int32(20)
+
+//n倍的池
+func InitElasticSize(addr string, size int) {
+	poolsize = int32(3 * size)
+	pool = make(chan *es.Client, poolsize)
+	for _, s := range strings.Split(addr, ",") {
+		addrs = append(addrs, s)
+	}
+	for i := 0; i < size; i++ {
+		client, _ := es.NewClient(es.SetURL(addrs...), es.SetMaxRetries(2), es.SetSniff(false))
+		pool <- client
+	}
+}
+
+//关闭连接
+func DestoryEsConn(client *es.Client) {
+	select {
+	case pool <- client:
+		break
+	case <-time.After(time.Second * 1):
+		if client != nil {
+			client.Stop()
+		}
+		client = nil
+	}
+}
+
+var (
+	lastTime     = int64(0)
+	lastTimeLock = &sync.Mutex{}
+)
+
+//获取连接
+
+func GetEsConn() *es.Client {
+	select {
+	case c := <-pool:
+		if c == nil || !c.IsRunning() {
+			log.Println("new esclient.", len(pool))
+			client, err := es.NewClient(es.SetURL(addrs...),
+				es.SetMaxRetries(2), es.SetSniff(false))
+			if err == nil && client.IsRunning() {
+				return client
+			}
+		}
+		return c
+	case <-time.After(time.Second * 4):
+		//超时
+		ntimeout++
+		lastTimeLock.Lock()
+		defer lastTimeLock.Unlock()
+		//12秒后允许创建链接
+		c := time.Now().Unix() - lastTime
+		if c > 12 {
+			lastTime = time.Now().Unix()
+			log.Println("add client..", len(pool))
+			c, _ := es.NewClient(es.SetURL(addrs...), es.SetMaxRetries(2), es.SetSniff(false))
+			go func() {
+				for i := 0; i < 2; i++ {
+					client, _ := es.NewClient(es.SetURL(addrs...), es.SetMaxRetries(2), es.SetSniff(false))
+					pool <- client
+				}
+			}()
+			return c
+		}
+		return nil
+	}
+}
+
+//通用查询
+//{"query": {"bool":{"must":[{"query_string":{"default_field":"name","query":"*"}}]}}}
+//{"query":{"bool":{"must":{"match":{"content":{"query":"fulltextsearch","operator":"and"}}},"should":[{"match":{"content":{"query":"Elasticsearch","boost":3}}},{"match":{"content":{"query":"Lucene","boost":2}}}]}}}
+//prefix
+//{"query":{"match":{"title":{"query":"brownfox","operator":"and"}}}} //默认为or
+//{"query":{"multi_match":{"query":"PolandStreetW1V","type":"most_fields","fields":["*_street","city^2","country","postcode"]}}}
+//{"query":{"wildcard":{"postcode":"W?F*HW"}}}
+//{"query":{"regexp":{"postcode":"W[0-9].+"}}}
+//{"query":{"filtered":{"filter":{"range":{"price":{"gte":10000}}}}},"aggs":{"single_avg_price":{"avg":{"field":"price"}}}}
+//{"query":{"match":{"make":"ford"}},"aggs":{"colors":{"terms":{"field":"color"}}}}//查fork有几种颜色
+//过滤器不会计算相关度的得分，所以它们在计算上更快一些
+//{"query":{"filtered":{"query":{"match_all":{}},"filter":{"range":{"balance":{"gte":20000,"lte":30000}}}}}}
+//{"query":{"match_all":{}},"from":10,"size":10,"_source":["account_number","balance"],"sort":{"balance":{"order":"desc"}}}
+//{"query":{"match_phrase":{"address":"milllane"}}}和match不同会去匹配整个短语，相当于must[]
+func Get(index, itype, query string) *[]map[string]interface{} {
+	log.Println("query  -- ", query)
+	client := GetEsConn()
+	defer func() {
+		go DestoryEsConn(client)
+	}()
+	var res []map[string]interface{}
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
+		if err != nil {
+			log.Println("从ES查询出错", err.Error())
+			return nil
+		}
+
+		if searchResult.Hits != nil {
+			resNum := len(searchResult.Hits.Hits)
+			if resNum < 5000 {
+				res = make([]map[string]interface{}, resNum)
+				for i, hit := range searchResult.Hits.Hits {
+					//d := json.NewDecoder(bytes.NewBuffer(*hit.Source))
+					//d.UseNumber()
+					//d.Decode(&res[i])
+					parseErr := json.Unmarshal(*hit.Source, &res[i])
+					if parseErr == nil && hit.Highlight != nil && res[i] != nil {
+						res[i]["highlight"] = map[string][]string(hit.Highlight)
+					}
+				}
+			} else {
+				log.Println("查询结果太多，查询到：", resNum, "条")
+			}
+
+		}
+	}
+	return &res
+}
+func GetOA(index, itype, query string) (*[]map[string]interface{}, int) {
+	//log.Println("query  -- ", query)
+	client := GetEsConn()
+	defer func() {
+		go DestoryEsConn(client)
+	}()
+	var res []map[string]interface{}
+	var resNum int
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
+		if err != nil {
+			log.Println("从ES查询出错", err.Error())
+			return nil, 0
+		}
+
+		if searchResult.Hits != nil {
+			resNum = len(searchResult.Hits.Hits)
+			if resNum < 5000 {
+				res = make([]map[string]interface{}, resNum)
+				for i, hit := range searchResult.Hits.Hits {
+					//d := json.NewDecoder(bytes.NewBuffer(*hit.Source))
+					//d.UseNumber()
+					//d.Decode(&res[i])
+					parseErr := json.Unmarshal(*hit.Source, &res[i])
+					if parseErr == nil && hit.Highlight != nil && res[i] != nil {
+						res[i]["highlight"] = map[string][]string(hit.Highlight)
+					}
+				}
+			} else {
+				log.Println("查询结果太多，查询到：", resNum, "条")
+			}
+
+		}
+	}
+	return &res, resNum
+}
+
+func GetNoLimit(index, itype, query string) *[]map[string]interface{} {
+	//log.Println("query  -- ", query)
+	client := GetEsConn()
+	defer DestoryEsConn(client)
+	var res []map[string]interface{}
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
+		if err != nil {
+			log.Println("从ES查询出错", err.Error())
+			return nil
+		}
+
+		if searchResult.Hits != nil {
+			resNum := len(searchResult.Hits.Hits)
+			res = make([]map[string]interface{}, resNum)
+			for i, hit := range searchResult.Hits.Hits {
+				json.Unmarshal(*hit.Source, &res[i])
+			}
+		}
+	}
+	return &res
+}
+
+//分页查询
+//{"name":"张三","$and":[{"age":{"$gt":10}},{"age":{"$lte":20}}]}
+//fields直接是 `"_id","title"`
+func GetPage(index, itype, query, order, field string, start, limit int) *[]map[string]interface{} {
+	return Get(index, itype, MakeQuery(query, order, field, start, limit))
+}
+
+//openapi
+func GetOAPage(index, itype, query, order, field string, start, limit int) (*[]map[string]interface{}, int) {
+	return GetOA(index, itype, MakeQuery(query, order, field, start, limit))
+}
+
+var SR = strings.Replace
+
+func MakeQuery(query, order, fileds string, start, limit int) string {
+	log.Println(query)
+	res := AnalyQuery(query, "", QStr)
+	log.Println(len(res), query)
+	if len(res) > 10 {
+		res = SR(SR(SR(SR(res, ",$and", "", -1), "$and", "", -1), ",$or", "", -1), "$or", "", -1)
+		log.Println("1", res)
+		if len(fileds) > 0 {
+			//"_source":["account_number","balance"]
+			res = res[:len(res)-1] + `,"_source":[` + fileds + "]}"
+		}
+		log.Println("2", res)
+		//{"name":-1,"age":1}
+		if len(order) > 0 {
+			res = res[:len(res)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
+		}
+		log.Println("3", res)
+		if start > -1 {
+			res = res[:len(res)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
+		}
+		log.Println("4", res)
+		return res
+	}
+	return ""
+}
+
+//{"name":"aaa"}
+func AnalyQuery(query interface{}, parent string, result string) string {
+	m := make(map[string]interface{})
+	if q1, ok := query.(string); ok {
+		json.Unmarshal([]byte(q1), &m)
+	} else if q2, ok2 := query.(map[string]interface{}); ok2 {
+		m = q2
+	}
+	if len(parent) == 0 {
+		for k, v := range m {
+			if k == "$and" || k == "$or" {
+				temps := ""
+				if map1, ok := v.([]interface{}); ok {
+					for i := 0; i < len(map1); i++ {
+						temps += "," + AnalyQuery(map1[i], k, "")
+					}
+				}
+				if len(temps) > 0 {
+					temps = temps[1:]
+				}
+				result = SR(result, k, temps+","+k, 1)
+			} else {
+				switch reflect.TypeOf(v).String() {
+				case "string":
+					if strings.Index(k, "TERM_") == 0 {
+						result = SR(result, "$and", `{"term":{"`+SR(k, "TERM_", "", 1)+`":"`+fmt.Sprintf("%v", v)+`"}},$and`, 1)
+					} else {
+						result = SR(result, "$and", `{"query_string":{"default_field":"`+k+`","query":"`+fmt.Sprintf("%v", v)+`"}},$and`, 1)
+					}
+				case "int", "int8", "int32", "int64", "float32", "float64":
+					if strings.Index(k, "TERM_") == 0 {
+						result = SR(result, "$and", `{"term":{"`+SR(k, "TERM_", "", 1)+`":`+fmt.Sprintf("%v", v)+`}},$and`, 1)
+					} else {
+						result = SR(result, "$and", `{"query_string":{"default_field":"`+k+`","query":`+fmt.Sprintf("%v", v)+`}},$and`, 1)
+					}
+				default:
+					result = SR(result, "$and", AnalyQuery(v, k, "")+",$and", 1)
+				}
+			}
+		}
+		return result
+	} else {
+		for k, v := range m {
+			if k == "$in" {
+				s := ""
+				if map1, ok := v.([]interface{}); ok {
+					for i := 0; i < len(map1); i++ {
+						s += "," + `"` + fmt.Sprintf("%v", map1[i]) + `"`
+					}
+				}
+				if len(s) > 0 {
+					s = s[1:]
+				}
+				return `{"terms":{"` + parent + `":[` + s + `]}}`
+			} else if strings.Contains(k, "$lt") || strings.Contains(k, "$gt") {
+				return `{"range":{"` + parent + `":{"` + SR(k, "$", "", 1) + `":` + fmt.Sprintf("%v", v) + `}}}`
+			} else {
+				switch reflect.TypeOf(v).String() {
+				case "string":
+					if strings.Index(k, "TERM_") == 0 {
+						return `{"term":{"` + SR(k, "TERM_", "", 1) + `":"` + fmt.Sprintf("%v", v) + `"}}`
+					} else {
+						return `{"query_string":{"default_field":"` + k + `","query":"` + fmt.Sprintf("%v", v) + `"}}`
+					}
+				case "int", "int8", "int32", "int64", "float32", "float64":
+					if strings.Index(k, "TERM_") == 0 {
+						return `{"term":{"` + SR(k, "TERM_", "", 1) + `":` + fmt.Sprintf("%v", v) + `}}`
+					} else {
+						return `{"query_string":{"default_field":"` + k + `","query":` + fmt.Sprintf("%v", v) + `}}`
+					}
+				default:
+					return AnalyQuery(v, k, result)
+				}
+			}
+		}
+	}
+	return result
+}
+func GetByIdField(index, itype, id, fields string) *map[string]interface{} {
+	client := GetEsConn()
+	defer DestoryEsConn(client)
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		query := `{"query":{"term":{"_id":"` + id + `"}}`
+		if len(fields) > 0 {
+			query = query + `,"_source":[` + fields + `]`
+		}
+		query = query + "}"
+		searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
+		if err != nil {
+			log.Println("从ES查询出错", err.Error())
+			return nil
+		}
+		var res map[string]interface{}
+		if searchResult.Hits != nil {
+			resNum := len(searchResult.Hits.Hits)
+			if resNum == 1 {
+				res = make(map[string]interface{})
+				for _, hit := range searchResult.Hits.Hits {
+					json.Unmarshal(*hit.Source, &res)
+				}
+				return &res
+			}
+		}
+	}
+	return nil
+}
+
+//根据id来查询文档
+func GetById(index, itype string, ids ...string) *[]map[string]interface{} {
+	client := GetEsConn()
+	defer DestoryEsConn(client)
+	var res []map[string]interface{}
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		query := es.NewIdsQuery().Ids(ids...)
+		searchResult, err := client.Search().Index(index).Type(itype).Query(&query).Do()
+		if err != nil {
+			log.Println("从ES查询出错", err.Error())
+			return nil
+		}
+
+		if searchResult.Hits != nil {
+			resNum := len(searchResult.Hits.Hits)
+			if resNum < 5000 {
+				res = make([]map[string]interface{}, resNum)
+				for i, hit := range searchResult.Hits.Hits {
+					json.Unmarshal(*hit.Source, &res[i])
+				}
+			} else {
+				log.Println("查询结果太多，查询到：", resNum, "条")
+			}
+
+		}
+	}
+	return &res
+}
+
+//删除某个索引，根据查询
+func Del(index, itype string, query interface{}) bool {
+	client := GetEsConn()
+	defer DestoryEsConn(client)
+	b := false
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		var err error
+		if qi, ok2 := query.(es.Query); ok2 {
+			_, err = client.DeleteByQuery().Index(index).Type(itype).Query(qi).Do()
+		}
+		if err != nil {
+			log.Println("删除索引出错:", err.Error())
+		} else {
+			b = true
+		}
+	}
+	return b
+}
+
+//根据语句更新对象
+func Update(index, itype, id string, updateStr string) bool {
+	client := GetEsConn()
+	defer DestoryEsConn(client)
+	b := false
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		var err error
+		_, err = client.Update().Index(index).Type(itype).Id(id).Script(updateStr).ScriptLang("groovy").Do()
+		if err != nil {
+			log.Println("更新检索出错:", err.Error())
+		} else {
+			b = true
+		}
+	}
+	return b
+}
+
+func BulkUpdate(index, itype string, ids []string, updateStr string) {
+	client := GetEsConn()
+	defer DestoryEsConn(client)
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		for _, id := range ids {
+			_, err := client.Update().Index(index).Type(itype).Id(id).Script(updateStr).ScriptLang("groovy").Do()
+			if err != nil {
+				log.Println("更新检索出错:", err.Error())
+			}
+		}
+	}
+}
+
+func BulkUpdateArr(index, itype string, update []map[string]string) {
+	client := GetEsConn()
+	defer DestoryEsConn(client)
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		for _, data := range update {
+			id := data["id"]
+			updateStr := data["updateStr"]
+			if id != "" && updateStr != "" {
+				_, err := client.Update().Index(index).Type(itype).Id(id).Script(updateStr).ScriptLang("groovy").Do()
+				if err != nil {
+					log.Println("更新检索出错:", err.Error())
+				}
+			} else {
+				log.Println("数据错误")
+			}
+		}
+
+	}
+}
+
+//根据id删除索引对象
+func DelById(index, itype, id string) bool {
+	client := GetEsConn()
+	defer DestoryEsConn(client)
+	b := false
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		var err error
+		_, err = client.Delete().Index(index).Type(itype).Id(id).Do()
+		if err != nil {
+			log.Println("更新检索出错:", err.Error())
+		} else {
+			b = true
+		}
+	}
+	return b
+}
+
+//把地市代码转为地市
+func getLoc(code string, res *map[string]string) (loc string) {
+	switch len(code) {
+	case 6:
+		loc = (*res)[code[:2]] + " " + (*res)[code[:4]] + " " + (*res)[code]
+		break
+	case 4:
+		loc = (*res)[code[:2]] + " " + (*res)[code]
+		break
+	case 2:
+		loc = (*res)[code]
+		break
+	}
+	return
+}
+
+//把地市代码转为地市
+func Loop(m interface{}, res *map[string]string) {
+	m1, ok := m.([]interface{})
+	if !ok {
+		m2, _ := m.([]map[string]interface{})
+		for i := 0; i < len(m2); i++ {
+			ms := m2[i]
+			(*res)[fmt.Sprintf("%1.0f", ms["k"])] = fmt.Sprintf("%s", ms["n"])
+			s := ms["s"]
+			if nil != s {
+				mss, _ := s.([]interface{})
+				if nil != mss {
+					Loop(mss, res)
+				}
+			}
+		}
+	} else {
+		for i := 0; i < len(m1); i++ {
+			ms, _ := m1[i].(map[string]interface{})
+			(*res)[fmt.Sprintf("%1.0f", ms["k"])] = fmt.Sprintf("%s", ms["n"])
+			s := ms["s"]
+			if nil != s {
+				mss, _ := s.([]interface{})
+				if nil != mss {
+					Loop(mss, res)
+				}
+			}
+		}
+	}
+}
+
+func ConverData(ent *map[string]interface{}) map[string]interface{} {
+	tmp := *ent
+	id64, _ := tmp["ID"].(int64)
+	ids := fmt.Sprintf("%d", id64)
+	tmp2 := make(map[string]interface{})
+	tmp2["ID"] = ids
+	tmp2["_id"] = tmp["_id"]
+	tmp2["Area"] = tmp["Area"]
+	tmp2["LeRep"] = tmp["LeRep"]
+	tmp2["RegNo"] = tmp["RegNo"]
+	tmp2["EntType"] = tmp["EntType"]
+	tmp2["EntName"] = tmp["EntName"]
+	tmp2["EntTypeName"] = tmp["EntTypeName"]
+	tmp2["Dom"] = tmp["Dom"]
+	tmp2["EstDate"] = tmp["EstDate"]
+	tmp2["OpStateName"] = tmp["OpStateName"]
+	tmp2["OpScope"] = tmp["OpScope"]
+	tmp2["OpState"] = tmp["OpState"]
+	tmp2["s_submitid"] = tmp["s_submitid"]
+	tmp2["l_submittime"] = tmp["l_submittime"]
+	tmp2["s_submitname"] = tmp["s_submitname"]
+	tmp2["RegCapCurName"] = tmp["RegCapCurName"]
+	//增加营业状态排序
+	if tmp2["OpState"] == "06" {
+		tmp2["OpSint"] = true
+	} else {
+		tmp2["OpSint"] = false
+	}
+	tmp2["OpLocDistrict"] = tmp["OpLocDistrict"]
+	//增加代码转名称
+	tmpLoc, _ := tmp["OpLocDistrict"].(string)
+	tmp2["OpLocDistrictName"] = getLoc(tmpLoc, &LocCity)
+
+	tmp2["RecCap"] = tmp["RecCap"]
+	tmp2["RegCap"] = tmp["RegCap"]
+	tmp2["IndustryPhy"] = tmp["IndustryPhy"]
+	tmp2["IndustryPhyName"] = tmp["IndustryPhyName"]
+	tmp2["RegOrg"] = tmp["RegOrg"]
+	tmp2["RegOrgName"] = tmp["RegOrgName"]
+	tmp2["Tel"] = tmp["Tel"]
+	tmp2["CompForm"] = tmp["CompForm"]
+	tmp2["CompFormName"] = tmp["CompFormName"]
+	//增加异常名录标记 Ycml可能是bool也可能是string
+	Ycmlb, _ := tmp["Ycml"].(bool)
+	Ycmls, _ := tmp["Ycml"].(string)
+	if Ycmlb || Ycmls == "1" {
+		tmp2["Ycml"] = true
+	} else {
+		tmp2["Ycml"] = false
+	}
+	//增加年报联系信息
+	if tmp["Nb_email"] != nil {
+		tmp2["Nb_email"] = tmp["Nb_email"]
+	}
+	if tmp["Nb_tel"] != nil {
+		tmp2["Nb_tel"] = tmp["Nb_tel"]
+	}
+	if tmp["Nb_addr"] != nil {
+		tmp2["Nb_addr"] = tmp["Nb_addr"]
+	}
+
+	s_synopsis := tmp["s_synopsis"]
+	if s_synopsis == nil {
+		s_synopsis = ""
+	}
+	tmp2["s_synopsis"] = s_synopsis //企业简介
+
+	//股东
+	stock := getStock(tmp["investor"])
+	tmp2["stock"] = stock
+
+	tmp2["LegCerNO"] = tmp["LegCerNO"]
+	if tmp["s_microwebsite"] != nil {
+		tmp2["s_microwebsite"] = tmp["s_microwebsite"]
+	}
+
+	tmp2["SourceType"] = tmp["SourceType"] //数据来源
+	s_servicenames := tmp["s_servicenames"]
+	if s_servicenames == nil {
+		s_servicenames = ""
+	}
+	tmp2["s_servicenames"] = s_servicenames //服务名称
+	s_action := tmp["s_action"]
+	if s_action == nil {
+		s_action = "N"
+	}
+	tmp2["s_action"] = s_action
+	tmp2["s_persion"] = tmp["s_persion"]
+	tmp2["s_mobile"] = tmp["s_mobile"]
+	tmp2["s_enturl"] = tmp["s_enturl"]
+	tmp2["s_weixin"] = tmp["s_weixin"]
+	tmp2["s_avatar"] = tmp["s_avatar"]
+	return tmp2
+}
+
+func getStock(obj interface{}) string {
+	stock := ""
+	if ns, ok := obj.([]interface{}); ok {
+		stock = " "
+		for _, ns1 := range ns {
+			if nn, ok1 := ns1.(map[string]interface{}); ok1 {
+				tmp := fmt.Sprintf("%s", nn["Inv"])
+				if strings.Index(stock, tmp) < 0 {
+					stock += tmp + " "
+				}
+			}
+		}
+	}
+	return stock
+}
+
+func BulkSave(index, itype string, obj *[]map[string]interface{}, isDelBefore bool) {
+	client := GetEsConn()
+	defer DestoryEsConn(client)
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		req := client.Bulk()
+		for _, v := range *obj {
+			if isDelBefore {
+				req = req.Add(es.NewBulkDeleteRequest().Index(index).Type(itype).Id(fmt.Sprintf("%v", v["_id"])))
+			}
+			req = req.Add(es.NewBulkIndexRequest().Index(index).Type(itype).Doc(v))
+		}
+		_, err := req.Do()
+		if err != nil {
+			log.Println("批量保存到ES出错", err.Error())
+		}
+	}
+}
+
+func Count(index, itype string, query interface{}) int64 {
+	client := GetEsConn()
+	defer DestoryEsConn(client)
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		var qq es.Query
+		if qi, ok2 := query.(es.Query); ok2 {
+			qq = qi
+		}
+		n, err := client.Count(index).Type(itype).Query(qq).Do()
+		if err != nil {
+			log.Println("统计出错", err.Error())
+		}
+
+		return n
+	}
+	return 0
+}
+
+//ngram精确查询
+/*
+{
+  "query": {
+    "bool": {
+      "should": [
+        {
+	"bool":{
+	  "must":[
+	  {     "multi_match": {
+            "query": "智能",
+            "type": "phrase",
+            "fields": [
+              "title"
+            ],
+           "analyzer": "my_ngram"
+          }
+        },{
+          "multi_match": {
+            "query": "机器",
+            "type": "phrase",
+            "fields": [
+              "title"
+            ],
+           "analyzer": "my_ngram"
+          }
+        },{
+          "multi_match": {
+            "query": "2016",
+            "type": "phrase",
+            "fields": [
+              "title"
+            ],
+           "analyzer": "my_ngram"
+          }
+	  }
+	  ]
+	}
+        },
+
+{
+	"bool":{
+	  "must":[
+	  {          "multi_match": {
+            "query": "河南",
+            "type": "phrase",
+            "fields": [
+              "title"
+            ],
+           "analyzer": "my_ngram"
+          }
+        },{
+          "multi_match": {
+            "query": "工商",
+            "type": "phrase",
+            "fields": [
+              "title"
+            ],
+           "analyzer": "my_ngram"
+          }
+        },{
+          "multi_match": {
+            "query": "2016",
+            "type": "phrase",
+            "fields": [
+              "title"
+            ],
+           "analyzer": "my_ngram"
+          }
+	  }
+	  ]
+	}
+        }
+      ],"minimum_should_match": 1
+    }
+  },
+  "_source": [
+    "_id",
+    "title"
+  ],
+  "from": 0,
+  "size": 10,
+  "sort": [{
+      "publishtime": "desc"
+    }]
+}
+
+*/
+//"2016+智能+办公,"河南+工商"
+//["2016+智能+办公","河南+工商"]
+//QStr = `{"query":{"bool":{should":[$or],"minimum_should_match" : 1}}}`
+//{"bool":{"must":[]}}
+//{"multi_match": {"query": "$word","type": "phrase", "fields": [$field],"analyzer": "my_ngram"}}
+//"highlight": {"pre_tags": [""],"post_tags": [""],"fields": {"detail": {"fragment_size": 1,"number_of_fragments": 1},"title": {"fragment_size": 1,"number_of_fragments": 1}}}
+const (
+	//此处最后少一个}，正好NgramStr取[1:]多一个}
+	FilterQuery     = `{"query": {"filtered": {"filter": {"bool": {"must": [%s]}},%s}}`
+	NgramStr        = `{"query":{"bool":{"must":[%s],"should":[%s],"minimum_should_match": 1}}}`
+	NgramMust       = `{"bool":{"must":[%s]}}`
+	NgramMustAndNot = `{"bool":{"must":[%s],"must_not":[%s]}}`
+	minq            = `{"multi_match": {"query": "%s","type": "phrase", "fields": [%s]}}`
+	HL              = `"highlight": {"pre_tags": [""],"post_tags": [""],"fields": {%s}}`
+	highlightStr    = `%s: {"fragment_size": %d,"number_of_fragments": 1}`
+
+	FilterQuery_New  = `{"query":{"bool":{"must": [%s%s%s],"should":[]}}}`
+	MatchQueryString = `{"match": {%s: { "query":"%s", "operator": "and"}}}`
+	HL_New           = `"highlight": {"pre_tags": ["<HL>"],"post_tags": ["<HL>"],"fields": {%s}}`
+
+	//数据查询高亮标记2019-07-10
+	HL_MP = `"highlight": {"pre_tags": ["<HL>"],"post_tags": ["</HL>"],"fields": {%s}}`
+
+	ik_highlightStr = `%s: {"fragment_size": %d,"number_of_fragments": 1,"require_field_match": true}`
+	IK_pre_tags     = `<font class=\"es-highlight\">`
+	IK_post_tags    = `</font>`
+	HL_IK           = `"highlight": {"pre_tags": ["` + IK_pre_tags + `"],"post_tags": ["` + IK_post_tags + `"],"fields": {%s}}`
+)
+
+type KeyConfig struct {
+	Keys      []string `json:"key"`
+	NotKeys   []string `json:"notkey"`
+	InfoTypes []string `json:"infotype"`
+	Areas     []string `json:"area"`
+}
+
+//替换了"号
+func GetResForJY(index, itype string, keys []KeyConfig, allquery, findfields, SortQuery, fields string, start, limit int) *[]map[string]interface{} {
+	if len(keys) > 0 {
+		qstr := ""
+		new_minq := fmt.Sprintf(minq, "%s", findfields)
+		not_new_minq := fmt.Sprintf(minq, "%s", findfields) //排除词只查询标题
+		musts := []string{}
+		for _, qs_words := range keys {
+			mq := []string{}
+			notmq := []string{}
+			for _, qs_word := range qs_words.Keys {
+				mq = append(mq, fmt.Sprintf(new_minq, ReplaceYH(qs_word)))
+				/*
+					qs := AnalyzerWord("bidding", qs_word)
+					for _, qw := range qs {
+						mq = append(mq, fmt.Sprintf(new_minq, ReplaceYH(qw)))
+					}
+				*/
+			}
+			for _, qs_word := range qs_words.NotKeys {
+				notmq = append(notmq, fmt.Sprintf(not_new_minq, ReplaceYH(qs_word)))
+			}
+			if len(qs_words.Areas) > 0 {
+				mq = append(mq, fmt.Sprintf(`{"terms":{"area":["%s"]}}`, strings.Join(qs_words.Areas, `","`)))
+			}
+			if len(qs_words.InfoTypes) > 0 {
+				mq = append(mq, fmt.Sprintf(`{"terms":{"toptype":["%s"]}}`, strings.Join(qs_words.InfoTypes, `","`)))
+			}
+			musts = append(musts, fmt.Sprintf(NgramMustAndNot, strings.Join(mq, ","), strings.Join(notmq, ",")))
+		}
+		qstr = fmt.Sprintf(NgramStr, "", strings.Join(musts, ","))
+
+		qstr = fmt.Sprintf(FilterQuery, allquery, qstr[1:])
+		ws := []string{}
+		for _, w := range strings.Split(findfields, ",") {
+			ws = append(ws, fmt.Sprintf(highlightStr, w, 1))
+		}
+		qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL, strings.Join(ws, ",")) + `}`
+		if len(fields) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
+		}
+		if len(SortQuery) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"sort":` + SortQuery + `}`
+		}
+		if start > -1 {
+			qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
+		}
+		//log.Println("jy-ngram-find", qstr)
+		return Get(index, itype, qstr)
+	} else {
+		return nil
+	}
+}
+
+func ReplaceYH(src string) (rpl string) {
+	return strings.Replace(src, `"`, `\"`, -1)
+}
+
+//
+func GetAllByNgram(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
+	if qstr != "" {
+		if highlight {
+			ws := []string{}
+			for _, w := range strings.Split(findfields, ",") {
+				ws = append(ws, fmt.Sprintf(highlightStr, w, count))
+			}
+			qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL, strings.Join(ws, ",")) + `}`
+		}
+		if len(fields) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
+		}
+		if len(order) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
+		}
+		if start > -1 {
+			qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
+		}
+		log.Println("GetAllByNgram：", qstr)
+		return Get(index, itype, qstr)
+	} else {
+		return nil
+	}
+}
+
+//数据标记2019-07-10
+func GetAllByNgram_MP(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
+	if qstr != "" {
+		if highlight {
+			ws := []string{}
+			for _, w := range strings.Split(findfields, ",") {
+				ws = append(ws, fmt.Sprintf(highlightStr, w, count))
+			}
+			qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL_MP, strings.Join(ws, ",")) + `}`
+		}
+		if len(fields) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
+		}
+		if len(order) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
+		}
+		if start > -1 {
+			qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
+		}
+		//		log.Println("GetAllByNgram：", qstr)
+		return Get(index, itype, qstr)
+	} else {
+		return nil
+	}
+}
+
+//ik 分词
+func GetAllByIk(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
+	if qstr != "" {
+		if highlight {
+			ws := []string{}
+			for _, w := range strings.Split(findfields, ",") {
+				ws = append(ws, fmt.Sprintf(ik_highlightStr, w, count))
+			}
+			qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL_IK, strings.Join(ws, ",")) + `}`
+		}
+		if len(fields) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
+		}
+		if len(order) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
+		}
+		if start > -1 {
+			qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
+		}
+		//log.Println("GetAllByNgram：", qstr)
+		return Get(index, itype, qstr)
+	} else {
+		return nil
+	}
+}
+
+//分词
+func AnalyzerWord(index, word string) (result []string) {
+	client := GetEsConn()
+	defer DestoryEsConn(client)
+	result = []string{}
+	p := url.Values{}
+	p["text"] = []string{word}
+	p["analyzer"] = []string{"ik"}
+	by, err := client.PerformRequest("GET", "/"+index+"/_analyze", p, nil)
+	if err != nil {
+		log.Println("AnalyzerWord Error：", err)
+		return
+	}
+	b, err := by.Body.MarshalJSON()
+	if err != nil {
+		log.Println("AnalyzerWord MarshalJSON Error：", err)
+		return
+	}
+	var res map[string][]map[string]interface{}
+	err = json.Unmarshal(b, &res)
+	if err != nil {
+		log.Println("AnalyzerWord Unmarshal Error：", err)
+		return
+	}
+	if res == nil {
+		return
+	}
+	for _, v := range res["tokens"] {
+		token, _ := v["token"].(string)
+		if token != "" {
+			result = append(result, token)
+		}
+	}
+	return
+}
+
+func GetResForJYView(index, itype string, keys []KeyConfig, allquery, findfields, SortQuery, fields string, start, limit int) *[]map[string]interface{} {
+	if len(keys) > 0 {
+		qstr := ""
+		new_minq := fmt.Sprintf(minq, "%s", findfields)
+		not_new_minq := fmt.Sprintf(minq, "%s", findfields) //排除词只查询标题
+		musts := []string{}
+		for _, qs_words := range keys {
+			mq := []string{}
+			notmq := []string{}
+			for _, qs_word := range qs_words.Keys {
+				mq = append(mq, fmt.Sprintf(new_minq, ReplaceYH(qs_word)))
+			}
+			for _, qs_word := range qs_words.NotKeys {
+				notmq = append(notmq, fmt.Sprintf(not_new_minq, ReplaceYH(qs_word)))
+			}
+			if len(qs_words.Areas) > 0 {
+				mq = append(mq, fmt.Sprintf(`{"terms":{"area":["%s"]}}`, strings.Join(qs_words.Areas, `","`)))
+			}
+			if len(qs_words.InfoTypes) > 0 {
+				mq = append(mq, fmt.Sprintf(`{"terms":{"toptype":["%s"]}}`, strings.Join(qs_words.InfoTypes, `","`)))
+			}
+			musts = append(musts, fmt.Sprintf(NgramMustAndNot, strings.Join(mq, ","), strings.Join(notmq, ",")))
+		}
+		qstr = fmt.Sprintf(NgramStr, "", strings.Join(musts, ","))
+
+		qstr = fmt.Sprintf(FilterQuery, allquery, qstr[1:])
+		ws := []string{}
+		for _, w := range strings.Split(findfields, ",") {
+			ws = append(ws, fmt.Sprintf(highlightStr, w, 1))
+		}
+		qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL, strings.Join(ws, ",")) + `}`
+		if len(fields) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
+		}
+		if len(SortQuery) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"sort":` + SortQuery + `}`
+		}
+		if start > -1 {
+			qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
+		}
+		return Get(index, itype, qstr)
+	} else {
+		return nil
+	}
+}

common_utils/encryptarticle.go

@@ -0,0 +1,60 @@
+package util
+
+import (
+	"net/url"
+	"strings"
+)
+
+//正文
+var SE = &SimpleEncrypt{Key: "topnet2015topnet2015"}
+var SE2 = &SimpleEncrypt{Key: "2017jianyu"}
+
+//百度
+var BSE = &SimpleEncrypt{Key: "HNtopnet2017jy"}
+var BSE2 = &SimpleEncrypt{Key: "TOP2017jianyu"}
+
+//订阅推送邮件
+var ESE = &SimpleEncrypt{Key: "PEjy2018topnet"}
+var ESE2 = &SimpleEncrypt{Key: "TopnetJy2018Pe"}
+
+//通用加密
+func CommonEncodeArticle(stype string, keys ...string) (id string) {
+	switch stype {
+	case "content":
+		id = BEncodeArticleId2ByCheck("A", SE, SE2, keys...)
+	case "bdprivate":
+		id = BEncodeArticleId2ByCheck("B", BSE, BSE2, keys...)
+	case "mailprivate":
+		id = BEncodeArticleId2ByCheck("C", ESE, ESE2, keys...)
+	}
+	return
+}
+
+//通用解密
+func CommonDecodeArticle(stype string, id string) (res []string) {
+	switch stype {
+	case "content":
+		res = BDecodeArticleId2ByCheck(id, SE, SE2)
+	case "bdprivate":
+		res = BDecodeArticleId2ByCheck(id, BSE, BSE2)
+	case "mailprivate":
+		res = BDecodeArticleId2ByCheck(id, ESE, ESE2)
+	}
+	return
+}
+
+//短地址加密,二次加密带校验和
+func BEncodeArticleId2ByCheck(h string, s1, s2 *SimpleEncrypt, keys ...string) string {
+	kstr := strings.Join(keys, ",")
+	kstr = s1.EncodeStringByCheck(kstr)
+	return url.QueryEscape(h + GetLetterRandom(2) + s2.EncodeStringByCheck(kstr))
+}
+
+//短地址解密,二次解密带校验和
+func BDecodeArticleId2ByCheck(id string, s1, s2 *SimpleEncrypt) []string {
+	if !strings.Contains(id, "+") { //新加密算法解密
+		id, _ = url.QueryUnescape(id)
+	}
+	kstr := s2.DecodeStringByCheck(id[3:])
+	return strings.Split(s1.DecodeStringByCheck(kstr), ",")
+}

common_utils/go.mod

@@ -0,0 +1,18 @@
+module common_utils
+
+go 1.14
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0
+	github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
+	github.com/fortytw2/leaktest v1.3.0 // indirect
+	github.com/go-sql-driver/mysql v1.5.0
+	github.com/gomodule/redigo v1.8.9
+	github.com/nsqio/go-nsq v1.1.0
+	go.mongodb.org/mongo-driver v1.4.1
+	go.uber.org/zap v1.21.0
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+	gopkg.in/olivere/elastic.v2 v2.0.61
+
+)

common_utils/go.sum

@@ -0,0 +1,179 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.29.15 h1:0ms/213murpsujhsnxnNKNeVouW60aJqSd992Ks3mxs=
+github.com/aws/aws-sdk-go v1.29.15/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
+github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
+github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
+github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
+github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
+github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
+github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
+github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
+github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
+github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
+github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
+github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
+github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
+github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
+github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
+github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
+github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
+github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
+github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M=
+github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
+github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo=
+github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.4.1 h1:38NSAyDPagwnFpUA/D5SFgbugUYR3NzYRNa4Qk9UxKs=
+go.mongodb.org/mongo-driver v1.4.1/go.mod h1:llVBH2pkj9HywK0Dtdt6lDikOjFLbceHVu/Rc0iMKLs=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
+go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/olivere/elastic.v2 v2.0.61 h1:7cpl3MW8ysa4GYFBXklpo5mspe4NK0rpZTdyZ+QcD4U=
+gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

common_utils/log/log.go

@@ -0,0 +1,230 @@
+package log
+
+import (
+	"io"
+	"os"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"gopkg.in/natefinch/lumberjack.v2"
+)
+
+var (
+	_logger *zap.Logger
+)
+
+const (
+	// FormatText format log text
+	FormatText = "text"
+	// FormatJSON format log json
+	FormatJSON = "json"
+)
+
+// type Level uint
+
+// 日志配置
+type logConfig struct {
+	LogPath    string
+	LogLevel   string
+	Compress   bool
+	MaxSize    int
+	MaxAge     int
+	MaxBackups int
+	Format     string
+}
+
+func getzapLevel(level string) zapcore.Level {
+	switch level {
+	case "debug":
+		return zap.DebugLevel
+	case "info":
+		return zap.InfoLevel
+	case "warn":
+		return zap.WarnLevel
+	case "error":
+		return zap.ErrorLevel
+	case "panic":
+		return zap.PanicLevel
+	case "fatal":
+		return zap.FatalLevel
+	default:
+		return zap.InfoLevel
+	}
+}
+
+func newLogWriter(logpath string, maxsize int, compress bool) io.Writer {
+	if logpath == "" || logpath == "-" {
+		return os.Stdout
+	}
+	return &lumberjack.Logger{
+		Filename: logpath,
+		MaxSize:  maxsize,
+		Compress: compress,
+	}
+}
+
+func newZapEncoder() zapcore.EncoderConfig {
+	encoderConfig := zapcore.EncoderConfig{
+		TimeKey:        "time",
+		LevelKey:       "level",
+		NameKey:        "logger",
+		CallerKey:      "line",
+		MessageKey:     "msg",
+		StacktraceKey:  "stacktrace",
+		LineEnding:     zapcore.DefaultLineEnding,
+		EncodeLevel:    zapcore.LowercaseLevelEncoder,  // 小写编码器
+		EncodeTime:     zapcore.ISO8601TimeEncoder,     // ISO8601 UTC 时间格式
+		EncodeDuration: zapcore.SecondsDurationEncoder, //
+		EncodeCaller:   zapcore.ShortCallerEncoder,     // 全路径编码器
+		EncodeName:     zapcore.FullNameEncoder,
+	}
+	return encoderConfig
+}
+func newLoggerCore(log *logConfig) zapcore.Core {
+	hook := newLogWriter(log.LogPath, log.MaxSize, log.Compress)
+
+	encoderConfig := newZapEncoder()
+
+	atomLevel := zap.NewAtomicLevelAt(getzapLevel(log.LogLevel))
+
+	var encoder zapcore.Encoder
+	if log.Format == FormatJSON {
+		encoder = zapcore.NewJSONEncoder(encoderConfig)
+	} else {
+		encoder = zapcore.NewConsoleEncoder(encoderConfig)
+	}
+
+	core := zapcore.NewCore(
+		encoder,
+		zapcore.NewMultiWriteSyncer(zapcore.AddSync(hook)),
+		atomLevel,
+	)
+	return core
+}
+
+func newLoggerOptions() []zap.Option {
+	// 开启开发模式，堆栈跟踪
+	caller := zap.AddCaller()
+	callerskip := zap.AddCallerSkip(1)
+	// 开发者
+	development := zap.Development()
+	options := []zap.Option{
+		caller,
+		callerskip,
+		development,
+	}
+	return options
+}
+
+// Option function option
+type Option func(*logConfig)
+
+// Path set logpath
+// if is zero will print,or write file
+func Path(logpath string) Option {
+	return func(logcfg *logConfig) {
+		logcfg.LogPath = logpath
+	}
+}
+
+// Compress compress log
+func Compress(compress bool) Option {
+	return func(logcfg *logConfig) {
+		logcfg.Compress = compress
+	}
+}
+
+// Level set log level default info
+func Level(level string) Option {
+	return func(logcfg *logConfig) {
+		logcfg.LogLevel = level
+	}
+}
+
+// MaxSize Log Max Size
+func MaxSize(size int) Option {
+	return func(logcfg *logConfig) {
+		logcfg.MaxSize = size
+	}
+}
+
+// MaxAge log store day
+func MaxAge(age int) Option {
+	return func(logcfg *logConfig) {
+		logcfg.MaxAge = age
+	}
+}
+
+// MaxBackups total store log
+func MaxBackups(backup int) Option {
+	return func(logcfg *logConfig) {
+		logcfg.MaxBackups = backup
+	}
+}
+
+// Format log json or text
+func Format(format string) Option {
+	return func(logcfg *logConfig) {
+		if format == FormatJSON {
+			logcfg.Format = FormatJSON
+		} else {
+			logcfg.Format = FormatText
+		}
+
+	}
+}
+
+func defaultOption() *logConfig {
+	return &logConfig{
+		LogPath:    "",
+		MaxSize:    20,
+		Compress:   true,
+		MaxAge:     7,
+		MaxBackups: 7,
+		LogLevel:   "debug",
+		Format:     FormatText,
+	}
+}
+
+// InitLog conf
+func InitLog(opts ...Option) error {
+	logcfg := defaultOption()
+	for _, opt := range opts {
+		opt(logcfg)
+	}
+	core := newLoggerCore(logcfg)
+
+	zapopts := newLoggerOptions()
+	_logger = zap.New(core, zapopts...)
+	return nil
+}
+
+// Debug output log
+func Debug(msg string, fields ...zap.Field) {
+	_logger.Debug(msg, fields...)
+}
+
+// Info output log
+func Info(msg string, fields ...zap.Field) {
+	_logger.Info(msg, fields...)
+}
+
+// Warn output log
+func Warn(msg string, fields ...zap.Field) {
+	_logger.Warn(msg, fields...)
+}
+
+// Error output log
+func Error(msg string, fields ...zap.Field) {
+	_logger.Error(msg, fields...)
+}
+
+// Panic output panic
+func Panic(msg string, fields ...zap.Field) {
+	_logger.Panic(msg, fields...)
+}
+
+// Fatal output log
+func Fatal(msg string, fields ...zap.Field) {
+	_logger.Fatal(msg, fields...)
+}

common_utils/log/log_test.go

@@ -0,0 +1,53 @@
+package log
+
+import (
+	"errors"
+	"fmt"
+	"go.uber.org/zap"
+	"runtime/debug"
+
+	"io"
+	"os"
+	"testing"
+)
+
+func TestNewLogger(t *testing.T) {
+	logcfg := &logConfig{
+		MaxSize:    10,
+		Compress:   true,
+		LogPath:    "",
+		MaxAge:     0,
+		MaxBackups: 0,
+		LogLevel:   "info",
+	}
+	err := InitLog(
+		Path(logcfg.LogPath),
+		Level(logcfg.LogLevel),
+		Compress(logcfg.Compress),
+		MaxSize(logcfg.MaxSize),
+		MaxBackups(logcfg.MaxBackups),
+		MaxAge(logcfg.MaxAge),
+		Format("json"),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v", err)
+		os.Exit(1)
+	}
+	defer func() {
+		if err := recover(); err != nil {
+			debug.PrintStack()
+		}
+	}()
+	Info("TestLog", zap.String("test", "eeyeyyeye"))
+	Debug("debug", zap.String("debug", "debug"))
+	Warn("warn", zap.String("warn", "warn"))
+	Error("error", zap.String("error", "error"))
+	Panic("panic", zap.String("panic", "panic"))
+	Fatal("fatal", zap.String("fatal", "fatal"))
+
+	err = io.EOF
+
+	err1 := fmt.Errorf("this is err: %w",err)
+	Info("111",zap.Error(err1))
+	fmt.Println(errors.Unwrap(err1),err1)
+}

common_utils/logger.go

@@ -0,0 +1,318 @@
+package util
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"strconv"
+	"sync"
+	"time"
+)
+
+const (
+	_VER string = "1.0.2"
+)
+
+type LEVEL int32
+
+var logLevel LEVEL = 1
+var maxFileSize int64
+var maxFileCount int32
+var dailyRolling bool = true
+var consoleAppender bool = true
+var RollingFile bool = false
+var logObj *_FILE
+
+const DATEFORMAT = "2006-01-02"
+
+type UNIT int64
+
+const (
+	_       = iota
+	KB UNIT = 1 << (iota * 10)
+	MB
+	GB
+	TB
+)
+
+const (
+	ALL LEVEL = iota
+	DEBUG
+	INFO
+	WARN
+	ERROR
+	FATAL
+	OFF
+)
+
+type _FILE struct {
+	dir      string
+	filename string
+	_suffix  int
+	isCover  bool
+	_date    *time.Time
+	mu       *sync.RWMutex
+	logfile  *os.File
+	lg       *log.Logger
+}
+
+func SetConsole(isConsole bool) {
+	consoleAppender = isConsole
+}
+
+func SetLevel(_level LEVEL) {
+	logLevel = _level
+}
+
+func SetRollingFile(fileDir, fileName string, maxNumber int32, maxSize int64, _unit UNIT) {
+	maxFileCount = maxNumber
+	maxFileSize = maxSize * int64(_unit)
+	RollingFile = true
+	dailyRolling = false
+	mkdirlog(fileDir)
+	logObj = &_FILE{dir: fileDir, filename: fileName, isCover: false, mu: new(sync.RWMutex)}
+	logObj.mu.Lock()
+	defer logObj.mu.Unlock()
+	for i := 1; i <= int(maxNumber); i++ {
+		if isExist(fileDir + "/" + fileName + "." + strconv.Itoa(i)) {
+			logObj._suffix = i
+		} else {
+			break
+		}
+	}
+	if !logObj.isMustRename() {
+		logObj.logfile, _ = os.OpenFile(fileDir+"/"+fileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
+		logObj.lg = log.New(logObj.logfile, "", log.Ldate|log.Ltime|log.Lshortfile)
+	} else {
+		logObj.rename()
+	}
+	go fileMonitor()
+}
+
+func SetRollingDaily(fileDir, fileName string) {
+	RollingFile = false
+	dailyRolling = true
+	t, _ := time.Parse(DATEFORMAT, time.Now().Format(DATEFORMAT))
+	mkdirlog(fileDir)
+	logObj = &_FILE{dir: fileDir, filename: fileName, _date: &t, isCover: false, mu: new(sync.RWMutex)}
+	logObj.mu.Lock()
+	defer logObj.mu.Unlock()
+
+	if !logObj.isMustRename() {
+		logObj.logfile, _ = os.OpenFile(fileDir+"/"+fileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
+		logObj.lg = log.New(logObj.logfile, "", log.Ldate|log.Ltime|log.Lshortfile)
+	} else {
+		logObj.rename()
+	}
+}
+
+func mkdirlog(dir string) (e error) {
+	_, er := os.Stat(dir)
+	b := er == nil || os.IsExist(er)
+	if !b {
+		if err := os.MkdirAll(dir, 0666); err != nil {
+			if os.IsPermission(err) {
+				fmt.Println("create dir error:", err.Error())
+				e = err
+			}
+		}
+	}
+	return
+}
+
+func console(s ...interface{}) {
+	if consoleAppender {
+		_, file, line, _ := runtime.Caller(2)
+		short := file
+		for i := len(file) - 1; i > 0; i-- {
+			if file[i] == '/' {
+				short = file[i+1:]
+				break
+			}
+		}
+		file = short
+		log.Println(file, strconv.Itoa(line), s)
+	}
+}
+
+func catchError() {
+	if err := recover(); err != nil {
+		log.Println("err", err)
+	}
+}
+
+func Debug(v ...interface{}) {
+	if dailyRolling {
+		fileCheck()
+	}
+	defer catchError()
+	if logObj != nil {
+		logObj.mu.RLock()
+		defer logObj.mu.RUnlock()
+	}
+
+	if logLevel <= DEBUG {
+		if logObj != nil {
+			logObj.lg.Output(2, fmt.Sprintln("debug", v))
+		}
+		console("debug", v)
+	}
+}
+func Info(v ...interface{}) {
+	if dailyRolling {
+		fileCheck()
+	}
+	defer catchError()
+	if logObj != nil {
+		logObj.mu.RLock()
+		defer logObj.mu.RUnlock()
+	}
+	if logLevel <= INFO {
+		if logObj != nil {
+			logObj.lg.Output(2, fmt.Sprintln("info", v))
+		}
+		console("info", v)
+	}
+}
+func Warn(v ...interface{}) {
+	if dailyRolling {
+		fileCheck()
+	}
+	defer catchError()
+	if logObj != nil {
+		logObj.mu.RLock()
+		defer logObj.mu.RUnlock()
+	}
+
+	if logLevel <= WARN {
+		if logObj != nil {
+			logObj.lg.Output(2, fmt.Sprintln("warn", v))
+		}
+		console("warn", v)
+	}
+}
+func Error(v ...interface{}) {
+	if dailyRolling {
+		fileCheck()
+	}
+	defer catchError()
+	if logObj != nil {
+		logObj.mu.RLock()
+		defer logObj.mu.RUnlock()
+	}
+	if logLevel <= ERROR {
+		if logObj != nil {
+			logObj.lg.Output(2, fmt.Sprintln("error", v))
+		}
+		console("error", v)
+	}
+}
+func Fatal(v ...interface{}) {
+	if dailyRolling {
+		fileCheck()
+	}
+	defer catchError()
+	if logObj != nil {
+		logObj.mu.RLock()
+		defer logObj.mu.RUnlock()
+	}
+	if logLevel <= FATAL {
+		if logObj != nil {
+			logObj.lg.Output(2, fmt.Sprintln("fatal", v))
+		}
+		console("fatal", v)
+	}
+}
+
+func (f *_FILE) isMustRename() bool {
+	if dailyRolling {
+		t, _ := time.Parse(DATEFORMAT, time.Now().Format(DATEFORMAT))
+		if t.After(*f._date) {
+			return true
+		}
+	} else {
+		if maxFileCount > 1 {
+			if fileSize(f.dir+"/"+f.filename) >= maxFileSize {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func (f *_FILE) rename() {
+	if dailyRolling {
+		fn := f.dir + "/" + f.filename + "." + f._date.Format(DATEFORMAT)
+		if !isExist(fn) && f.isMustRename() {
+			if f.logfile != nil {
+				f.logfile.Close()
+			}
+			err := os.Rename(f.dir+"/"+f.filename, fn)
+			if err != nil {
+				f.lg.Println("rename err", err.Error())
+			}
+			t, _ := time.Parse(DATEFORMAT, time.Now().Format(DATEFORMAT))
+			f._date = &t
+			f.logfile, _ = os.Create(f.dir + "/" + f.filename)
+			f.lg = log.New(logObj.logfile, "\n", log.Ldate|log.Ltime|log.Lshortfile)
+		}
+	} else {
+		f.coverNextOne()
+	}
+}
+
+func (f *_FILE) nextSuffix() int {
+	return int(f._suffix%int(maxFileCount) + 1)
+}
+
+func (f *_FILE) coverNextOne() {
+	f._suffix = f.nextSuffix()
+	if f.logfile != nil {
+		f.logfile.Close()
+	}
+	if isExist(f.dir + "/" + f.filename + "." + strconv.Itoa(int(f._suffix))) {
+		os.Remove(f.dir + "/" + f.filename + "." + strconv.Itoa(int(f._suffix)))
+	}
+	os.Rename(f.dir+"/"+f.filename, f.dir+"/"+f.filename+"."+strconv.Itoa(int(f._suffix)))
+	f.logfile, _ = os.Create(f.dir + "/" + f.filename)
+	f.lg = log.New(logObj.logfile, "\n", log.Ldate|log.Ltime|log.Lshortfile)
+}
+
+func fileSize(file string) int64 {
+	fmt.Println("fileSize", file)
+	f, e := os.Stat(file)
+	if e != nil {
+		fmt.Println(e.Error())
+		return 0
+	}
+	return f.Size()
+}
+
+func isExist(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil || os.IsExist(err)
+}
+
+func fileMonitor() {
+	timer := time.NewTicker(1 * time.Second)
+	for {
+		select {
+		case <-timer.C:
+			fileCheck()
+		}
+	}
+}
+
+func fileCheck() {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Println(err)
+		}
+	}()
+	if logObj != nil && logObj.isMustRename() {
+		logObj.mu.Lock()
+		defer logObj.mu.Unlock()
+		logObj.rename()
+	}
+}

common_utils/mfw/catch.go

@@ -0,0 +1,34 @@
+package mfw
+
+import (
+	"log"
+	"runtime"
+)
+
+//出错拦截
+func Catch() {
+	if r := recover(); r != nil {
+		log.Println("err:", r)
+		for skip := 0; ; skip++ {
+			_, file, line, ok := runtime.Caller(skip)
+			if !ok {
+				break
+			}
+			go log.Printf("%v,%v\n", file, line)
+		}
+	}
+}
+
+//出错拦截
+func CatchLogger() {
+	if r := recover(); r != nil {
+		log.Println(r)
+		for skip := 0; ; skip++ {
+			_, file, line, ok := runtime.Caller(skip)
+			if !ok {
+				break
+			}
+			go log.Printf(file, ", ", line)
+		}
+	}
+}

common_utils/mfw/client.go

@@ -0,0 +1,305 @@
+//客户端封装,支持短线重连接，支持客户端检测
+package mfw
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"sync"
+	"time"
+)
+
+var IsHiddenLog bool
+
+//封装，可以链接多个消息总线
+type Client struct {
+	conn                 *TCPClient
+	myid                 string //
+	oldmyid              string
+	businessEventHandler func(*Packet)
+	canHandleEvent       []int
+	lastCheckHeart       int64 //最后一次心跳检测
+	messageQueue         chan *Packet
+	writeQueue           chan RawData
+	resultlock           sync.Map //map[string]chan []byte
+	myname               string
+	OnConnectSuccess     func()
+}
+
+//客户端配置
+type ClientConfig struct {
+	ClientName       string          //名称
+	EventHandler     func(p *Packet) //事件处理函数
+	MsgServerAddr    string          //消息总线地址
+	CanHandleEvents  []int           //我可以处理的事件列表
+	OnRequestConnect func()          //请求重连的事件处理函数
+	OnConnectSuccess func()          //重连成功的时间处理函数
+	ReadBufferSize   int             //读缓冲区大小
+	WriteBufferSize  int             //写缓冲区大小
+}
+
+//
+func printlog(p ...interface{}) {
+	if !IsHiddenLog {
+		fmt.Print(p...)
+	}
+}
+
+//
+func (client *Client) WriteObj(to, msgid string, event, sendtotype int, obj interface{}) {
+	client.writeQueue <- RawData(Enpacket(client.myid, to, msgid, event, sendtotype, obj))
+}
+
+//同步调用方法
+func (client *Client) Call(to, msgid string, event, sendtotype int, obj interface{}, calltimeout int64) ([]byte, error) {
+	client.resultlock.Store(msgid, make(chan []byte))
+	client.writeQueue <- RawData(Enpacket(client.myid, to, msgid, event, sendtotype, obj))
+	tmp, _ := client.resultlock.Load(msgid)
+	lock := tmp.(chan []byte)
+	select {
+	case <-time.After(time.Duration(calltimeout) * time.Second):
+		client.resultlock.Delete(msgid)
+		return nil, errors.New("timeout")
+	case ret := <-lock:
+		client.resultlock.Delete(msgid)
+		return ret, nil
+	}
+}
+
+func (client *Client) GetMyclient() string {
+	return client.myid
+}
+
+//底层事件处理
+func (client *Client) baseEventHandle(msg *Packet) {
+	printlog(".")
+	event := int(msg.Event)
+	switch event {
+	case EVENT_RETURN_MACHINE_ID: //服务端分配id，发布我能处理的时间
+		printlog("*", string(msg.GetBusinessData()))
+		client.myid = string(msg.GetBusinessData())
+		client.conn.MyId = client.myid
+
+		if client.oldmyid != "" {
+			client.WriteObj("", "", EVENT_REMOVE_CLIENT, SENDTO_TYPE_P2P, []byte(client.oldmyid))
+			client.oldmyid = ""
+		}
+
+		bs := make([]byte, 0)
+		for _, v := range client.canHandleEvent {
+			bs = append(bs, Int2Byte(int32(v))...)
+		}
+		client.WriteObj("", "", EVENT_PUBLISH_MYSERVICES, SENDTO_TYPE_P2P, bs)
+		client.WriteObj("", "", EVENT_UPDATE_MYNAME, SENDTO_TYPE_P2P, []byte(client.myname))
+		if client.OnConnectSuccess != nil { //调用成功回调
+			go client.OnConnectSuccess()
+		}
+	case EVENT_REQUEST_HEARTBEAT: //请求的心跳，回应心跳请求
+		client.WriteObj("", "", EVENT_RETURN_HEARTBEAT, SENDTO_TYPE_P2P, msg.GetBusinessData())
+		client.lastCheckHeart = time.Now().Unix()
+	case EVENT_SYSTEM_COMMAND: //系统控制指令
+		go processSysCommand(client, msg.GetBusinessData())
+	case EVENT_RECIVE_CALLBACK: //回调处理
+		if v, ok := client.resultlock.Load(msg.Msgid); ok {
+			tmp := v.(chan []byte)
+			tmp <- msg.GetBusinessData()
+			return
+		}
+		fallthrough
+	default: //业务处理
+		go client.businessEventHandler(msg)
+	}
+
+}
+
+//从套接字读取数据
+func (client *Client) read4socket() (*Packet, error) {
+	defer Catch()
+	buffer := make([]byte, 4)
+	if _, err := io.ReadFull(client.conn, buffer); err != nil {
+		printlog("?_readFullerr", err)
+		return nil, errors.New("read packet error")
+	}
+	size := int32(binary.BigEndian.Uint32(buffer))
+	if size < 32 || size > 16384000 {
+		printlog("?_sizeerror[", size, "]")
+		return nil, errors.New("read packet error")
+	}
+	var buf []byte = make([]byte, size)
+	readlen, err := io.ReadFull(client.conn, buf)
+	if err != nil || int32(readlen) != size {
+		printlog("?_readpacketerror", err)
+		return nil, errors.New("read packet error")
+	}
+	//
+	raw := append(Int2Byte(size), buf...)
+	//解析包
+	packet := &Packet{Length: size,
+		From:       string(buf[:8]),
+		To:         string(buf[8:16]),
+		Msgid:      string(buf[16:24]),
+		Event:      Byte2Int(buf[24:28]),
+		SentToType: Byte2Int(buf[28:32]),
+		Raw:        raw,
+	}
+	return packet, nil
+}
+
+//读数据到缓冲区
+func (client *Client) readDump(queue chan<- *Packet) {
+	defer Catch()
+	for {
+		if p, err := client.read4socket(); err == nil && p != nil {
+			client.lastCheckHeart = time.Now().Unix()
+			queue <- p
+		} else {
+			time.Sleep(4 * time.Second)
+		}
+	}
+}
+
+//写数据到套接字
+func (client *Client) write2socket(msg RawData) (ret bool) {
+	defer Catch()
+	_, err := client.conn.Write([]byte(msg))
+	if err == nil {
+		client.lastCheckHeart = time.Now().Unix()
+		return true
+	}
+	return false
+}
+
+//写数据到缓冲区
+func (client *Client) writeDump(p <-chan RawData) {
+	defer Catch()
+	for {
+		select {
+		case msg := <-p:
+			//写数据，直到写入成功
+			client.write2socket(msg)
+		}
+	}
+}
+
+//启动客户端
+//@Deprecated 此方法已过期
+func StartClient(fn func(*Packet),
+	addr string, myname string,
+	handleevent []int) (*Client, error) {
+	//
+	client := &Client{}
+	client.resultlock = sync.Map{} //make(map[string]chan []byte)
+	client.businessEventHandler = fn
+	client.canHandleEvent = handleevent
+	client.myname = myname
+	client.messageQueue = make(chan *Packet, 500) //接受消息
+	client.writeQueue = make(chan RawData, 500)   //写入并发
+
+	go func(q <-chan *Packet, parseEvent func(*Packet)) {
+		for {
+			select {
+			case msg := <-q:
+				go parseEvent(msg)
+			}
+		}
+	}(client.messageQueue, client.baseEventHandle)
+	go client.writeDump(client.writeQueue)
+	//创建链接
+	var err error
+	for client.conn, err = Dial("tcp4", addr); err != nil; client.conn, err = Dial("tcp4", addr) {
+		printlog("?")
+		time.Sleep(5 * time.Second)
+	}
+	client.conn.SetMaxRetries(math.MaxInt32)      //重试次数
+	client.conn.SetRetryInterval(2 * time.Second) //重试间隔30秒
+	client.conn.OnRequestConnect = func() {
+		client.oldmyid = client.myid
+		client.myid = ""
+	}
+	go client.readDump(client.messageQueue)
+	return client, nil
+}
+
+//解析处理控制指令
+func processSysCommand(client *Client, data []byte) {
+	tmp := map[string]interface{}{}
+	err := json.Unmarshal(data, &tmp)
+	if err != nil {
+		return
+	}
+	cmd := tmp["act"].(string)
+	switch cmd {
+	case "pass": //暂停服务
+		t := int(tmp["passtime"].(float64))
+		client.WriteObj("", "", EVENT_PUBLISH_MYSERVICES, SENDTO_TYPE_P2P, []byte{})
+		time.Sleep(time.Duration(t) * time.Second)
+		bs := make([]byte, 0)
+		for _, v := range client.canHandleEvent {
+			bs = append(bs, Int2Byte(int32(v))...)
+		}
+		client.WriteObj("", "", EVENT_PUBLISH_MYSERVICES, SENDTO_TYPE_P2P, bs)
+	case "reconnect": //重新连接
+
+	case "quit": //退出
+		client.WriteObj("", "", EVENT_BYE, SENDTO_TYPE_P2P, []byte{})
+		os.Exit(0)
+	}
+}
+
+//启动客户端
+func NewClient(conf *ClientConfig) (*Client, error) {
+	if conf.MsgServerAddr == "" {
+		return nil, errors.New("配置项缺失(MsgServerAddr)")
+	} else if conf.EventHandler == nil {
+		return nil, errors.New("配置项缺失(EventHandler)")
+	}
+	if conf.ReadBufferSize == 0 {
+		conf.ReadBufferSize = 200
+	}
+	if conf.WriteBufferSize == 0 {
+		conf.WriteBufferSize = 150
+	}
+	client := &Client{}
+	client.resultlock = sync.Map{} // make(map[string]chan []byte)
+	client.businessEventHandler = conf.EventHandler
+	client.canHandleEvent = conf.CanHandleEvents
+	client.myname = conf.ClientName
+	client.messageQueue = make(chan *Packet, conf.ReadBufferSize) //接受消息
+	client.writeQueue = make(chan RawData, conf.WriteBufferSize)  //写入并发
+	go func(q <-chan *Packet, parseEvent func(*Packet)) {
+		for {
+			select {
+			case msg := <-q:
+				go parseEvent(msg)
+			}
+		}
+	}(client.messageQueue, client.baseEventHandle)
+	go client.writeDump(client.writeQueue)
+	//创建链接
+	var err error
+	for client.conn, err = Dial("tcp4", conf.MsgServerAddr); err != nil; client.conn, err = Dial("tcp4", conf.MsgServerAddr) {
+		printlog("?")
+		time.Sleep(5 * time.Second)
+	}
+	client.conn.SetMaxRetries(math.MaxInt32)      //重试次数
+	client.conn.SetRetryInterval(2 * time.Second) //重试间隔30秒
+	//事件绑定
+	client.conn.OnRequestConnect = func() {
+		client.oldmyid = client.myid
+		client.myid = ""
+		if conf.OnRequestConnect != nil {
+			conf.OnRequestConnect()
+		}
+	}
+	client.conn.OnConnectSuccess = func() {
+		if conf.OnConnectSuccess != nil {
+			conf.OnConnectSuccess()
+		}
+	}
+	go client.readDump(client.messageQueue)
+	return client, nil
+}

common_utils/mfw/event.go

@@ -0,0 +1,42 @@
+package mfw
+
+//只有事件，发送类型
+const (
+	EVENT_RETURN_MACHINE_ID    = iota
+	EVENT_REQUEST_HEARTBEAT    //心跳
+	EVENT_RETURN_HEARTBEAT     //
+	EVENT_PUBLISH_MYSERVICES   //发布我的服务
+	EVENT_REQUEST_SPIDER_STATE //获取爬虫状态
+	EVENT_RECIVE_SPIDER_STATE  //接受爬虫状态
+	EVENT_RECIVE_CALLBACK      //调用返回，用于调用服务需要同步返回值
+	EVENT_VIEWALL_SERVICE      //查看所有的客户端
+	EVENT_REMOVE_CLIENT        //删除客户端
+	EVENT_UPDATE_MYNAME        //更新客户端名称
+	EVENT_SYSTEM_COMMAND       //系统控制指令，每个客户端默认都会实现
+	EVENT_BYE                  //客户端主动断开
+	EVENT_GET_ALLDOWNLOADER    //取所有的下载器
+	//---------
+	SERVICE_DOWNLOAD             = 7070 //下载服务
+	SERVICE_GETPROXY             = 7071 //获取代理
+	SERVICE_GETNAME              = 7072 //获取企业名录
+	SERVICE_SPIDER_ECPS          = 7073 //公示爬虫分发服务
+	SERVICE_DISTINGUISH          = 7074 //人工识别验证码
+	SERVICE_DOWNLOAD_DELETE_NODE = 7075 //删除节点
+	SERVICE_DOWNLOAD_APPEND_NODE = 7076 //追加节点
+	SERVICE_REPORT_SAVE          = 7077 //保存报表数据
+	SERVICE_INVNAME_ANALYSIS     = 7078 //分析股东名录
+	SERVICE_YCML_SAVE            = 7079 //异常名录保存入内存数据库
+	SERVICE_YCML_NOTICE          = 7080 //异常名录下载完成通知
+	SERVICE_ECPS_INC             = 7081 //公示增量数据保存
+	SERVICE_OFFICE_ANALYSIS      = 7082 //word pdf excel文件解析服务 由java提供的一个服务
+
+	SERVICE_LOG_CHECK = 7200 //检测日志记录
+
+	//-------发送方式----------------
+	SENDTO_TYPE_RAND_RECIVER = 0 //发送给任一服务接收者，默认是这种模式
+	SENDTO_TYPE_ALL          = 1 //发送给所有客户端
+	SENDTO_TYPE_ALL_RECIVER  = 2 //发送给所有指定服务接收者
+	SENDTO_TYPE_P2P          = 3 //发送给指定客户端
+
+	SENDTO_TYPE_P2P_BYNAME = 4 //发送给指定客户，通过指定客户的名称
+)

common_utils/mfw/key.go

@@ -0,0 +1,30 @@
+package mfw
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	//	"encoding/hex"
+	//"fmt"
+	mr "math/rand"
+)
+
+const (
+	KEY = "()*+-_=~`!@#$%^&*{}[]|',.<>?abcdefghhijklmnopqrstuvwxyz0123456789ABCDEFJHIJKLMNOPQRSTUVWXYZ"
+)
+
+//
+func UUID(length int) string {
+	tmp := make([]byte, length>>1)
+	rand.Read(tmp)
+	return hex.EncodeToString(tmp)
+}
+
+//
+func OldUUID(length int) string {
+	ret := ""
+	for i := 0; i < length; i++ {
+		pos := mr.Intn(90)
+		ret += KEY[pos : pos+1]
+	}
+	return ret
+}

common_utils/mfw/protocol.go

@@ -0,0 +1,161 @@
+package mfw
+
+//底层网络通信协议
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+)
+
+const (
+	EMPTY_ADDR = "00000000"
+)
+
+//读取
+type Reader struct {
+	conn   net.Conn
+	reader *bufio.Reader
+	buffer [4]byte
+}
+
+//数据包，实际上就是协议定义
+type Packet struct {
+	//头开始
+	Length     int32  //4bit 包长度
+	From       string //8bit
+	To         string //8bit
+	Msgid      string //8bit
+	Event      int32  //4bit
+	SentToType int32  //4bit
+	//头结束
+	Raw []byte //数据包，头+数据
+}
+
+//返回
+func (p *Packet) GetBusinessData() []byte {
+	return p.Raw[36:]
+}
+
+//
+type RawData []byte //流数据
+
+//
+func NewReader(c net.Conn) *Reader {
+	return &Reader{
+		conn:   c,
+		reader: bufio.NewReader(c),
+	}
+}
+
+//读取头部
+func (p *Reader) readHeader() (int32, error) {
+	buf := p.buffer[:4]
+	if _, err := io.ReadFull(p.reader, buf); err != nil {
+		return 0, err
+	}
+	size := int32(binary.BigEndian.Uint32(buf))
+	if size < 32 || size > 16384000 {
+		return 0, fmt.Errorf("Incorrect frame size (%d)", size)
+	}
+	return size, nil
+}
+
+//读取完整一帧数据，防止粘包
+func (p *Reader) readFrame(size int32) (*Packet, error) {
+	var buf []byte
+	if int(size) <= len(p.buffer) {
+		buf = p.buffer[0:size]
+	} else {
+		buf = make([]byte, size)
+	}
+	_, err := io.ReadFull(p.reader, buf)
+	raw := append(Int2Byte(size), buf...)
+	if err == nil {
+		//解析包
+		packet := &Packet{
+			Length:     size,
+			From:       string(buf[:8]),
+			To:         string(buf[8:16]),
+			Msgid:      string(buf[16:24]),
+			Event:      Byte2Int(buf[24:28]),
+			SentToType: Byte2Int(buf[28:32]),
+			//Data:       buf[32:],
+			Raw: raw,
+		}
+		return packet, nil
+	}
+	return nil, err
+}
+
+//读取数据写入队列
+func forwardMessage(c net.Conn, queue chan<- *Packet) {
+	defer c.Close()
+	logReader := NewReader(c)
+	for {
+		size, err := logReader.readHeader()
+		if err != nil {
+			break
+		}
+		packet, err := logReader.readFrame(size)
+		if err != nil {
+			break
+		}
+		queue <- packet
+	}
+}
+
+//从队列中读取数据，定期处理
+func processMsg(q <-chan *Packet, parseEvent func(*Packet)) {
+	for {
+		select {
+		case msg := <-q:
+			go parseEvent(msg)
+		}
+	}
+}
+
+//封包，输出直接可以写到流中的数据包
+func Enpacket(from, to, msgid string, event, sendtotype int, obj interface{}) []byte {
+	if len(from) != 8 {
+		from = EMPTY_ADDR
+	}
+	if len(to) != 8 {
+		to = EMPTY_ADDR
+	}
+	if len(msgid) != 8 {
+		msgid = EMPTY_ADDR
+	}
+	var ret []byte
+	var bs []byte
+	if v, ok := obj.([]byte); ok {
+		bs = v
+	} else if v, ok := obj.(string); ok {
+		bs = []byte(v)
+	} else {
+		bs, _ = json.Marshal(obj)
+	}
+	ret = append(Int2Byte(int32(32 + len(bs))))
+	ret = append(ret, []byte(from+to+msgid)...)
+	ret = append(ret, Int2Byte(int32(event))...)
+	ret = append(ret, Int2Byte(int32(sendtotype))...)
+	ret = append(ret, bs...)
+	return ret
+}
+
+//
+func Byte2Int(src []byte) int32 {
+	var ret int32
+	binary.Read(bytes.NewReader(src), binary.BigEndian, &ret)
+	return ret
+}
+
+//
+func Int2Byte(src int32) []byte {
+	buf := bytes.NewBuffer([]byte{})
+	binary.Write(buf, binary.BigEndian, src)
+	return buf.Bytes()
+}

common_utils/mfw/tcpclient.go

@@ -0,0 +1,240 @@
+package mfw
+
+import (
+	"io"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+//
+const (
+	statusOnline             = iota
+	statusOffline            = iota
+	statusReconnecting       = iota
+	ErrMaxRetries      Error = 0x01
+)
+
+//
+type Error int
+
+//
+type TCPClient struct {
+	*net.TCPConn
+	MyId             string
+	lock             sync.RWMutex
+	status           int32
+	maxRetries       int
+	retryInterval    time.Duration
+	OnConnectSuccess func() //重连成功后的回调
+	OnRequestConnect func() //请求重连
+}
+
+//
+func (e Error) Error() string {
+	switch e {
+	case ErrMaxRetries:
+		return "超过最大尝试次数"
+	default:
+		return "未知错误"
+	}
+}
+
+//
+func Dial(network, addr string) (*TCPClient, error) {
+	raddr, err := net.ResolveTCPAddr(network, addr)
+	if err != nil {
+		return nil, err
+	}
+	return DialTCP(network, nil, raddr)
+}
+
+//
+func DialTCP(network string, laddr, raddr *net.TCPAddr) (*TCPClient, error) {
+	conn, err := net.DialTCP(network, laddr, raddr)
+	if err != nil {
+		return nil, err
+	}
+	return &TCPClient{
+		TCPConn:       conn,
+		lock:          sync.RWMutex{},
+		status:        0,
+		maxRetries:    10,
+		retryInterval: 10 * time.Millisecond,
+	}, nil
+}
+
+//建议用这个
+func DialTCPTimeout(network, raddr string, timeout time.Duration) (*TCPClient, error) {
+	conn, err := net.DialTimeout(network, raddr, timeout)
+	if err != nil {
+		return nil, err
+	}
+	return &TCPClient{
+		TCPConn:       conn.(*net.TCPConn),
+		lock:          sync.RWMutex{},
+		status:        0,
+		maxRetries:    10,
+		retryInterval: 10 * time.Millisecond,
+	}, nil
+}
+
+//设置最大尝试次数
+func (c *TCPClient) SetMaxRetries(maxRetries int) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.maxRetries = maxRetries
+}
+
+//
+func (c *TCPClient) GetMaxRetries() int {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	return c.maxRetries
+}
+
+//重连间隔时间
+func (c *TCPClient) SetRetryInterval(retryInterval time.Duration) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.retryInterval = retryInterval
+}
+
+//
+func (c *TCPClient) GetRetryInterval() time.Duration {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	return c.retryInterval
+}
+
+//
+func (c *TCPClient) reconnect() error {
+	if !atomic.CompareAndSwapInt32(&c.status, statusOffline, statusReconnecting) {
+		return nil
+	}
+	//
+	if c.OnRequestConnect != nil {
+		c.OnRequestConnect()
+	}
+	raddr := c.TCPConn.RemoteAddr()
+	conn, err := net.DialTCP(raddr.Network(), nil, raddr.(*net.TCPAddr))
+	if err != nil {
+		defer atomic.StoreInt32(&c.status, statusOffline)
+		switch err.(type) {
+		case *net.OpError:
+			return err
+		default:
+			return err
+		}
+	}
+	c.TCPConn.Close()
+	c.TCPConn = conn
+	atomic.StoreInt32(&c.status, statusOnline)
+	if c.OnConnectSuccess != nil {
+		c.OnConnectSuccess() //执行后续操作
+	}
+	return nil
+}
+
+//读操作
+func (c *TCPClient) Read(b []byte) (int, error) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	for i := 0; i < c.maxRetries; i++ {
+		if atomic.LoadInt32(&c.status) == statusOnline {
+			n, err := c.TCPConn.Read(b)
+			if err == nil {
+				return n, err
+			}
+			switch err.(type) {
+			case *net.OpError:
+				atomic.StoreInt32(&c.status, statusOffline)
+			default:
+				if err.Error() == "EOF" {
+					atomic.StoreInt32(&c.status, statusOffline)
+				} else {
+					return n, err
+				}
+			}
+		} else if atomic.LoadInt32(&c.status) == statusOffline {
+			if err := c.reconnect(); err != nil {
+				return -1, err
+			}
+		}
+		if i < (c.maxRetries - 1) {
+			time.Sleep(c.retryInterval)
+		}
+	}
+	return -1, ErrMaxRetries
+}
+
+//
+func (c *TCPClient) ReadFrom(r io.Reader) (int64, error) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	for i := 0; i < c.maxRetries; i++ {
+		if atomic.LoadInt32(&c.status) == statusOnline {
+			n, err := c.TCPConn.ReadFrom(r)
+			if err == nil {
+				return n, err
+			}
+			switch err.(type) {
+			case *net.OpError:
+				atomic.StoreInt32(&c.status, statusOffline)
+			default:
+				if err.Error() == "EOF" {
+					atomic.StoreInt32(&c.status, statusOffline)
+				} else {
+					return n, err
+				}
+			}
+		} else if atomic.LoadInt32(&c.status) == statusOffline {
+			if err := c.reconnect(); err != nil {
+				return -1, err
+			}
+		}
+		if i < (c.maxRetries - 1) {
+			time.Sleep(c.retryInterval)
+		}
+	}
+	return -1, ErrMaxRetries
+}
+
+//
+func (c *TCPClient) Write(b []byte) (int, error) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	for i := 0; i < c.maxRetries; i++ {
+		if atomic.LoadInt32(&c.status) == statusOnline {
+			for c.MyId == "" {
+				time.Sleep(1 * time.Second)
+			}
+			if (c.MyId != "" && string(b[4:12]) != c.MyId) || i == 1 { //重连过了，老数据要切换来源（仅修一次）
+				tmp := append(b[:4])
+				tmp = append(tmp, []byte(c.MyId)...)
+				tmp = append(tmp, b[12:]...)
+				b = append(tmp)
+			}
+			n, err := c.TCPConn.Write(b)
+			if err == nil {
+				return n, err
+			}
+			switch err.(type) {
+			case *net.OpError:
+				atomic.StoreInt32(&c.status, statusOffline)
+
+			default:
+				return n, err
+			}
+		} else if atomic.LoadInt32(&c.status) == statusOffline {
+			if err := c.reconnect(); err != nil {
+				return -1, err
+			}
+		}
+		if i < (c.maxRetries - 1) {
+			time.Sleep(c.retryInterval)
+		}
+	}
+	return -1, ErrMaxRetries
+}

common_utils/mongodb/mongodb.go

@@ -0,0 +1,614 @@
+package mongodb
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"log"
+	"math/big"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type MgoSess struct {
+	db     string
+	coll   string
+	query  interface{}
+	sorts  []string
+	fields interface{}
+	limit  int64
+	skip   int64
+	pipe   []map[string]interface{}
+	all    interface{}
+	M      *MongodbSim
+}
+
+type MgoIter struct {
+	Cursor *mongo.Cursor
+}
+
+func NewMgo(addr, db string, size int) *MongodbSim {
+	mgo := &MongodbSim{
+		MongodbAddr: addr,
+		Size:        size,
+		DbName:      db,
+	}
+	mgo.InitPool()
+	return mgo
+}
+
+func (mt *MgoIter) Next(result interface{}) bool {
+	if mt.Cursor != nil {
+		if mt.Cursor.Next(nil) {
+			err := mt.Cursor.Decode(result)
+			if err != nil {
+				log.Println("mgo cur err", err.Error())
+				mt.Cursor.Close(nil)
+				return false
+			}
+			return true
+		} else {
+			mt.Cursor.Close(nil)
+			return false
+		}
+	} else {
+		return false
+	}
+}
+
+func (ms *MgoSess) DB(name string) *MgoSess {
+	ms.db = name
+	return ms
+}
+
+func (ms *MgoSess) C(name string) *MgoSess {
+	ms.coll = name
+	return ms
+}
+
+func (ms *MgoSess) Find(q interface{}) *MgoSess {
+	if q == nil {
+		q = map[string]interface{}{}
+	}
+	ms.query = q
+	return ms
+}
+
+func (ms *MgoSess) Select(fields interface{}) *MgoSess {
+	ms.fields = fields
+	return ms
+}
+
+func (ms *MgoSess) Limit(limit int64) *MgoSess {
+	ms.limit = limit
+	return ms
+}
+func (ms *MgoSess) Skip(skip int64) *MgoSess {
+	ms.skip = skip
+	return ms
+}
+
+func (ms *MgoSess) Sort(sorts ...string) *MgoSess {
+	ms.sorts = sorts
+	return ms
+}
+func (ms *MgoSess) Pipe(p []map[string]interface{}) *MgoSess {
+	ms.pipe = p
+	return ms
+}
+func (ms *MgoSess) All(v *[]map[string]interface{}) {
+	cur, err := ms.M.C.Database(ms.db).Collection(ms.coll).Aggregate(ms.M.Ctx, ms.pipe)
+	if err == nil && cur.Err() == nil {
+		cur.All(ms.M.Ctx, v)
+	}
+}
+func (ms *MgoSess) Iter() *MgoIter {
+	it := &MgoIter{}
+	find := options.Find()
+	if ms.skip > 0 {
+		find.SetSkip(ms.skip)
+	}
+	if ms.limit > 0 {
+		find.SetLimit(ms.limit)
+	}
+	find.SetBatchSize(100)
+	if len(ms.sorts) > 0 {
+		sort := bson.M{}
+		for _, k := range ms.sorts {
+			switch k[:1] {
+			case "-":
+				sort[k[1:]] = -1
+			case "+":
+				sort[k[1:]] = 1
+			default:
+				sort[k] = 1
+			}
+		}
+		find.SetSort(sort)
+	}
+	if ms.fields != nil {
+		find.SetProjection(ms.fields)
+	}
+	cur, err := ms.M.C.Database(ms.db).Collection(ms.coll).Find(ms.M.Ctx, ms.query, find)
+	if err != nil {
+		log.Println("mgo find err", err.Error())
+	} else {
+		it.Cursor = cur
+	}
+	return it
+}
+
+func (ms *MgoSess) Count() (int64, error) {
+	return ms.M.C.Database(ms.db).Collection(ms.coll).CountDocuments(ms.M.Ctx, ms.query)
+}
+
+type MongodbSim struct {
+	MongodbAddr string
+	Size        int
+	//	MinSize     int
+	DbName   string
+	C        *mongo.Client
+	Ctx      context.Context
+	ShortCtx context.Context
+	pool     chan bool
+	UserName string
+	Password string
+	ReplSet  string
+}
+
+func (m *MongodbSim) GetMgoConn() *MgoSess {
+	//m.Open()
+	ms := &MgoSess{}
+	ms.M = m
+	return ms
+}
+
+func (m *MongodbSim) DestoryMongoConn(ms *MgoSess) {
+	//m.Close()
+	ms.M = nil
+	ms = nil
+}
+
+func (m *MongodbSim) Destroy() {
+	//m.Close()
+	m.C.Disconnect(nil)
+	m.C = nil
+}
+
+func (m *MongodbSim) InitPool() {
+	opts := options.Client()
+	opts.SetConnectTimeout(3 * time.Second)
+	opts.SetHosts(strings.Split(m.MongodbAddr, ","))
+	//opts.ApplyURI("mongodb://" + m.MongodbAddr)
+	opts.SetMaxPoolSize(uint64(m.Size))
+	if m.UserName != "" && m.Password != "" {
+		cre := options.Credential{
+			Username: m.UserName,
+			Password: m.Password,
+		}
+		opts.SetAuth(cre)
+	}
+	//ms := strings.Split(m.MongodbAddr, ",")
+	//if m.ReplSet == "" && len(ms) > 1 {
+	//	m.ReplSet = "qfws"
+	//}
+	if m.ReplSet != "" {
+		opts.SetReplicaSet(m.ReplSet)
+		opts.SetDirect(false)
+	}
+	m.pool = make(chan bool, m.Size)
+	opts.SetMaxConnIdleTime(2 * time.Hour)
+	m.Ctx, _ = context.WithTimeout(context.Background(), 99999*time.Hour)
+	m.ShortCtx, _ = context.WithTimeout(context.Background(), 1*time.Minute)
+	client, err := mongo.Connect(m.ShortCtx, opts)
+	if err != nil {
+		log.Println("mgo init error:", err.Error())
+	} else {
+		m.C = client
+	}
+}
+
+func (m *MongodbSim) Open() {
+	m.pool <- true
+}
+func (m *MongodbSim) Close() {
+	<-m.pool
+}
+
+func (m *MongodbSim) Save(c string, doc interface{}) string {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	obj := ObjToM(doc)
+	id := primitive.NewObjectID()
+	(*obj)["_id"] = id
+	_, err := coll.InsertOne(m.Ctx, obj)
+	if nil != err {
+		log.Println("SaveError", err)
+		return ""
+	}
+	return id.Hex()
+}
+
+//原_id不变
+func (m *MongodbSim) SaveByOriID(c string, doc interface{}) bool {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	_, err := coll.InsertOne(m.Ctx, ObjToM(doc))
+	if nil != err {
+		log.Println("SaveByOriIDError", err)
+		return false
+	}
+	return true
+}
+
+//批量插入
+func (m *MongodbSim) SaveBulk(c string, doc ...map[string]interface{}) bool {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	for _, d := range doc {
+		write := mongo.NewInsertOneModel()
+		write.SetDocument(d)
+		writes = append(writes, write)
+	}
+	br, e := coll.BulkWrite(m.Ctx, writes)
+	if e != nil {
+		b := strings.Index(e.Error(), "duplicate") > -1
+		log.Println("mgo savebulk error:", e.Error())
+		if br != nil {
+			log.Println("mgo savebulk size:", br.InsertedCount)
+		}
+		return b
+	}
+	return true
+}
+
+//批量插入
+func (m *MongodbSim) SaveBulkInterface(c string, doc ...interface{}) bool {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	for _, d := range doc {
+		write := mongo.NewInsertOneModel()
+		write.SetDocument(d)
+		writes = append(writes, write)
+	}
+	br, e := coll.BulkWrite(m.Ctx, writes)
+	if e != nil {
+		b := strings.Index(e.Error(), "duplicate") > -1
+		log.Println("mgo SaveBulkInterface error:", e.Error())
+		if br != nil {
+			log.Println("mgo SaveBulkInterface size:", br.InsertedCount)
+		}
+		return b
+	}
+	return true
+}
+
+//按条件统计
+func (m *MongodbSim) Count(c string, q interface{}) int {
+	r, _ := m.CountByErr(c, q)
+	return r
+}
+
+//统计
+func (m *MongodbSim) CountByErr(c string, q interface{}) (int, error) {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	res, err := m.C.Database(m.DbName).Collection(c).CountDocuments(m.Ctx, ObjToM(q))
+	if err != nil {
+		log.Println("统计错误", err.Error())
+		return 0, err
+	} else {
+		return int(res), nil
+	}
+}
+
+//按条件删除
+func (m *MongodbSim) Delete(c string, q interface{}) int64 {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	res, err := m.C.Database(m.DbName).Collection(c).DeleteMany(m.Ctx, ObjToM(q))
+	if err != nil && res == nil {
+		log.Println("删除错误", err.Error())
+	}
+	return res.DeletedCount
+}
+
+//删除对象
+func (m *MongodbSim) Del(c string, q interface{}) bool {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	_, err := m.C.Database(m.DbName).Collection(c).DeleteMany(m.Ctx, ObjToM(q))
+	if err != nil {
+		log.Println("删除错误", err.Error())
+		return false
+	}
+	return true
+}
+
+//删除表
+func (m *MongodbSim) DelColl(c string) bool {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	err := m.C.Database(m.DbName).Collection(c).Drop(m.Ctx)
+	if err != nil {
+		log.Println("删除错误", err.Error())
+		return false
+	}
+	return true
+}
+
+//按条件更新
+func (m *MongodbSim) Update(c string, q, u interface{}, upsert bool, multi bool) bool {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	ct := options.Update()
+	if upsert {
+		ct.SetUpsert(true)
+	}
+	coll := m.C.Database(m.DbName).Collection(c)
+	var err error
+	if multi {
+		_, err = coll.UpdateMany(m.Ctx, ObjToM(q), ObjToM(u), ct)
+	} else {
+		_, err = coll.UpdateOne(m.Ctx, ObjToM(q), ObjToM(u), ct)
+	}
+	if err != nil {
+		log.Println("删除错误", err.Error())
+		return false
+	}
+	return true
+}
+func (m *MongodbSim) UpdateById(c string, id interface{}, set interface{}) bool {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	q := make(map[string]interface{})
+	if sid, ok := id.(string); ok {
+		q["_id"], _ = primitive.ObjectIDFromHex(sid)
+	} else {
+		q["_id"] = id
+	}
+	_, err := m.C.Database(m.DbName).Collection(c).UpdateOne(m.Ctx, q, ObjToM(set))
+	if nil != err {
+		log.Println("UpdateByIdError", err)
+		return false
+	}
+	return true
+}
+
+//批量更新
+func (m *MongodbSim) UpdateBulkAll(db, c string, doc ...[]map[string]interface{}) bool {
+	return m.upSertBulk(db, c, false, doc...)
+}
+
+func (m *MongodbSim) UpdateBulk(c string, doc ...[]map[string]interface{}) bool {
+	return m.UpdateBulkAll(m.DbName, c, doc...)
+}
+
+//批量插入
+func (m *MongodbSim) UpSertBulk(c string, doc ...[]map[string]interface{}) bool {
+	return m.upSertBulk(m.DbName, c, true, doc...)
+}
+
+//批量插入
+func (m *MongodbSim) upSertBulk(db, c string, upsert bool, doc ...[]map[string]interface{}) bool {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(db).Collection(c)
+	var writes []mongo.WriteModel
+	for _, d := range doc {
+		write := mongo.NewUpdateOneModel()
+		write.SetFilter(d[0])
+		write.SetUpdate(d[1])
+		write.SetUpsert(upsert)
+		writes = append(writes, write)
+	}
+	br, e := coll.BulkWrite(m.Ctx, writes)
+	if e != nil {
+		log.Println("mgo upsert error:", e.Error())
+		return br == nil || br.UpsertedCount == 0
+	}
+	//	else {
+	//		if r.UpsertedCount != int64(len(doc)) {
+	//			log.Println("mgo upsert uncomplete:uc/dc", r.UpsertedCount, len(doc))
+	//		}
+	//		return true
+	//	}
+	return true
+}
+
+//查询单条对象
+func (m *MongodbSim) FindOne(c string, query interface{}) (*map[string]interface{}, bool) {
+	return m.FindOneByField(c, query, nil)
+}
+
+//查询单条对象
+func (m *MongodbSim) FindOneByField(c string, query interface{}, fields interface{}) (*map[string]interface{}, bool) {
+	defer catch()
+	res, ok := m.Find(c, query, nil, fields, true, -1, -1)
+	if nil != res && len(*res) > 0 {
+		return &((*res)[0]), ok
+	}
+	return nil, ok
+}
+
+//查询单条对象
+func (m *MongodbSim) FindById(c string, query string, fields interface{}) (*map[string]interface{}, bool) {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	of := options.FindOne()
+	of.SetProjection(ObjToOth(fields))
+	b := false
+	res := make(map[string]interface{})
+	_id, err := primitive.ObjectIDFromHex(query)
+	if err != nil {
+		log.Println("_id error", err)
+		return &res, b
+	}
+	sr := m.C.Database(m.DbName).Collection(c).FindOne(m.Ctx, map[string]interface{}{"_id": _id}, of)
+	if sr.Err() == nil {
+		b = true
+		sr.Decode(&res)
+	}
+	return &res, b
+}
+
+//底层查询方法
+func (m *MongodbSim) Find(c string, query interface{}, order interface{}, fields interface{}, single bool, start int, limit int) (*[]map[string]interface{}, bool) {
+	defer catch()
+	m.Open()
+	defer m.Close()
+	res := make([]map[string]interface{}, 1)
+	coll := m.C.Database(m.DbName).Collection(c)
+	if single {
+		of := options.FindOne()
+		of.SetProjection(ObjToOth(fields))
+		of.SetSort(ObjToM(order))
+		if sr := coll.FindOne(m.Ctx, ObjToM(query), of); sr.Err() == nil {
+			sr.Decode(&res[0])
+		}
+	} else {
+		of := options.Find()
+		of.SetProjection(ObjToOth(fields))
+		of.SetSort(ObjToM(order))
+		if start > -1 {
+			of.SetSkip(int64(start))
+			of.SetLimit(int64(limit))
+		}
+		cur, err := coll.Find(m.Ctx, ObjToM(query), of)
+		if err == nil && cur.Err() == nil {
+			cur.All(m.Ctx, &res)
+		}
+	}
+	return &res, true
+}
+
+func ObjToOth(query interface{}) *bson.M {
+	return ObjToMQ(query, false)
+}
+func ObjToM(query interface{}) *bson.M {
+	return ObjToMQ(query, true)
+}
+
+//obj(string,M)转M,查询用到
+func ObjToMQ(query interface{}, isQuery bool) *bson.M {
+	data := make(bson.M)
+	defer catch()
+	if s2, ok2 := query.(*map[string]interface{}); ok2 {
+		data = bson.M(*s2)
+	} else if s3, ok3 := query.(*bson.M); ok3 {
+		return s3
+	} else if s3, ok3 := query.(*primitive.M); ok3 {
+		return s3
+	} else if s, ok := query.(string); ok {
+		json.Unmarshal([]byte(strings.Replace(s, "'", "\"", -1)), &data)
+		if ss, oks := data["_id"]; oks && isQuery {
+			switch ss.(type) {
+			case string:
+				data["_id"], _ = primitive.ObjectIDFromHex(ss.(string))
+			case map[string]interface{}:
+				tmp := ss.(map[string]interface{})
+				for k, v := range tmp {
+					tmp[k], _ = primitive.ObjectIDFromHex(v.(string))
+				}
+				data["_id"] = tmp
+			}
+
+		}
+	} else if s1, ok1 := query.(map[string]interface{}); ok1 {
+		data = s1
+	} else if s4, ok4 := query.(bson.M); ok4 {
+		data = s4
+	} else if s4, ok4 := query.(primitive.M); ok4 {
+		data = s4
+	} else {
+		data = nil
+	}
+	return &data
+}
+func intAllDef(num interface{}, defaultNum int) int {
+	if i, ok := num.(int); ok {
+		return int(i)
+	} else if i0, ok0 := num.(int32); ok0 {
+		return int(i0)
+	} else if i1, ok1 := num.(float64); ok1 {
+		return int(i1)
+	} else if i2, ok2 := num.(int64); ok2 {
+		return int(i2)
+	} else if i3, ok3 := num.(float32); ok3 {
+		return int(i3)
+	} else if i4, ok4 := num.(string); ok4 {
+		in, _ := strconv.Atoi(i4)
+		return int(in)
+	} else if i5, ok5 := num.(int16); ok5 {
+		return int(i5)
+	} else if i6, ok6 := num.(int8); ok6 {
+		return int(i6)
+	} else if i7, ok7 := num.(*big.Int); ok7 {
+		in, _ := strconv.Atoi(fmt.Sprint(i7))
+		return int(in)
+	} else if i8, ok8 := num.(*big.Float); ok8 {
+		in, _ := strconv.Atoi(fmt.Sprint(i8))
+		return int(in)
+	} else {
+		return defaultNum
+	}
+}
+
+//出错拦截
+func catch() {
+	if r := recover(); r != nil {
+		log.Println(r)
+		for skip := 0; ; skip++ {
+			_, file, line, ok := runtime.Caller(skip)
+			if !ok {
+				break
+			}
+			go log.Printf("%v,%v\n", file, line)
+		}
+	}
+}
+
+//根据bsonID转string
+func BsonIdToSId(uid interface{}) string {
+	if uid == nil {
+		return ""
+	} else if u, ok := uid.(string); ok {
+		return u
+	} else if u, ok := uid.(primitive.ObjectID); ok {
+		return u.Hex()
+	} else {
+		return ""
+	}
+}
+
+func StringTOBsonId(id string) (bid primitive.ObjectID) {
+	defer catch()
+	if id != "" {
+		bid, _ = primitive.ObjectIDFromHex(id)
+	}
+	return
+}

common_utils/mysql.go

@@ -0,0 +1,504 @@
+package util
+
+import (
+	"bytes"
+	"database/sql"
+	"fmt"
+	"log"
+	"reflect"
+	"strings"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+type Mysql struct {
+	Address      string  //数据库地址:端口
+	UserName     string  //用户名
+	PassWord     string  //密码
+	DBName       string  //数据库名
+	DB           *sql.DB //数据库连接池对象
+	MaxOpenConns int     //用于设置最大打开的连接数，默认值为0表示不限制。
+	MaxIdleConns int     //用于设置闲置的连接数。
+}
+
+func (m *Mysql) Init() {
+	if m.MaxOpenConns <= 0 {
+		m.MaxOpenConns = 20
+	}
+	if m.MaxIdleConns <= 0 {
+		m.MaxIdleConns = 20
+	}
+	var err error
+	m.DB, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=true&loc=Local", m.UserName, m.PassWord, m.Address, m.DBName))
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	m.DB.SetMaxOpenConns(m.MaxOpenConns)
+	m.DB.SetMaxIdleConns(m.MaxIdleConns)
+	m.DB.SetConnMaxLifetime(time.Minute * 3)
+	err = m.DB.Ping()
+	if err != nil {
+		log.Println(err)
+	}
+}
+
+//新增
+func (m *Mysql) Insert(tableName string, data map[string]interface{}) int64 {
+	return m.InsertByTx(nil, tableName, data)
+}
+
+//带有事务的新增
+func (m *Mysql) InsertByTx(tx *sql.Tx, tableName string, data map[string]interface{}) int64 {
+	fields := []string{}
+	values := []interface{}{}
+	placeholders := []string{}
+	if tableName == "dataexport_order" {
+		if _, ok := data["user_nickname"]; ok {
+			data["user_nickname"] = ""
+		}
+	}
+	for k, v := range data {
+		fields = append(fields, k)
+		values = append(values, v)
+		placeholders = append(placeholders, "?")
+	}
+	q := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", tableName, strings.Join(fields, ","), strings.Join(placeholders, ","))
+	//log.Println("mysql", q, values)
+	return m.InsertBySqlByTx(tx, q, values...)
+}
+
+//sql语句新增
+func (m *Mysql) InsertBySql(q string, args ...interface{}) int64 {
+	return m.InsertBySqlByTx(nil, q, args...)
+}
+
+//带有事务的sql语句新增
+func (m *Mysql) InsertBySqlByTx(tx *sql.Tx, q string, args ...interface{}) int64 {
+	result, _ := m.ExecBySqlByTx(tx, q, args...)
+	if result == nil {
+		return -1
+	}
+	id, err := result.LastInsertId()
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	return id
+}
+
+//批量新增
+func (m *Mysql) InsertIgnoreBatch(tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.InsertIgnoreBatchByTx(nil, tableName, fields, values)
+}
+
+//带事务的批量新增
+func (m *Mysql) InsertIgnoreBatchByTx(tx *sql.Tx, tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.insertOrReplaceBatchByTx(tx, "INSERT", "IGNORE", tableName, fields, values)
+}
+
+//批量新增
+func (m *Mysql) InsertBatch(tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.InsertBatchByTx(nil, tableName, fields, values)
+}
+
+//带事务的批量新增
+func (m *Mysql) InsertBatchByTx(tx *sql.Tx, tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.insertOrReplaceBatchByTx(tx, "INSERT", "", tableName, fields, values)
+}
+
+//批量更新
+func (m *Mysql) ReplaceBatch(tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.ReplaceBatchByTx(nil, tableName, fields, values)
+}
+
+//带事务的批量更新
+func (m *Mysql) ReplaceBatchByTx(tx *sql.Tx, tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.insertOrReplaceBatchByTx(tx, "REPLACE", "", tableName, fields, values)
+}
+
+func (m *Mysql) insertOrReplaceBatchByTx(tx *sql.Tx, tp string, afterInsert, tableName string, fields []string, values []interface{}) (int64, int64) {
+	placeholders := []string{}
+	for range fields {
+		placeholders = append(placeholders, "?")
+	}
+	placeholder := strings.Join(placeholders, ",")
+	array := []string{}
+	for i := 0; i < len(values)/len(fields); i++ {
+		array = append(array, fmt.Sprintf("(%s)", placeholder))
+	}
+	q := fmt.Sprintf("%s %s INTO %s (%s) VALUES %s", tp, afterInsert, tableName, strings.Join(fields, ","), strings.Join(array, ","))
+	result, _ := m.ExecBySqlByTx(tx, q, values...)
+	if result == nil {
+		return -1, -1
+	}
+	v1, e1 := result.RowsAffected()
+	if e1 != nil {
+		log.Println(e1)
+		return -1, -1
+	}
+	v2, e2 := result.LastInsertId()
+	if e2 != nil {
+		log.Println(e2)
+		return -1, -1
+	}
+	return v1, v2
+}
+
+//sql语句执行
+func (m *Mysql) ExecBySql(q string, args ...interface{}) (sql.Result, error) {
+	return m.ExecBySqlByTx(nil, q, args...)
+}
+
+//sql语句执行,带有事务
+func (m *Mysql) ExecBySqlByTx(tx *sql.Tx, q string, args ...interface{}) (sql.Result, error) {
+	var stmtIns *sql.Stmt
+	var err error
+	if tx == nil {
+		stmtIns, err = m.DB.Prepare(q)
+	} else {
+		stmtIns, err = tx.Prepare(q)
+	}
+	if err != nil {
+		log.Println(err)
+		return nil, err
+	}
+	defer stmtIns.Close()
+	result, err := stmtIns.Exec(args...)
+	if err != nil {
+		log.Println(args, err)
+		return nil, err
+	}
+	return result, nil
+}
+
+/*不等于 map[string]string{"ne":"1"}
+ *不等于多个 map[string]string{"notin":[]interface{}{1,2}}
+ *字段为空 map[string]string{"name":"$isNull"}
+ *字段不为空 map[string]string{"name":"$isNotNull"}
+ */
+func (m *Mysql) Find(tableName string, query map[string]interface{}, fields, order string, start, pageSize int) *[]map[string]interface{} {
+	fs := []string{}
+	vs := []interface{}{}
+	for k, v := range query {
+		rt := reflect.TypeOf(v)
+		rv := reflect.ValueOf(v)
+		if rt.Kind() == reflect.Map {
+			for _, rv_k := range rv.MapKeys() {
+				if rv_k.String() == "ne" {
+					fs = append(fs, fmt.Sprintf("%s!=?", k))
+					vs = append(vs, rv.MapIndex(rv_k).Interface())
+				}
+				if rv_k.String() == "notin" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						for _, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							fs = append(fs, fmt.Sprintf("%s!=?", k))
+							vs = append(vs, v)
+						}
+					}
+				}
+				if rv_k.String() == "in" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						_fs := fmt.Sprintf("%s in (?", k)
+						for k, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							if k > 0 {
+								_fs += ",?"
+							}
+							vs = append(vs, v)
+						}
+						_fs += ")"
+						fs = append(fs, _fs)
+					}
+				}
+			}
+		} else {
+			if v == "$isNull" {
+				fs = append(fs, fmt.Sprintf("%s is null", k))
+			} else if v == "$isNotNull" {
+				fs = append(fs, fmt.Sprintf("%s is not null", k))
+			} else {
+				fs = append(fs, fmt.Sprintf("%s=?", k))
+				vs = append(vs, v)
+			}
+		}
+	}
+	var buffer bytes.Buffer
+	buffer.WriteString("select ")
+	if fields == "" {
+		buffer.WriteString("*")
+	} else {
+		buffer.WriteString(fields)
+	}
+	buffer.WriteString(" from ")
+	buffer.WriteString(tableName)
+	if len(fs) > 0 {
+		buffer.WriteString(" where ")
+		buffer.WriteString(strings.Join(fs, " and "))
+	}
+	if order != "" {
+		buffer.WriteString(" order by ")
+		buffer.WriteString(order)
+	}
+	if start > -1 && pageSize > 0 {
+		buffer.WriteString(" limit ")
+		buffer.WriteString(fmt.Sprint(start))
+		buffer.WriteString(",")
+		buffer.WriteString(fmt.Sprint(pageSize))
+	}
+	q := buffer.String()
+	//log.Println(q, vs)
+	return m.SelectBySql(q, vs...)
+}
+
+//sql语句查询
+func (m *Mysql) SelectBySql(q string, args ...interface{}) *[]map[string]interface{} {
+	return m.SelectBySqlByTx(nil, q, args...)
+}
+func (m *Mysql) SelectBySqlByTx(tx *sql.Tx, q string, args ...interface{}) *[]map[string]interface{} {
+	return m.Select(0, nil, tx, q, args...)
+}
+func (m *Mysql) Select(bath int, f func(l *[]map[string]interface{}), tx *sql.Tx, q string, args ...interface{}) *[]map[string]interface{} {
+	var stmtOut *sql.Stmt
+	var err error
+	if tx == nil {
+		stmtOut, err = m.DB.Prepare(q)
+	} else {
+		stmtOut, err = tx.Prepare(q)
+	}
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	defer stmtOut.Close()
+	rows, err := stmtOut.Query(args...)
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	if rows != nil {
+		defer rows.Close()
+	}
+	columns, err := rows.Columns()
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	list := []map[string]interface{}{}
+	for rows.Next() {
+		scanArgs := make([]interface{}, len(columns))
+		values := make([]interface{}, len(columns))
+		ret := make(map[string]interface{})
+		for k, _ := range values {
+			scanArgs[k] = &values[k]
+		}
+		err = rows.Scan(scanArgs...)
+		if err != nil {
+			log.Println(err)
+			break
+		}
+		for i, col := range values {
+			if v, ok := col.([]uint8); ok {
+				ret[columns[i]] = string(v)
+			} else {
+				ret[columns[i]] = col
+			}
+		}
+		list = append(list, ret)
+		if bath > 0 && len(list) == bath {
+			f(&list)
+			list = []map[string]interface{}{}
+		}
+	}
+	if bath > 0 && len(list) > 0 {
+		f(&list)
+		list = []map[string]interface{}{}
+	}
+	return &list
+}
+func (m *Mysql) SelectByBath(bath int, f func(l *[]map[string]interface{}), q string, args ...interface{}) {
+	m.SelectByBathByTx(bath, f, nil, q, args...)
+}
+func (m *Mysql) SelectByBathByTx(bath int, f func(l *[]map[string]interface{}), tx *sql.Tx, q string, args ...interface{}) {
+	m.Select(bath, f, tx, q, args...)
+}
+func (m *Mysql) FindOne(tableName string, query map[string]interface{}, fields, order string) *map[string]interface{} {
+	list := m.Find(tableName, query, fields, order, 0, 1)
+	if list != nil && len(*list) == 1 {
+		temp := (*list)[0]
+		return &temp
+	}
+	return nil
+}
+
+//修改
+func (m *Mysql) Update(tableName string, query, update map[string]interface{}) bool {
+	return m.UpdateByTx(nil, tableName, query, update)
+}
+
+//带事务的修改
+func (m *Mysql) UpdateByTx(tx *sql.Tx, tableName string, query, update map[string]interface{}) bool {
+	q_fs := []string{}
+	u_fs := []string{}
+	values := []interface{}{}
+	for k, v := range update {
+		q_fs = append(q_fs, fmt.Sprintf("%s=?", k))
+		values = append(values, v)
+	}
+	for k, v := range query {
+		u_fs = append(u_fs, fmt.Sprintf("%s=?", k))
+		values = append(values, v)
+	}
+	q := fmt.Sprintf("update %s set %s where %s", tableName, strings.Join(q_fs, ","), strings.Join(u_fs, " and "))
+	log.Println(q, values)
+	return m.UpdateOrDeleteBySqlByTx(tx, q, values...) >= 0
+}
+
+//删除
+func (m *Mysql) Delete(tableName string, query map[string]interface{}) bool {
+	return m.DeleteByTx(nil, tableName, query)
+}
+func (m *Mysql) DeleteByTx(tx *sql.Tx, tableName string, query map[string]interface{}) bool {
+	fields := []string{}
+	values := []interface{}{}
+	for k, v := range query {
+		fields = append(fields, fmt.Sprintf("%s=?", k))
+		values = append(values, v)
+	}
+	q := fmt.Sprintf("delete from %s where %s", tableName, strings.Join(fields, " and "))
+	log.Println(q, values)
+	return m.UpdateOrDeleteBySqlByTx(tx, q, values...) > 0
+}
+
+//修改或删除
+func (m *Mysql) UpdateOrDeleteBySql(q string, args ...interface{}) int64 {
+	return m.UpdateOrDeleteBySqlByTx(nil, q, args...)
+}
+
+//带事务的修改或删除
+func (m *Mysql) UpdateOrDeleteBySqlByTx(tx *sql.Tx, q string, args ...interface{}) int64 {
+	result, err := m.ExecBySqlByTx(tx, q, args...)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	count, err := result.RowsAffected()
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	return count
+}
+
+//总数
+func (m *Mysql) Count(tableName string, query map[string]interface{}) int64 {
+	fields := []string{}
+	values := []interface{}{}
+	for k, v := range query {
+		rt := reflect.TypeOf(v)
+		rv := reflect.ValueOf(v)
+		if rt.Kind() == reflect.Map {
+			for _, rv_k := range rv.MapKeys() {
+				if rv_k.String() == "ne" {
+					fields = append(fields, fmt.Sprintf("%s!=?", k))
+					values = append(values, rv.MapIndex(rv_k).Interface())
+				}
+				if rv_k.String() == "notin" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						for _, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							fields = append(fields, fmt.Sprintf("%s!=?", k))
+							values = append(values, v)
+						}
+					}
+				}
+				if rv_k.String() == "in" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						_fs := fmt.Sprintf("%s in (?", k)
+						for k, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							if k > 0 {
+								_fs += ",?"
+							}
+							values = append(values, v)
+						}
+						_fs += ")"
+						fields = append(fields, _fs)
+					}
+				}
+			}
+		} else if v == "$isNull" {
+			fields = append(fields, fmt.Sprintf("%s is null", k))
+		} else if v == "$isNotNull" {
+			fields = append(fields, fmt.Sprintf("%s is not null", k))
+		} else {
+			fields = append(fields, fmt.Sprintf("%s=?", k))
+			values = append(values, v)
+		}
+	}
+	q := fmt.Sprintf("select count(1) as count from %s", tableName)
+	if len(query) > 0 {
+		q += fmt.Sprintf(" where %s", strings.Join(fields, " and "))
+	}
+	log.Println(q, values)
+	return m.CountBySql(q, values...)
+}
+func (m *Mysql) CountBySql(q string, args ...interface{}) int64 {
+	stmtIns, err := m.DB.Prepare(q)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	defer stmtIns.Close()
+
+	rows, err := stmtIns.Query(args...)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	if rows != nil {
+		defer rows.Close()
+	}
+	var count int64 = -1
+	if rows.Next() {
+		err = rows.Scan(&count)
+		if err != nil {
+			log.Println(err)
+		}
+	}
+	return count
+}
+
+//执行事务
+func (m *Mysql) ExecTx(msg string, f func(tx *sql.Tx) bool) bool {
+	tx, err := m.DB.Begin()
+	if err != nil {
+		log.Println(msg, "获取事务错误", err)
+	} else {
+		if f(tx) {
+			if err := tx.Commit(); err != nil {
+				log.Println(msg, "提交事务错误", err)
+			} else {
+				return true
+			}
+		} else {
+			if err := tx.Rollback(); err != nil {
+				log.Println(msg, "事务回滚错误", err)
+			}
+		}
+	}
+	return false
+}
+
+/*************方法命名不规范，上面有替代方法*************/
+func (m *Mysql) Query(query string, args ...interface{}) *[]map[string]interface{} {
+	return m.SelectBySql(query, args...)
+}
+
+func (m *Mysql) QueryCount(query string, args ...interface{}) (count int) {
+	count = -1
+	if !strings.Contains(strings.ToLower(query), "count(*)") {
+		fmt.Println("QueryCount need query like < select count(*) from ..... >")
+		return
+	}
+	count = int(m.CountBySql(query, args...))
+	return
+}

common_utils/mysqldb/mysql.go

@@ -0,0 +1,504 @@
+package mysqldb
+
+import (
+	"bytes"
+	"database/sql"
+	"fmt"
+	"log"
+	"reflect"
+	"strings"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+type Mysql struct {
+	Address      string  //数据库地址:端口
+	UserName     string  //用户名
+	PassWord     string  //密码
+	DBName       string  //数据库名
+	DB           *sql.DB //数据库连接池对象
+	MaxOpenConns int     //用于设置最大打开的连接数，默认值为0表示不限制。
+	MaxIdleConns int     //用于设置闲置的连接数。
+}
+
+func (m *Mysql) Init() {
+	if m.MaxOpenConns <= 0 {
+		m.MaxOpenConns = 20
+	}
+	if m.MaxIdleConns <= 0 {
+		m.MaxIdleConns = 20
+	}
+	var err error
+	m.DB, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=true&loc=Local", m.UserName, m.PassWord, m.Address, m.DBName))
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	m.DB.SetMaxOpenConns(m.MaxOpenConns)
+	m.DB.SetMaxIdleConns(m.MaxIdleConns)
+	m.DB.SetConnMaxLifetime(time.Minute * 3)
+	err = m.DB.Ping()
+	if err != nil {
+		log.Println(err)
+	}
+}
+
+//新增
+func (m *Mysql) Insert(tableName string, data map[string]interface{}) int64 {
+	return m.InsertByTx(nil, tableName, data)
+}
+
+//带有事务的新增
+func (m *Mysql) InsertByTx(tx *sql.Tx, tableName string, data map[string]interface{}) int64 {
+	fields := []string{}
+	values := []interface{}{}
+	placeholders := []string{}
+	if tableName == "dataexport_order" {
+		if _, ok := data["user_nickname"]; ok {
+			data["user_nickname"] = ""
+		}
+	}
+	for k, v := range data {
+		fields = append(fields, k)
+		values = append(values, v)
+		placeholders = append(placeholders, "?")
+	}
+	q := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", tableName, strings.Join(fields, ","), strings.Join(placeholders, ","))
+	//log.Println("mysql", q, values)
+	return m.InsertBySqlByTx(tx, q, values...)
+}
+
+//sql语句新增
+func (m *Mysql) InsertBySql(q string, args ...interface{}) int64 {
+	return m.InsertBySqlByTx(nil, q, args...)
+}
+
+//带有事务的sql语句新增
+func (m *Mysql) InsertBySqlByTx(tx *sql.Tx, q string, args ...interface{}) int64 {
+	result, _ := m.ExecBySqlByTx(tx, q, args...)
+	if result == nil {
+		return -1
+	}
+	id, err := result.LastInsertId()
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	return id
+}
+
+//批量新增
+func (m *Mysql) InsertIgnoreBatch(tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.InsertIgnoreBatchByTx(nil, tableName, fields, values)
+}
+
+//带事务的批量新增
+func (m *Mysql) InsertIgnoreBatchByTx(tx *sql.Tx, tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.insertOrReplaceBatchByTx(tx, "INSERT", "IGNORE", tableName, fields, values)
+}
+
+//批量新增
+func (m *Mysql) InsertBatch(tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.InsertBatchByTx(nil, tableName, fields, values)
+}
+
+//带事务的批量新增
+func (m *Mysql) InsertBatchByTx(tx *sql.Tx, tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.insertOrReplaceBatchByTx(tx, "INSERT", "", tableName, fields, values)
+}
+
+//批量更新
+func (m *Mysql) ReplaceBatch(tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.ReplaceBatchByTx(nil, tableName, fields, values)
+}
+
+//带事务的批量更新
+func (m *Mysql) ReplaceBatchByTx(tx *sql.Tx, tableName string, fields []string, values []interface{}) (int64, int64) {
+	return m.insertOrReplaceBatchByTx(tx, "REPLACE", "", tableName, fields, values)
+}
+
+func (m *Mysql) insertOrReplaceBatchByTx(tx *sql.Tx, tp string, afterInsert, tableName string, fields []string, values []interface{}) (int64, int64) {
+	placeholders := []string{}
+	for range fields {
+		placeholders = append(placeholders, "?")
+	}
+	placeholder := strings.Join(placeholders, ",")
+	array := []string{}
+	for i := 0; i < len(values)/len(fields); i++ {
+		array = append(array, fmt.Sprintf("(%s)", placeholder))
+	}
+	q := fmt.Sprintf("%s %s INTO %s (%s) VALUES %s", tp, afterInsert, tableName, strings.Join(fields, ","), strings.Join(array, ","))
+	result, _ := m.ExecBySqlByTx(tx, q, values...)
+	if result == nil {
+		return -1, -1
+	}
+	v1, e1 := result.RowsAffected()
+	if e1 != nil {
+		log.Println(e1)
+		return -1, -1
+	}
+	v2, e2 := result.LastInsertId()
+	if e2 != nil {
+		log.Println(e2)
+		return -1, -1
+	}
+	return v1, v2
+}
+
+//sql语句执行
+func (m *Mysql) ExecBySql(q string, args ...interface{}) (sql.Result, error) {
+	return m.ExecBySqlByTx(nil, q, args...)
+}
+
+//sql语句执行,带有事务
+func (m *Mysql) ExecBySqlByTx(tx *sql.Tx, q string, args ...interface{}) (sql.Result, error) {
+	var stmtIns *sql.Stmt
+	var err error
+	if tx == nil {
+		stmtIns, err = m.DB.Prepare(q)
+	} else {
+		stmtIns, err = tx.Prepare(q)
+	}
+	if err != nil {
+		log.Println(err)
+		return nil, err
+	}
+	defer stmtIns.Close()
+	result, err := stmtIns.Exec(args...)
+	if err != nil {
+		log.Println(args, err)
+		return nil, err
+	}
+	return result, nil
+}
+
+/*不等于 map[string]string{"ne":"1"}
+ *不等于多个 map[string]string{"notin":[]interface{}{1,2}}
+ *字段为空 map[string]string{"name":"$isNull"}
+ *字段不为空 map[string]string{"name":"$isNotNull"}
+ */
+func (m *Mysql) Find(tableName string, query map[string]interface{}, fields, order string, start, pageSize int) *[]map[string]interface{} {
+	fs := []string{}
+	vs := []interface{}{}
+	for k, v := range query {
+		rt := reflect.TypeOf(v)
+		rv := reflect.ValueOf(v)
+		if rt.Kind() == reflect.Map {
+			for _, rv_k := range rv.MapKeys() {
+				if rv_k.String() == "ne" {
+					fs = append(fs, fmt.Sprintf("%s!=?", k))
+					vs = append(vs, rv.MapIndex(rv_k).Interface())
+				}
+				if rv_k.String() == "notin" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						for _, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							fs = append(fs, fmt.Sprintf("%s!=?", k))
+							vs = append(vs, v)
+						}
+					}
+				}
+				if rv_k.String() == "in" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						_fs := fmt.Sprintf("%s in (?", k)
+						for k, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							if k > 0 {
+								_fs += ",?"
+							}
+							vs = append(vs, v)
+						}
+						_fs += ")"
+						fs = append(fs, _fs)
+					}
+				}
+			}
+		} else {
+			if v == "$isNull" {
+				fs = append(fs, fmt.Sprintf("%s is null", k))
+			} else if v == "$isNotNull" {
+				fs = append(fs, fmt.Sprintf("%s is not null", k))
+			} else {
+				fs = append(fs, fmt.Sprintf("%s=?", k))
+				vs = append(vs, v)
+			}
+		}
+	}
+	var buffer bytes.Buffer
+	buffer.WriteString("select ")
+	if fields == "" {
+		buffer.WriteString("*")
+	} else {
+		buffer.WriteString(fields)
+	}
+	buffer.WriteString(" from ")
+	buffer.WriteString(tableName)
+	if len(fs) > 0 {
+		buffer.WriteString(" where ")
+		buffer.WriteString(strings.Join(fs, " and "))
+	}
+	if order != "" {
+		buffer.WriteString(" order by ")
+		buffer.WriteString(order)
+	}
+	if start > -1 && pageSize > 0 {
+		buffer.WriteString(" limit ")
+		buffer.WriteString(fmt.Sprint(start))
+		buffer.WriteString(",")
+		buffer.WriteString(fmt.Sprint(pageSize))
+	}
+	q := buffer.String()
+	//log.Println(q, vs)
+	return m.SelectBySql(q, vs...)
+}
+
+//sql语句查询
+func (m *Mysql) SelectBySql(q string, args ...interface{}) *[]map[string]interface{} {
+	return m.SelectBySqlByTx(nil, q, args...)
+}
+func (m *Mysql) SelectBySqlByTx(tx *sql.Tx, q string, args ...interface{}) *[]map[string]interface{} {
+	return m.Select(0, nil, tx, q, args...)
+}
+func (m *Mysql) Select(bath int, f func(l *[]map[string]interface{}), tx *sql.Tx, q string, args ...interface{}) *[]map[string]interface{} {
+	var stmtOut *sql.Stmt
+	var err error
+	if tx == nil {
+		stmtOut, err = m.DB.Prepare(q)
+	} else {
+		stmtOut, err = tx.Prepare(q)
+	}
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	defer stmtOut.Close()
+	rows, err := stmtOut.Query(args...)
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	if rows != nil {
+		defer rows.Close()
+	}
+	columns, err := rows.Columns()
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	list := []map[string]interface{}{}
+	for rows.Next() {
+		scanArgs := make([]interface{}, len(columns))
+		values := make([]interface{}, len(columns))
+		ret := make(map[string]interface{})
+		for k, _ := range values {
+			scanArgs[k] = &values[k]
+		}
+		err = rows.Scan(scanArgs...)
+		if err != nil {
+			log.Println(err)
+			break
+		}
+		for i, col := range values {
+			if v, ok := col.([]uint8); ok {
+				ret[columns[i]] = string(v)
+			} else {
+				ret[columns[i]] = col
+			}
+		}
+		list = append(list, ret)
+		if bath > 0 && len(list) == bath {
+			f(&list)
+			list = []map[string]interface{}{}
+		}
+	}
+	if bath > 0 && len(list) > 0 {
+		f(&list)
+		list = []map[string]interface{}{}
+	}
+	return &list
+}
+func (m *Mysql) SelectByBath(bath int, f func(l *[]map[string]interface{}), q string, args ...interface{}) {
+	m.SelectByBathByTx(bath, f, nil, q, args...)
+}
+func (m *Mysql) SelectByBathByTx(bath int, f func(l *[]map[string]interface{}), tx *sql.Tx, q string, args ...interface{}) {
+	m.Select(bath, f, tx, q, args...)
+}
+func (m *Mysql) FindOne(tableName string, query map[string]interface{}, fields, order string) *map[string]interface{} {
+	list := m.Find(tableName, query, fields, order, 0, 1)
+	if list != nil && len(*list) == 1 {
+		temp := (*list)[0]
+		return &temp
+	}
+	return nil
+}
+
+//修改
+func (m *Mysql) Update(tableName string, query, update map[string]interface{}) bool {
+	return m.UpdateByTx(nil, tableName, query, update)
+}
+
+//带事务的修改
+func (m *Mysql) UpdateByTx(tx *sql.Tx, tableName string, query, update map[string]interface{}) bool {
+	q_fs := []string{}
+	u_fs := []string{}
+	values := []interface{}{}
+	for k, v := range update {
+		q_fs = append(q_fs, fmt.Sprintf("%s=?", k))
+		values = append(values, v)
+	}
+	for k, v := range query {
+		u_fs = append(u_fs, fmt.Sprintf("%s=?", k))
+		values = append(values, v)
+	}
+	q := fmt.Sprintf("update %s set %s where %s", tableName, strings.Join(q_fs, ","), strings.Join(u_fs, " and "))
+	log.Println(q, values)
+	return m.UpdateOrDeleteBySqlByTx(tx, q, values...) >= 0
+}
+
+//删除
+func (m *Mysql) Delete(tableName string, query map[string]interface{}) bool {
+	return m.DeleteByTx(nil, tableName, query)
+}
+func (m *Mysql) DeleteByTx(tx *sql.Tx, tableName string, query map[string]interface{}) bool {
+	fields := []string{}
+	values := []interface{}{}
+	for k, v := range query {
+		fields = append(fields, fmt.Sprintf("%s=?", k))
+		values = append(values, v)
+	}
+	q := fmt.Sprintf("delete from %s where %s", tableName, strings.Join(fields, " and "))
+	log.Println(q, values)
+	return m.UpdateOrDeleteBySqlByTx(tx, q, values...) > 0
+}
+
+//修改或删除
+func (m *Mysql) UpdateOrDeleteBySql(q string, args ...interface{}) int64 {
+	return m.UpdateOrDeleteBySqlByTx(nil, q, args...)
+}
+
+//带事务的修改或删除
+func (m *Mysql) UpdateOrDeleteBySqlByTx(tx *sql.Tx, q string, args ...interface{}) int64 {
+	result, err := m.ExecBySqlByTx(tx, q, args...)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	count, err := result.RowsAffected()
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	return count
+}
+
+//总数
+func (m *Mysql) Count(tableName string, query map[string]interface{}) int64 {
+	fields := []string{}
+	values := []interface{}{}
+	for k, v := range query {
+		rt := reflect.TypeOf(v)
+		rv := reflect.ValueOf(v)
+		if rt.Kind() == reflect.Map {
+			for _, rv_k := range rv.MapKeys() {
+				if rv_k.String() == "ne" {
+					fields = append(fields, fmt.Sprintf("%s!=?", k))
+					values = append(values, rv.MapIndex(rv_k).Interface())
+				}
+				if rv_k.String() == "notin" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						for _, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							fields = append(fields, fmt.Sprintf("%s!=?", k))
+							values = append(values, v)
+						}
+					}
+				}
+				if rv_k.String() == "in" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						_fs := fmt.Sprintf("%s in (?", k)
+						for k, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							if k > 0 {
+								_fs += ",?"
+							}
+							values = append(values, v)
+						}
+						_fs += ")"
+						fields = append(fields, _fs)
+					}
+				}
+			}
+		} else if v == "$isNull" {
+			fields = append(fields, fmt.Sprintf("%s is null", k))
+		} else if v == "$isNotNull" {
+			fields = append(fields, fmt.Sprintf("%s is not null", k))
+		} else {
+			fields = append(fields, fmt.Sprintf("%s=?", k))
+			values = append(values, v)
+		}
+	}
+	q := fmt.Sprintf("select count(1) as count from %s", tableName)
+	if len(query) > 0 {
+		q += fmt.Sprintf(" where %s", strings.Join(fields, " and "))
+	}
+	log.Println(q, values)
+	return m.CountBySql(q, values...)
+}
+func (m *Mysql) CountBySql(q string, args ...interface{}) int64 {
+	stmtIns, err := m.DB.Prepare(q)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	defer stmtIns.Close()
+
+	rows, err := stmtIns.Query(args...)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+	if rows != nil {
+		defer rows.Close()
+	}
+	var count int64 = -1
+	if rows.Next() {
+		err = rows.Scan(&count)
+		if err != nil {
+			log.Println(err)
+		}
+	}
+	return count
+}
+
+//执行事务
+func (m *Mysql) ExecTx(msg string, f func(tx *sql.Tx) bool) bool {
+	tx, err := m.DB.Begin()
+	if err != nil {
+		log.Println(msg, "获取事务错误", err)
+	} else {
+		if f(tx) {
+			if err := tx.Commit(); err != nil {
+				log.Println(msg, "提交事务错误", err)
+			} else {
+				return true
+			}
+		} else {
+			if err := tx.Rollback(); err != nil {
+				log.Println(msg, "事务回滚错误", err)
+			}
+		}
+	}
+	return false
+}
+
+/*************方法命名不规范，上面有替代方法*************/
+func (m *Mysql) Query(query string, args ...interface{}) *[]map[string]interface{} {
+	return m.SelectBySql(query, args...)
+}
+
+func (m *Mysql) QueryCount(query string, args ...interface{}) (count int) {
+	count = -1
+	if !strings.Contains(strings.ToLower(query), "count(*)") {
+		fmt.Println("QueryCount need query like < select count(*) from ..... >")
+		return
+	}
+	count = int(m.CountBySql(query, args...))
+	return
+}

common_utils/nsq/consumer.go

@@ -0,0 +1,92 @@
+package gonsq
+
+import (
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/nsqio/go-nsq"
+)
+
+type Consumer struct {
+	Ch           chan interface{}
+	C            *nsq.Consumer
+	Topic        string
+	Channel      string
+	IsJsonEncode bool
+	Conf         *Cconfig
+}
+
+type Cconfig struct {
+	IsJsonEncode         bool   //是否进行json序列化，解码也进行序列化,默认不进行json序列化
+	ConnectType          int    //连接类型 0连nsqd 1连nsqlookup
+	Interval             int    //设置服务发现的轮询时间,例如新的nsq出现,默认10秒
+	Addr, Topic, Channel string //连接地址（支持逗号分割多个），主题，通道
+	Concurrent           int    //并发数，默认为1
+}
+
+//处理消息
+func (c *Consumer) HandleMessage(msg *nsq.Message) error {
+	if c.IsJsonEncode {
+		if len(msg.Body) > 1 {
+			var err error
+			switch msg.Body[0] {
+			case 0x00:
+				var obj interface{}
+				err = json.Unmarshal(msg.Body[1:], &obj)
+				if err == nil && obj != nil {
+					c.Ch <- obj
+				}
+			case 0x01: //[]byte数组
+				var obj []byte
+				err = json.Unmarshal(msg.Body[1:], &obj)
+				if err == nil && obj != nil {
+					c.Ch <- obj
+				}
+			default:
+				var obj interface{}
+				err = json.Unmarshal(msg.Body, &obj)
+				if err == nil && obj != nil {
+					c.Ch <- obj
+				}
+			}
+			return err
+		}
+	} else {
+		c.Ch <- msg.Body
+	}
+	return nil
+}
+
+func NewConsumer(cc *Cconfig) (*Consumer, error) {
+	cfg := nsq.NewConfig()
+	if cc.Interval == 0 {
+		cc.Interval = 10
+	}
+	cfg.LookupdPollInterval = time.Duration(cc.Interval) * time.Second //设置服务发现的轮询时间,例如新的nsq出现
+	c, err := nsq.NewConsumer(cc.Topic, cc.Channel, cfg)               // 新建一个消费者
+	if err != nil {
+		return nil, err
+	}
+	if cc.Concurrent == 0 {
+		cc.Concurrent = 1
+	}
+	consumer := &Consumer{make(chan interface{}, cc.Concurrent), c, cc.Topic, cc.Channel, cc.IsJsonEncode, cc}
+	c.AddConcurrentHandlers(consumer, cc.Concurrent) // 添加消费者接口
+	addrs := strings.Split(cc.Addr, ",")
+	var err1 error
+	if cc.ConnectType == 0 {
+		err1 = c.ConnectToNSQDs(addrs)
+	} else if cc.ConnectType == 1 {
+		err1 = c.ConnectToNSQLookupds(addrs)
+	}
+	return consumer, err1
+}
+
+//处理消息
+func (c *Consumer) Close(msg *nsq.Message) error {
+	if c.Conf.ConnectType == 1 {
+		return c.C.DisconnectFromNSQLookupd(c.Conf.Addr)
+	}
+	return c.C.DisconnectFromNSQD(c.Conf.Addr)
+}

common_utils/nsq/producer.go

@@ -0,0 +1,51 @@
+package gonsq
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/nsqio/go-nsq"
+)
+
+type Producer struct {
+	//Ch    chan interface{}
+	P            *nsq.Producer
+	Topic        string
+	IsJsonEncode bool //是否进行json序列化，如果否则必须以[]byte传递，如果是则必须用对应的消费者对象[也设置了序列化]处理
+}
+
+func NewProducer(addr, toppic string, IsJsonEncode bool) (*Producer, error) {
+	config := nsq.NewConfig()
+	producer, err := nsq.NewProducer(addr, config)
+	if err != nil {
+		return nil, err
+	} else {
+		return &Producer{producer, toppic, IsJsonEncode}, nil
+	}
+}
+
+func (p *Producer) Publish(msg interface{}) error {
+	if p.IsJsonEncode {
+		//var infoType byte
+		//switch msg.(type) {
+		//case []byte: //原本就是byte数组
+		//	infoType = 0x01
+		//default:
+		//	infoType = 0x00
+		//}
+		data, err := json.Marshal(msg)
+		if err != nil {
+			return err
+		} else if len(data) > 0 { //头部插入类型，用于解码[]byte
+			//data = append([]byte{infoType}, data...)
+			return p.P.Publish(p.Topic, data)
+		} else {
+			return errors.New("producer msg err")
+		}
+	} else { //必须传入[]byte
+		if bs, ok := msg.([]byte); ok {
+			return p.P.Publish(p.Topic, bs)
+		}
+		return errors.New("producer msg err: no []byte")
+	}
+}

common_utils/protocol.go

@@ -0,0 +1,161 @@
+package util
+
+//底层网络通信协议
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+)
+
+const (
+	EMPTY_ADDR = "00000000"
+)
+
+//读取
+type Reader struct {
+	conn   net.Conn
+	reader *bufio.Reader
+	buffer [4]byte
+}
+
+//数据包，实际上就是协议定义
+type Packet struct {
+	//头开始
+	Length     int32  //4bit 包长度
+	From       string //8bit
+	To         string //8bit
+	Msgid      string //8bit
+	Event      int32  //4bit
+	SentToType int32  //4bit
+	//头结束
+	Raw []byte //数据包，头+数据
+}
+
+//返回
+func (p *Packet) GetBusinessData() []byte {
+	return p.Raw[36:]
+}
+
+//
+type RawData []byte //流数据
+
+//
+func NewReader(c net.Conn) *Reader {
+	return &Reader{
+		conn:   c,
+		reader: bufio.NewReader(c),
+	}
+}
+
+//读取头部
+func (p *Reader) readHeader() (int32, error) {
+	buf := p.buffer[:4]
+	if _, err := io.ReadFull(p.reader, buf); err != nil {
+		return 0, err
+	}
+	size := int32(binary.BigEndian.Uint32(buf))
+	if size < 32 || size > 16384000 {
+		return 0, fmt.Errorf("Incorrect frame size (%d)", size)
+	}
+	return size, nil
+}
+
+//读取完整一帧数据，防止粘包
+func (p *Reader) readFrame(size int32) (*Packet, error) {
+	var buf []byte
+	if int(size) <= len(p.buffer) {
+		buf = p.buffer[0:size]
+	} else {
+		buf = make([]byte, size)
+	}
+	_, err := io.ReadFull(p.reader, buf)
+	raw := append(Int2Byte(size), buf...)
+	if err == nil {
+		//解析包
+		packet := &Packet{
+			Length:     size,
+			From:       string(buf[:8]),
+			To:         string(buf[8:16]),
+			Msgid:      string(buf[16:24]),
+			Event:      Byte2Int(buf[24:28]),
+			SentToType: Byte2Int(buf[28:32]),
+			//Data:       buf[32:],
+			Raw: raw,
+		}
+		return packet, nil
+	}
+	return nil, err
+}
+
+//读取数据写入队列
+func forwardMessage(c net.Conn, queue chan<- *Packet) {
+	defer c.Close()
+	logReader := NewReader(c)
+	for {
+		size, err := logReader.readHeader()
+		if err != nil {
+			break
+		}
+		packet, err := logReader.readFrame(size)
+		if err != nil {
+			break
+		}
+		queue <- packet
+	}
+}
+
+//从队列中读取数据，定期处理
+func processMsg(q <-chan *Packet, parseEvent func(*Packet)) {
+	for {
+		select {
+		case msg := <-q:
+			go parseEvent(msg)
+		}
+	}
+}
+
+//封包，输出直接可以写到流中的数据包
+func Enpacket(from, to, msgid string, event, sendtotype int, obj interface{}) []byte {
+	if len(from) != 8 {
+		from = EMPTY_ADDR
+	}
+	if len(to) != 8 {
+		to = EMPTY_ADDR
+	}
+	if len(msgid) != 8 {
+		msgid = EMPTY_ADDR
+	}
+	var ret []byte
+	var bs []byte
+	if v, ok := obj.([]byte); ok {
+		bs = v
+	} else if v, ok := obj.(string); ok {
+		bs = []byte(v)
+	} else {
+		bs, _ = json.Marshal(obj)
+	}
+	ret = append(Int2Byte(int32(32 + len(bs))))
+	ret = append(ret, []byte(from+to+msgid)...)
+	ret = append(ret, Int2Byte(int32(event))...)
+	ret = append(ret, Int2Byte(int32(sendtotype))...)
+	ret = append(ret, bs...)
+	return ret
+}
+
+//
+func Byte2Int(src []byte) int32 {
+	var ret int32
+	binary.Read(bytes.NewReader(src), binary.BigEndian, &ret)
+	return ret
+}
+
+//
+func Int2Byte(src int32) []byte {
+	buf := bytes.NewBuffer([]byte{})
+	binary.Write(buf, binary.BigEndian, src)
+	return buf.Bytes()
+}

common_utils/redis/redisloginutil.go

@@ -0,0 +1,70 @@
+package redis
+
+import (
+	"log"
+	"strings"
+	"time"
+
+	redisLogin "github.com/gomodule/redigo/redis"
+)
+
+var RedisLoginPool *redisLogin.Pool
+
+func InitRedisLogin(addrs string) {
+	addr := strings.Split(addrs, ",")
+	for _, v := range addr {
+		saddr := strings.Split(v, "=")
+		if saddr[0] == "login" {
+			RedisLoginPool = &redisLogin.Pool{MaxActive: 10, MaxIdle: 5,
+				IdleTimeout: time.Duration(10) * time.Second, Dial: func() (redisLogin.Conn, error) {
+					c, err := redisLogin.Dial("tcp", saddr[1])
+					if err != nil {
+						return nil, err
+					}
+					return c, nil
+				}}
+		}
+	}
+
+}
+
+//
+func SetLoginVal(key, value string) {
+	conn := RedisLoginPool.Get()
+	defer conn.Close()
+	conn.Do("PUBLISH", key, value)
+}
+
+//
+func GetLoginVal(key string, wxFunc func(wxParams []string) bool) {
+	for {
+		defer catch()
+	L:
+		for {
+			conn := RedisLoginPool.Get()
+			defer conn.Close()
+			if conn.Err() == nil {
+				psc := redisLogin.PubSubConn{Conn: conn}
+				if err := psc.Subscribe(redisLogin.Args{}.AddFlat(key)...); err != nil {
+					log.Println(err)
+				}
+				for {
+					msg := psc.Receive()
+					//					go func(msg interface{}) {
+					switch n := msg.(type) {
+					case error:
+						log.Println("wxlogin err", msg)
+						break L
+					case redisLogin.Message:
+						res := string(n.Data)
+						param := strings.Split(res, ",")
+						go wxFunc(param)
+					}
+					//					}(msg)
+				}
+			}
+			time.Sleep(2 * time.Second)
+		}
+		time.Sleep(1 * time.Second)
+	}
+}

common_utils/redis/redisutil.go

@@ -0,0 +1,547 @@
+package redis
+
+import (
+	"encoding/json"
+	"errors"
+	redigo "github.com/gomodule/redigo/redis"
+	"log"
+	"runtime"
+	"strings"
+	"time"
+)
+
+var RedisPool map[string]*redigo.Pool
+
+//初始化redis 1为多个连接池，2为共用一个连接池
+func InitRedis(addrs string) {
+	InitRedisBySize(addrs, 300, 30, 240)
+}
+
+func InitRedis1(addrs string, db int) {
+	InitRedisByDb(addrs, 300, 30, db, 240)
+}
+
+func InitRedisBySize(addrs string, maxSize, maxIdle, timeout int) {
+	if RedisPool == nil {
+		RedisPool = map[string]*redigo.Pool{}
+	}
+	RedisPool = map[string]*redigo.Pool{}
+	addr := strings.Split(addrs, ",")
+	for _, v := range addr {
+		saddr := strings.Split(v, "=")
+		RedisPool[saddr[0]] = &redigo.Pool{MaxActive: maxSize, MaxIdle: maxIdle,
+			IdleTimeout: time.Duration(timeout) * time.Second, Dial: func() (redigo.Conn, error) {
+				c, err := redigo.Dial("tcp", saddr[1])
+				if err != nil {
+					return nil, err
+				}
+				return c, nil
+			}}
+	}
+}
+
+func InitRedisByDb(addrs string, maxSize, maxIdle, db, timeout int) {
+	if RedisPool == nil {
+		RedisPool = map[string]*redigo.Pool{}
+	}
+	addr := strings.Split(addrs, ",")
+	for _, v := range addr {
+		saddr := strings.Split(v, "=")
+		RedisPool[saddr[0]] = &redigo.Pool{MaxActive: maxSize, MaxIdle: maxIdle,
+			IdleTimeout: time.Duration(timeout) * time.Second, Dial: func() (redigo.Conn, error) {
+				c, err := redigo.Dial("tcp", saddr[1])
+				if err != nil {
+					return nil, err
+				}
+				_, err = c.Do("SELECT", db)
+				if err != nil {
+					c.Close()
+					return nil, err
+				}
+				return c, nil
+			}, Wait: true}
+	}
+}
+
+//分流redis`
+//并存入字符串缓存
+func PutKV(key string, obj interface{}) bool {
+	return Put("other", key, obj, -1)
+}
+func PutCKV(code, key string, obj interface{}) bool {
+	return Put(code, key, obj, -1)
+}
+func Put(code, key string, obj interface{}, timeout int) bool {
+	b := false
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	var err error
+	_obj, _err := json.Marshal(obj)
+	if _err != nil {
+		log.Println("redisutil-SET-序列化出错Error", _err)
+		return b
+	}
+	if timeout < 1 {
+		_, err = conn.Do("SET", key, _obj)
+	} else {
+		_, err = conn.Do("SET", key, _obj, "EX", timeout)
+	}
+	if nil != err {
+		log.Println("redisutil-SETError-put", err)
+	} else {
+		b = true
+	}
+	return b
+}
+
+func BulkPut(code string, timeout int, obj ...interface{}) bool {
+	b := false
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	var err error
+	for _, _tmp := range obj {
+		tmp, ok := _tmp.([]interface{})
+		if ok && len(tmp) == 2 {
+			key, kok := tmp[0].(string)
+			if kok && key != "" {
+				_obj, _err := json.Marshal(tmp[1])
+				if _err != nil {
+					log.Println("redisutil-SET-序列化出错Error", _err)
+					return b
+				}
+				if timeout < 1 {
+					_, err = conn.Do("SET", key, _obj)
+				} else {
+					_, err = conn.Do("SET", key, _obj, "EX", timeout)
+				}
+			}
+		}
+	}
+	if nil != err {
+		b = false
+		log.Println("redisutil-SETError-put", err)
+	} else {
+		b = b && true
+	}
+	return b
+}
+
+//直接存字节流
+func PutBytes(code, key string, data *[]byte, timeout int) (err error) {
+	defer catch()
+
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+
+	if timeout < 1 {
+		_, err = conn.Do("SET", key, *data)
+	} else {
+		_, err = conn.Do("SET", key, *data, "EX", timeout)
+	}
+	if nil != err {
+		log.Println("redisutil-SETError", err)
+	}
+	return
+}
+
+//设置超时时间,单位秒
+func SetExpire(code, key string, expire int) error {
+	defer catch()
+
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	_, err := conn.Do("expire", key, expire)
+	return err
+}
+
+//判断一个key是否存在
+func Exists(code, key string) (bool, error) {
+	defer catch()
+
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	repl, err := conn.Do("exists", key)
+	ret, _ := redigo.Int(repl, err)
+	return ret == 1, err
+}
+
+//获取string
+func GetStr(code, key string) string {
+	res := Get(code, key)
+	str, _ := res.(string)
+	return str
+}
+
+//获取int
+func GetInt(code, key string) int {
+	result, _ := GetNewInt(code, key)
+	return result
+}
+
+func GetNewInt(code, key string) (int, error) {
+	var res interface{}
+	err := GetNewInterface(code, key, &res)
+	var result int
+	if str, ok := res.(float64); ok {
+		result = int(str)
+	}
+	return result, err
+}
+
+//取得字符串,支持变参，2个 (key,code)，返回后自己断言
+func Get(code, key string) (result interface{}) {
+	GetInterface(code, key, &result)
+	return
+}
+
+func GetInterface(code, key string, result interface{}) {
+	GetNewInterface(code, key, result)
+}
+
+func GetNewInterface(code, key string, result interface{}) error {
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	ret, err := conn.Do("GET", key)
+	if nil != err {
+		log.Println("redisutil-GetError", err)
+	} else {
+		var ok bool
+		var res []byte
+		if res, ok = ret.([]byte); ok {
+			err = json.Unmarshal(res, result)
+			if err != nil {
+				log.Println("Get ERROR:", err.Error())
+			}
+		}
+	}
+	return err
+}
+
+//直接返回字节流
+func GetBytes(code, key string) (ret *[]byte, err error) {
+	defer catch()
+
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	var r interface{}
+	r, err = conn.Do("GET", key)
+	if err != nil {
+		log.Println("redisutil-GetBytesError", err)
+	} else {
+		if tmp, ok := r.([]byte); ok {
+			ret = &tmp
+		} else {
+			err = errors.New("redis返回数据格式不对")
+		}
+	}
+	return
+}
+func GetNewBytes(code, key string) (ret *[]byte, err error) {
+	defer catch()
+	redisPool := RedisPool[code]
+	if redisPool == nil {
+		err = errors.New("redis code " + code + " is nil")
+		log.Println("redisutil-GetNewBytesError", err)
+		return
+	}
+	conn := redisPool.Get()
+	defer conn.Close()
+	var r interface{}
+	r, err = conn.Do("GET", key)
+	if err != nil {
+		log.Println("redisutil-GetNewBytesError", err)
+	} else if r != nil {
+		if tmp, ok := r.([]byte); ok {
+			ret = &tmp
+		}
+	}
+	return
+}
+
+//删所有key
+func FlushDB(code string) bool {
+	b := false
+	defer catch()
+
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+
+	var err error
+	_, err = conn.Do("FLUSHDB")
+	if nil != err {
+		log.Println("redisutil-FLUSHDBError", err)
+	} else {
+		b = true
+	}
+	return b
+}
+
+//支持删除多个key
+func Del(code string, key ...interface{}) bool {
+	defer catch()
+	b := false
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+
+	var err error
+	_, err = conn.Do("DEL", key...)
+	if nil != err {
+		log.Println("redisutil-DELError", err)
+	} else {
+		b = true
+	}
+	return b
+}
+
+/**
+func DelKey(key ...interface{}) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Println("[E]", r)
+			for skip := 1; ; skip++ {
+				_, file, line, ok := runtime.Caller(skip)
+				if !ok {
+					break
+				}
+				go log.Printf("%v,%v\n", file, line)
+			}
+		}
+	}()
+	for i := 0; i < len(RedisPool); i++ {
+		delByNum(i, key...)
+	}
+
+}
+**/
+
+/**
+func delByNum(n int, key ...interface{}) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Println("[E]", r)
+			for skip := 1; ; skip++ {
+				_, file, line, ok := runtime.Caller(skip)
+				if !ok {
+					break
+				}
+				go log.Printf("%v,%v\n", file, line)
+			}
+		}
+	}()
+	i := 0
+	for _, v := range RedisPool {
+		if i == n {
+			conn := v.Get()
+			defer conn.Close()
+			conn.Do("DEL", key...)
+			break
+		}
+		i++
+	}
+}
+**/
+
+//根据代码和前辍key删除多个
+func DelByCodePattern(code, key string) {
+	defer catch()
+
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	ret, err := conn.Do("KEYS", key)
+	var result []interface{}
+	if nil != err {
+		log.Println("redisutil-GetError", err)
+	} else {
+		result = ret.([]interface{})
+		for k := 0; k < len(result); k++ {
+			conn.Do("DEL", string(result[k].([]uint8)))
+		}
+	}
+}
+
+/**
+func DelByPattern(key string) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Println("[E]", r)
+			for skip := 1; ; skip++ {
+				_, file, line, ok := runtime.Caller(skip)
+				if !ok {
+					break
+				}
+				go log.Printf("%v,%v\n", file, line)
+			}
+		}
+	}()
+	i := 0
+	for _, v := range RedisPool {
+		conn := v.Get()
+		defer conn.Close()
+		ret, err := conn.Do("KEYS", key)
+		var result []interface{}
+		if nil != err {
+			log.Println("redisutil-GetError", err)
+		} else {
+			result = ret.([]interface{})
+			for k := 0; k < len(result); k++ {
+				delByNum(i, string(result[k].([]uint8)))
+			}
+		}
+		i++
+	}
+
+}
+**/
+//自增计数器
+func Incr(code, key string) int64 {
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	ret, err := conn.Do("INCR", key)
+	if nil != err {
+		log.Println("redisutil-INCR-Error", err)
+	} else {
+		if res, ok := ret.(int64); ok {
+			return res
+		} else {
+			return 0
+		}
+	}
+	return 0
+}
+
+//自减
+func Decrby(code, key string, val int) int64 {
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	ret, err := conn.Do("DECRBY", key, val)
+	if nil != err {
+		log.Println("redisutil-DECR-Error", err)
+	} else {
+		if res, ok := ret.(int64); ok {
+			return res
+		} else {
+			return 0
+		}
+	}
+	return 0
+}
+
+//根据正则去取
+func GetKeysByPattern(code, key string) []interface{} {
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	ret, err := conn.Do("KEYS", key)
+	if nil != err {
+		log.Println("redisutil-GetKeysError", err)
+		return nil
+	} else {
+		res, _ := ret.([]interface{})
+		return res
+	}
+}
+
+//批量取多个key
+func Mget(code string, key []string) []interface{} {
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	interfaceKeys := make([]interface{}, len(key))
+	for n, k := range key {
+		interfaceKeys[n] = k
+	}
+	ret, err := conn.Do("MGET", interfaceKeys...)
+	if nil != err {
+		log.Println("redisutil-MgetError", err)
+		return nil
+	} else {
+		res, _ := ret.([]interface{})
+		return res
+	}
+}
+
+//取出并删除Key
+func Pop(code string, key string) (result interface{}) {
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	ret, err := conn.Do("GET", key)
+	if nil != err {
+		log.Println("redisutil-PopError", err)
+	} else {
+		var ok bool
+		var res []byte
+		if res, ok = ret.([]byte); ok {
+			err = json.Unmarshal(res, &result)
+			if err != nil {
+				log.Println("Poperr", err)
+			}
+		}
+		conn.Do("DEL", key)
+	}
+	return
+}
+
+//list操作
+func LPOP(code, list string) (result interface{}) {
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	ret, err := conn.Do("LPOP", list)
+	if nil != err {
+		log.Println("redisutil-LPopError", err)
+	} else {
+		if res, ok := ret.([]byte); ok {
+			err = json.Unmarshal(res, &result)
+			log.Println(err)
+		}
+	}
+	return
+}
+
+func RPUSH(code, list string, val interface{}) bool {
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	_obj, _ := json.Marshal(val)
+	_, err := conn.Do("RPUSH", list, _obj)
+	if nil != err {
+		log.Println("redisutil-RPUSHError", err)
+		return false
+	}
+	return true
+}
+
+func LLEN(code, list string) int64 {
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	ret, err := conn.Do("LLEN", list)
+	if nil != err {
+		log.Println("redisutil-LLENError", err)
+		return 0
+	}
+	if res, ok := ret.(int64); ok {
+		return res
+	} else {
+		return 0
+	}
+}
+
+func catch() {
+	if r := recover(); r != nil {
+		log.Println(r)
+		for skip := 0; ; skip++ {
+			_, file, line, ok := runtime.Caller(skip)
+			if !ok {
+				break
+			}
+			go log.Printf("%v,%v\n", file, line)
+		}
+	}
+}

common_utils/redis/redisutil_test.go

@@ -0,0 +1,86 @@
+package redis
+
+import (
+	"fmt"
+	"log"
+	"testing"
+	"time"
+)
+
+func Test_redis(t *testing.T) {
+	InitRedis("192.168.3.142:6379,192.168.3.142:7379")
+
+	log.Println(PutKV("key-BBB", "v-oooo"))
+	log.Println(GetStr("", "key-BBB"))
+
+	log.Println(PutCKV("code-K", "key-BBB2", 123))
+	log.Println(GetInt("code-K", "key-BBB2"))
+
+	log.Println(Put("codeA", "BBB", "oooo", 0))
+	//log.Println(Del("codeA", "AAA"))
+	log.Println(GetStr("codeA", "AAA"))
+
+	log.Println(Put("code", "Key", &map[string]string{"name": "张三"}, 0))
+	log.Println(Get("code", "Key"))
+}
+
+func Test_redisMorePut(t *testing.T) {
+	//by := []byte("dddddddddddd555d")
+
+	//InitRedis("enterprise=192.168.3.14:1379,service=192.168.3.14:2379,other=192.168.3.14:3379")
+	InitRedis("sso=192.168.3.14:1379")
+	//PutBytes("enterprise", "A", &by, 0)
+	//Put("sso", "new_1070776706", "obEpLuMMDUYUM-zlzCbQ0MbuQOzc", 180)
+	Put("sso", "1903540616", "oJULtwzXo6EFV1Ah-XeyRBimXGM8", 180)
+	//log.Println(Get("enterprise", "A"))
+	//GetMoreKey("ent*")
+}
+
+func Test_other(t *testing.T) {
+	//s := `[{"S_remark":"kokopkopkop","s_pic":"/upload/2015/10/16/2015101611194101018681.jpg","o_extend":{"k1":"ioio","k7":"000"}}]`
+	//mo := []map[string]interface{}{}
+	//json.Unmarshal([]byte(s), &mo)
+	//log.Println(mo)
+
+	pools := make(chan bool, 1)
+	pool := make(chan bool, 2)
+	for i := 0; i < 1000; i++ {
+		go func(i int) {
+			pool <- true
+			defer func() {
+				<-pool
+			}()
+			log.Println("--", i)
+			time.Sleep(5 * time.Millisecond)
+
+		}(i)
+	}
+	<-pools
+}
+
+func TestIncr(t *testing.T) {
+	InitRedis("other=192.168.3.14:1379")
+	//log.Println(GetBytes("sso", "p_share_3200000006"))
+	Put("other", "test", "12212212", 50000)
+}
+
+func Test_Pop(t *testing.T) {
+	InitRedis("sso=192.168.3.207:1601")
+	m := "aaaa"
+	Put("sso", "aaa", m, 1000)
+	log.Println(Pop("sso", "aaa"))
+}
+
+func Test_MGET(t *testing.T) {
+	InitRedis("sso=192.168.3.207:3378")
+	InitRedis("sso=192.168.3.207:1711")
+	n := 0
+	for {
+		n++
+		Put("sso", fmt.Sprintf("%d_1", n), n, 3600)
+		time.Sleep(8 * time.Second)
+
+	}
+	res := Mget("sso", []string{"pn_金业街（西自由街-虎泉路）道路建设工程"})
+	log.Println(res)
+}

common_utils/simple_encrypt.go

@@ -0,0 +1,135 @@
+package util
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/hex"
+)
+
+/**
+加解密
+	数据结构
+	密文+sha32校验
+*/
+//
+type SimpleEncrypt struct {
+	Key string //加解密用到的key(加密key索引)+
+}
+
+//计算检验和
+func (s *SimpleEncrypt) calaCheckCode(src []byte) []byte {
+	check := 0
+	for i := 0; i < len(src); i++ {
+		check += int(src[i])
+	}
+	return []byte{byte((check >> 8) & 0xff), byte(check & 0xff)}
+}
+
+//验证数据有效性
+func (s *SimpleEncrypt) verify(src []byte) bool {
+	v := s.calaCheckCode(src[:len(src)-2])
+	return bytes.Equal(v, src[len(src)-2:])
+}
+
+//加密String
+func (s *SimpleEncrypt) EncodeString(str string) string {
+	data := []byte(str)
+	s.doEncode(data)
+	return base64.StdEncoding.EncodeToString(data)
+}
+
+//加密String,ByCheck
+func (s *SimpleEncrypt) EncodeStringByCheck(str string) string {
+	data := []byte(str)
+	s.doEncode(data)
+	v := s.calaCheckCode(data)
+	data = append(data, v...)
+	return base64.StdEncoding.EncodeToString(data)
+}
+
+//
+func (s *SimpleEncrypt) Encode2Hex(str string) string {
+	data := []byte(str)
+	s.doEncode(data)
+	return hex.EncodeToString(data)
+}
+func (s *SimpleEncrypt) Encode2HexByCheck(str string) string {
+	data := []byte(str)
+	s.doEncode(data)
+	v := s.calaCheckCode(data)
+	data = append(data, v...)
+	return hex.EncodeToString(data)
+}
+
+//解密String
+func (s *SimpleEncrypt) DecodeString(str string) string {
+	data, _ := base64.StdEncoding.DecodeString(str)
+	s.doEncode(data)
+	return string(data)
+}
+
+//解密String,ByCheck
+func (s *SimpleEncrypt) DecodeStringByCheck(str string) string {
+	data, _ := base64.StdEncoding.DecodeString(str)
+	if len(data) < 2 || !s.verify(data) {
+		return ""
+	}
+	data = data[:len(data)-2]
+	s.doEncode(data)
+	return string(data)
+}
+
+//
+func (s *SimpleEncrypt) Decode4Hex(str string) string {
+	data, _ := hex.DecodeString(str)
+	s.doEncode(data)
+	return string(data)
+}
+func (s *SimpleEncrypt) Decode4HexByCheck(str string) string {
+	data, _ := hex.DecodeString(str)
+	if len(data) < 2 || !s.verify(data) {
+		return ""
+	}
+	data = data[:len(data)-2]
+	s.doEncode(data)
+	return string(data)
+}
+
+//加密
+func (s *SimpleEncrypt) Encode(data []byte) {
+	s.doEncode(data)
+
+}
+
+func (s *SimpleEncrypt) EncodeByCheck(data []byte) {
+	s.doEncode(data)
+	v := s.calaCheckCode(data)
+	data = append(data, v...)
+}
+
+//解密
+func (s *SimpleEncrypt) Decode(data []byte) {
+	s.doEncode(data)
+}
+
+//解密
+func (s *SimpleEncrypt) DecodeByCheck(data []byte) {
+	if len(data) < 2 || !s.verify(data) {
+		data = []byte{}
+		return
+	}
+	s.doEncode(data)
+}
+
+func (s *SimpleEncrypt) doEncode(bs []byte) {
+	tmp := []byte(s.Key)
+THEFOR:
+	for i := 0; i < len(bs); {
+		for j := 0; j < len(tmp); j, i = j+1, i+1 {
+			if i >= len(bs) {
+				break THEFOR
+			}
+			bs[i] = bs[i] ^ tmp[j]
+		}
+	}
+}

common_utils/udp/protocol.go

@@ -0,0 +1,161 @@
+package udp
+
+//底层网络通信协议
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+)
+
+const (
+	EMPTY_ADDR = "00000000"
+)
+
+//读取
+type Reader struct {
+	conn   net.Conn
+	reader *bufio.Reader
+	buffer [4]byte
+}
+
+//数据包，实际上就是协议定义
+type Packet struct {
+	//头开始
+	Length     int32  //4bit 包长度
+	From       string //8bit
+	To         string //8bit
+	Msgid      string //8bit
+	Event      int32  //4bit
+	SentToType int32  //4bit
+	//头结束
+	Raw []byte //数据包，头+数据
+}
+
+//返回
+func (p *Packet) GetBusinessData() []byte {
+	return p.Raw[36:]
+}
+
+//
+type RawData []byte //流数据
+
+//
+func NewReader(c net.Conn) *Reader {
+	return &Reader{
+		conn:   c,
+		reader: bufio.NewReader(c),
+	}
+}
+
+//读取头部
+func (p *Reader) readHeader() (int32, error) {
+	buf := p.buffer[:4]
+	if _, err := io.ReadFull(p.reader, buf); err != nil {
+		return 0, err
+	}
+	size := int32(binary.BigEndian.Uint32(buf))
+	if size < 32 || size > 16384000 {
+		return 0, fmt.Errorf("Incorrect frame size (%d)", size)
+	}
+	return size, nil
+}
+
+//读取完整一帧数据，防止粘包
+func (p *Reader) readFrame(size int32) (*Packet, error) {
+	var buf []byte
+	if int(size) <= len(p.buffer) {
+		buf = p.buffer[0:size]
+	} else {
+		buf = make([]byte, size)
+	}
+	_, err := io.ReadFull(p.reader, buf)
+	raw := append(Int2Byte(size), buf...)
+	if err == nil {
+		//解析包
+		packet := &Packet{
+			Length:     size,
+			From:       string(buf[:8]),
+			To:         string(buf[8:16]),
+			Msgid:      string(buf[16:24]),
+			Event:      Byte2Int(buf[24:28]),
+			SentToType: Byte2Int(buf[28:32]),
+			//Data:       buf[32:],
+			Raw: raw,
+		}
+		return packet, nil
+	}
+	return nil, err
+}
+
+//读取数据写入队列
+func forwardMessage(c net.Conn, queue chan<- *Packet) {
+	defer c.Close()
+	logReader := NewReader(c)
+	for {
+		size, err := logReader.readHeader()
+		if err != nil {
+			break
+		}
+		packet, err := logReader.readFrame(size)
+		if err != nil {
+			break
+		}
+		queue <- packet
+	}
+}
+
+//从队列中读取数据，定期处理
+func processMsg(q <-chan *Packet, parseEvent func(*Packet)) {
+	for {
+		select {
+		case msg := <-q:
+			go parseEvent(msg)
+		}
+	}
+}
+
+//封包，输出直接可以写到流中的数据包
+func Enpacket(from, to, msgid string, event, sendtotype int, obj interface{}) []byte {
+	if len(from) != 8 {
+		from = EMPTY_ADDR
+	}
+	if len(to) != 8 {
+		to = EMPTY_ADDR
+	}
+	if len(msgid) != 8 {
+		msgid = EMPTY_ADDR
+	}
+	var ret []byte
+	var bs []byte
+	if v, ok := obj.([]byte); ok {
+		bs = v
+	} else if v, ok := obj.(string); ok {
+		bs = []byte(v)
+	} else {
+		bs, _ = json.Marshal(obj)
+	}
+	ret = append(Int2Byte(int32(32 + len(bs))))
+	ret = append(ret, []byte(from+to+msgid)...)
+	ret = append(ret, Int2Byte(int32(event))...)
+	ret = append(ret, Int2Byte(int32(sendtotype))...)
+	ret = append(ret, bs...)
+	return ret
+}
+
+//
+func Byte2Int(src []byte) int32 {
+	var ret int32
+	binary.Read(bytes.NewReader(src), binary.BigEndian, &ret)
+	return ret
+}
+
+//
+func Int2Byte(src int32) []byte {
+	buf := bytes.NewBuffer([]byte{})
+	binary.Write(buf, binary.BigEndian, src)
+	return buf.Bytes()
+}

common_utils/udp/udp.go

@@ -0,0 +1,78 @@
+package udp
+
+import (
+	"bytes"
+	"net"
+	"time"
+)
+
+//
+type UdpClient struct {
+	Local   string
+	Connect *net.UDPConn
+	BufSize int
+}
+
+//指令
+const (
+	OP_NOOP = byte(iota)
+	OP_JOIN
+	OP_BYE
+	OP_SAY_ONLINE
+	OP_TYPE_DATA
+	OP_GET_DOWNLOADERCODE     //上报下载器代码
+	OP_WILLCHANGEIP           //将要切换IP
+	OP_PUSH_DOWNLOADERCODES   //
+	OP_DELETE_DOWNLOADERCODES //需要删除
+	OP_SEND_EMAIL             //发送邮件
+)
+
+//
+func (c *UdpClient) Listen(fn func(byte, []byte, *net.UDPAddr)) {
+	listenAddr, _ := net.ResolveUDPAddr("udp4", c.Local)
+	c.Connect, _ = net.ListenUDP("udp4", listenAddr)
+	go c.readUdp(fn)
+}
+
+//写
+func (c *UdpClient) WriteUdp(data []byte /*写入数据*/, act byte /*动作*/, toaddr *net.UDPAddr /*目标端地址*/) error {
+	bs := make([]byte, 1)
+	bs[0] = act
+	sizebs := Int2Byte(int32(len(data)))
+	bs = append(bs, sizebs...)
+	bs = append(bs, data...)
+	if len(bs) > c.BufSize {
+		bs = bs[:c.BufSize]
+	} else if len(bs) < c.BufSize {
+		bs = append(bs, bytes.Repeat([]byte{0}, c.BufSize-len(bs))...)
+	}
+	for i := 0; i < 5; i++ { //尝试5次
+		_, err := c.Connect.WriteToUDP(bs, toaddr)
+		if err == nil {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+	return nil
+}
+
+//读
+func (c *UdpClient) readUdp(fn func(byte, []byte, *net.UDPAddr)) {
+	for {
+		head := make([]byte, c.BufSize)
+		n, ra, err := c.Connect.ReadFromUDP(head)
+		if err != nil || n == 0 { //取不到数据
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		size := int(Byte2Int(head[1:5]))
+		if size < 1 { //无数据，仅仅是控制指令
+			go fn(head[0], []byte{}, ra)
+		} else {
+			if size > c.BufSize-5 {
+				size = c.BufSize - 5
+			}
+			go fn(head[0], head[5:5+size], ra)
+		}
+	}
+}

common_utils/validator/converter.go

@@ -0,0 +1,64 @@
+package govalidator
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"strconv"
+)
+
+// ToString convert the input to a string.
+func ToString(obj interface{}) string {
+	res := fmt.Sprintf("%v", obj)
+	return res
+}
+
+// ToJSON convert the input to a valid JSON string
+func ToJSON(obj interface{}) (string, error) {
+	res, err := json.Marshal(obj)
+	if err != nil {
+		res = []byte("")
+	}
+	return string(res), err
+}
+
+// ToFloat convert the input string to a float, or 0.0 if the input is not a float.
+func ToFloat(str string) (float64, error) {
+	res, err := strconv.ParseFloat(str, 64)
+	if err != nil {
+		res = 0.0
+	}
+	return res, err
+}
+
+// ToInt convert the input string or any int type to an integer type 64, or 0 if the input is not an integer.
+func ToInt(value interface{}) (res int64, err error) {
+	val := reflect.ValueOf(value)
+
+	switch value.(type) {
+	case int, int8, int16, int32, int64:
+		res = val.Int()
+	case uint, uint8, uint16, uint32, uint64:
+		res = int64(val.Uint())
+	case string:
+		if IsInt(val.String()) {
+			res, err = strconv.ParseInt(val.String(), 0, 64)
+			if err != nil {
+				res = 0
+			}
+		} else {
+			err = fmt.Errorf("ToInt: invalid numeric format %g", value)
+			res = 0
+		}
+	default:
+		err = fmt.Errorf("ToInt: unknown interface type %T", value)
+		res = 0
+	}
+
+	return
+}
+
+// ToBoolean convert the input string to a boolean.
+func ToBoolean(str string) (bool, error) {
+	return strconv.ParseBool(str)
+}

common_utils/validator/error.go

@@ -0,0 +1,47 @@
+package govalidator
+
+import (
+	"sort"
+	"strings"
+)
+
+// Errors is an array of multiple errors and conforms to the error interface.
+type Errors []error
+
+// Errors returns itself.
+func (es Errors) Errors() []error {
+	return es
+}
+
+func (es Errors) Error() string {
+	var errs []string
+	for _, e := range es {
+		errs = append(errs, e.Error())
+	}
+	sort.Strings(errs)
+	return strings.Join(errs, ";")
+}
+
+// Error encapsulates a name, an error and whether there's a custom error message or not.
+type Error struct {
+	Name                     string
+	Err                      error
+	CustomErrorMessageExists bool
+
+	// Validator indicates the name of the validator that failed
+	Validator string
+	Path      []string
+}
+
+func (e Error) Error() string {
+	if e.CustomErrorMessageExists {
+		return e.Err.Error()
+	}
+
+	errName := e.Name
+	if len(e.Path) > 0 {
+		errName = strings.Join(append(e.Path, e.Name), ".")
+	}
+
+	return errName + ": " + e.Err.Error()
+}

common_utils/validator/numerics.go

@@ -0,0 +1,96 @@
+package govalidator
+
+import (
+	"math"
+)
+
+// Abs returns absolute value of number
+func Abs(value float64) float64 {
+	return math.Abs(value)
+}
+
+// Sign returns signum of number: 1 in case of value > 0, -1 in case of value < 0, 0 otherwise
+func Sign(value float64) float64 {
+	if value > 0 {
+		return 1
+	} else if value < 0 {
+		return -1
+	} else {
+		return 0
+	}
+}
+
+// IsNegative returns true if value < 0
+func IsNegative(value float64) bool {
+	return value < 0
+}
+
+// IsPositive returns true if value > 0
+func IsPositive(value float64) bool {
+	return value > 0
+}
+
+// IsNonNegative returns true if value >= 0
+func IsNonNegative(value float64) bool {
+	return value >= 0
+}
+
+// IsNonPositive returns true if value <= 0
+func IsNonPositive(value float64) bool {
+	return value <= 0
+}
+
+// InRangeInt returns true if value lies between left and right border
+func InRangeInt(value, left, right interface{}) bool {
+	value64, _ := ToInt(value)
+	left64, _ := ToInt(left)
+	right64, _ := ToInt(right)
+	if left64 > right64 {
+		left64, right64 = right64, left64
+	}
+	return value64 >= left64 && value64 <= right64
+}
+
+// InRangeFloat32 returns true if value lies between left and right border
+func InRangeFloat32(value, left, right float32) bool {
+	if left > right {
+		left, right = right, left
+	}
+	return value >= left && value <= right
+}
+
+// InRangeFloat64 returns true if value lies between left and right border
+func InRangeFloat64(value, left, right float64) bool {
+	if left > right {
+		left, right = right, left
+	}
+	return value >= left && value <= right
+}
+
+// InRange returns true if value lies between left and right border, generic type to handle int, float32, float64 and string.
+// All types must the same type.
+// False if value doesn't lie in range or if it incompatible or not comparable
+func InRange(value interface{}, left interface{}, right interface{}) bool {
+	switch value.(type) {
+	case int:
+		return InRangeInt(value.(int), left.(int), right.(int))
+	case float32:
+		return InRangeFloat32(value.(float32), left.(float32), right.(float32))
+	case float64:
+		return InRangeFloat64(value.(float64), left.(float64), right.(float64))
+	case string:
+		return value.(string) >= left.(string) && value.(string) <= right.(string)
+	default:
+		return false
+	}
+}
+
+// IsWhole returns true if value is whole number
+func IsWhole(value float64) bool {
+	return math.Remainder(value, 1) == 0
+}
+
+// IsNatural returns true if value is natural number (positive and whole)
+func IsNatural(value float64) bool {
+	return IsWhole(value) && IsPositive(value)
+}

common_utils/validator/patterns.go

@@ -0,0 +1,107 @@
+package govalidator
+
+import "regexp"
+
+// Basic regular expressions for validating strings
+const (
+	Email             string = "^(((([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+(\\.([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+)*)|((\\x22)((((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(([\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f]|\\x21|[\\x23-\\x5b]|[\\x5d-\\x7e]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(\\([\\x01-\\x09\\x0b\\x0c\\x0d-\\x7f]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}]))))*(((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(\\x22)))@((([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|\\.|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.)+(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.?$"
+	CreditCard        string = "^(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|(222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12}|6(?:011|5[0-9][0-9])[0-9]{12}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11}|(?:2131|1800|35\\d{3})\\d{11}|6[27][0-9]{14})$"
+	ISBN10            string = "^(?:[0-9]{9}X|[0-9]{10})$"
+	ISBN13            string = "^(?:[0-9]{13})$"
+	UUID3             string = "^[0-9a-f]{8}-[0-9a-f]{4}-3[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$"
+	UUID4             string = "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
+	UUID5             string = "^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
+	UUID              string = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
+	Alpha             string = "^[a-zA-Z]+$"
+	Alphanumeric      string = "^[a-zA-Z0-9]+$"
+	Numeric           string = "^[0-9]+$"
+	Int               string = "^(?:[-+]?(?:0|[1-9][0-9]*))$"
+	Float             string = "^(?:[-+]?(?:[0-9]+))?(?:\\.[0-9]*)?(?:[eE][\\+\\-]?(?:[0-9]+))?$"
+	Hexadecimal       string = "^[0-9a-fA-F]+$"
+	Hexcolor          string = "^#?([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$"
+	RGBcolor          string = "^rgb\\(\\s*(0|[1-9]\\d?|1\\d\\d?|2[0-4]\\d|25[0-5])\\s*,\\s*(0|[1-9]\\d?|1\\d\\d?|2[0-4]\\d|25[0-5])\\s*,\\s*(0|[1-9]\\d?|1\\d\\d?|2[0-4]\\d|25[0-5])\\s*\\)$"
+	ASCII             string = "^[\x00-\x7F]+$"
+	Multibyte         string = "[^\x00-\x7F]"
+	FullWidth         string = "[^\u0020-\u007E\uFF61-\uFF9F\uFFA0-\uFFDC\uFFE8-\uFFEE0-9a-zA-Z]"
+	HalfWidth         string = "[\u0020-\u007E\uFF61-\uFF9F\uFFA0-\uFFDC\uFFE8-\uFFEE0-9a-zA-Z]"
+	Base64            string = "^(?:[A-Za-z0-9+\\/]{4})*(?:[A-Za-z0-9+\\/]{2}==|[A-Za-z0-9+\\/]{3}=|[A-Za-z0-9+\\/]{4})$"
+	PrintableASCII    string = "^[\x20-\x7E]+$"
+	DataURI           string = "^data:.+\\/(.+);base64$"
+	MagnetURI         string = "^magnet:\\?xt=urn:[a-zA-Z0-9]+:[a-zA-Z0-9]{32,40}&dn=.+&tr=.+$"
+	Latitude          string = "^[-+]?([1-8]?\\d(\\.\\d+)?|90(\\.0+)?)$"
+	Longitude         string = "^[-+]?(180(\\.0+)?|((1[0-7]\\d)|([1-9]?\\d))(\\.\\d+)?)$"
+	DNSName           string = `^([a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62}){1}(\.[a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62})*[\._]?$`
+	IP                string = `(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))`
+	URLSchema         string = `((ftp|tcp|udp|wss?|https?):\/\/)`
+	URLUsername       string = `(\S+(:\S*)?@)`
+	URLPath           string = `((\/|\?|#)[^\s]*)`
+	URLPort           string = `(:(\d{1,5}))`
+	URLIP             string = `([1-9]\d?|1\d\d|2[01]\d|22[0-3]|24\d|25[0-5])(\.(\d{1,2}|1\d\d|2[0-4]\d|25[0-5])){2}(?:\.([0-9]\d?|1\d\d|2[0-4]\d|25[0-5]))`
+	URLSubdomain      string = `((www\.)|([a-zA-Z0-9]+([-_\.]?[a-zA-Z0-9])*[a-zA-Z0-9]\.[a-zA-Z0-9]+))`
+	URL                      = `^` + URLSchema + `?` + URLUsername + `?` + `((` + URLIP + `|(\[` + IP + `\])|(([a-zA-Z0-9]([a-zA-Z0-9-_]+)?[a-zA-Z0-9]([-\.][a-zA-Z0-9]+)*)|(` + URLSubdomain + `?))?(([a-zA-Z\x{00a1}-\x{ffff}0-9]+-?-?)*[a-zA-Z\x{00a1}-\x{ffff}0-9]+)(?:\.([a-zA-Z\x{00a1}-\x{ffff}]{1,}))?))\.?` + URLPort + `?` + URLPath + `?$`
+	SSN               string = `^\d{3}[- ]?\d{2}[- ]?\d{4}$`
+	WinPath           string = `^[a-zA-Z]:\\(?:[^\\/:*?"<>|\r\n]+\\)*[^\\/:*?"<>|\r\n]*$`
+	UnixPath          string = `^(/[^/\x00]*)+/?$`
+	Semver            string = "^v?(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(-(0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(\\.(0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\\+[0-9a-zA-Z-]+(\\.[0-9a-zA-Z-]+)*)?$"
+	tagName           string = "valid"
+	hasLowerCase      string = ".*[[:lower:]]"
+	hasUpperCase      string = ".*[[:upper:]]"
+	hasWhitespace     string = ".*[[:space:]]"
+	hasWhitespaceOnly string = "^[[:space:]]+$"
+	IMEI              string = "^[0-9a-f]{14}$|^\\d{15}$|^\\d{18}$"
+	IMSI              string = "^\\d{14,15}$"
+)
+
+// Used by IsFilePath func
+const (
+	// Unknown is unresolved OS type
+	Unknown = iota
+	// Win is Windows type
+	Win
+	// Unix is *nix OS types
+	Unix
+)
+
+var (
+	userRegexp          = regexp.MustCompile("^[a-zA-Z0-9!#$%&'*+/=?^_`{|}~.-]+$")
+	hostRegexp          = regexp.MustCompile("^[^\\s]+\\.[^\\s]+$")
+	userDotRegexp       = regexp.MustCompile("(^[.]{1})|([.]{1}$)|([.]{2,})")
+	rxEmail             = regexp.MustCompile(Email)
+	rxCreditCard        = regexp.MustCompile(CreditCard)
+	rxISBN10            = regexp.MustCompile(ISBN10)
+	rxISBN13            = regexp.MustCompile(ISBN13)
+	rxUUID3             = regexp.MustCompile(UUID3)
+	rxUUID4             = regexp.MustCompile(UUID4)
+	rxUUID5             = regexp.MustCompile(UUID5)
+	rxUUID              = regexp.MustCompile(UUID)
+	rxAlpha             = regexp.MustCompile(Alpha)
+	rxAlphanumeric      = regexp.MustCompile(Alphanumeric)
+	rxNumeric           = regexp.MustCompile(Numeric)
+	rxInt               = regexp.MustCompile(Int)
+	rxFloat             = regexp.MustCompile(Float)
+	rxHexadecimal       = regexp.MustCompile(Hexadecimal)
+	rxHexcolor          = regexp.MustCompile(Hexcolor)
+	rxRGBcolor          = regexp.MustCompile(RGBcolor)
+	rxASCII             = regexp.MustCompile(ASCII)
+	rxPrintableASCII    = regexp.MustCompile(PrintableASCII)
+	rxMultibyte         = regexp.MustCompile(Multibyte)
+	rxFullWidth         = regexp.MustCompile(FullWidth)
+	rxHalfWidth         = regexp.MustCompile(HalfWidth)
+	rxBase64            = regexp.MustCompile(Base64)
+	rxDataURI           = regexp.MustCompile(DataURI)
+	rxMagnetURI         = regexp.MustCompile(MagnetURI)
+	rxLatitude          = regexp.MustCompile(Latitude)
+	rxLongitude         = regexp.MustCompile(Longitude)
+	rxDNSName           = regexp.MustCompile(DNSName)
+	rxURL               = regexp.MustCompile(URL)
+	rxSSN               = regexp.MustCompile(SSN)
+	rxWinPath           = regexp.MustCompile(WinPath)
+	rxUnixPath          = regexp.MustCompile(UnixPath)
+	rxSemver            = regexp.MustCompile(Semver)
+	rxHasLowerCase      = regexp.MustCompile(hasLowerCase)
+	rxHasUpperCase      = regexp.MustCompile(hasUpperCase)
+	rxHasWhitespace     = regexp.MustCompile(hasWhitespace)
+	rxHasWhitespaceOnly = regexp.MustCompile(hasWhitespaceOnly)
+	rxIMEI              = regexp.MustCompile(IMEI)
+	rxIMSI              = regexp.MustCompile(IMSI)
+)

common_utils/validator/types.go

@@ -0,0 +1,655 @@
+package govalidator
+
+import (
+	"reflect"
+	"regexp"
+	"sort"
+	"sync"
+)
+
+// Validator is a wrapper for a validator function that returns bool and accepts string.
+type Validator func(str string) bool
+
+// CustomTypeValidator is a wrapper for validator functions that returns bool and accepts any type.
+// The second parameter should be the context (in the case of validating a struct: the whole object being validated).
+type CustomTypeValidator func(i interface{}, o interface{}) bool
+
+// ParamValidator is a wrapper for validator functions that accept additional parameters.
+type ParamValidator func(str string, params ...string) bool
+
+// InterfaceParamValidator is a wrapper for functions that accept variants parameters for an interface value
+type InterfaceParamValidator func(in interface{}, params ...string) bool
+type tagOptionsMap map[string]tagOption
+
+func (t tagOptionsMap) orderedKeys() []string {
+	var keys []string
+	for k := range t {
+		keys = append(keys, k)
+	}
+
+	sort.Slice(keys, func(a, b int) bool {
+		return t[keys[a]].order < t[keys[b]].order
+	})
+
+	return keys
+}
+
+type tagOption struct {
+	name               string
+	customErrorMessage string
+	order              int
+}
+
+// UnsupportedTypeError is a wrapper for reflect.Type
+type UnsupportedTypeError struct {
+	Type reflect.Type
+}
+
+// stringValues is a slice of reflect.Value holding *reflect.StringValue.
+// It implements the methods to sort by string.
+type stringValues []reflect.Value
+
+// InterfaceParamTagMap is a map of functions accept variants parameters for an interface value
+var InterfaceParamTagMap = map[string]InterfaceParamValidator{
+	"type": IsType,
+}
+
+// InterfaceParamTagRegexMap maps interface param tags to their respective regexes.
+var InterfaceParamTagRegexMap = map[string]*regexp.Regexp{
+	"type": regexp.MustCompile(`^type\((.*)\)$`),
+}
+
+// ParamTagMap is a map of functions accept variants parameters
+var ParamTagMap = map[string]ParamValidator{
+	"length":          ByteLength,
+	"range":           Range,
+	"runelength":      RuneLength,
+	"stringlength":    StringLength,
+	"matches":         StringMatches,
+	"in":              IsInRaw,
+	"rsapub":          IsRsaPub,
+	"minstringlength": MinStringLength,
+	"maxstringlength": MaxStringLength,
+}
+
+// ParamTagRegexMap maps param tags to their respective regexes.
+var ParamTagRegexMap = map[string]*regexp.Regexp{
+	"range":           regexp.MustCompile("^range\\((\\d+)\\|(\\d+)\\)$"),
+	"length":          regexp.MustCompile("^length\\((\\d+)\\|(\\d+)\\)$"),
+	"runelength":      regexp.MustCompile("^runelength\\((\\d+)\\|(\\d+)\\)$"),
+	"stringlength":    regexp.MustCompile("^stringlength\\((\\d+)\\|(\\d+)\\)$"),
+	"in":              regexp.MustCompile(`^in\((.*)\)`),
+	"matches":         regexp.MustCompile(`^matches\((.+)\)$`),
+	"rsapub":          regexp.MustCompile("^rsapub\\((\\d+)\\)$"),
+	"minstringlength": regexp.MustCompile("^minstringlength\\((\\d+)\\)$"),
+	"maxstringlength": regexp.MustCompile("^maxstringlength\\((\\d+)\\)$"),
+}
+
+type customTypeTagMap struct {
+	validators map[string]CustomTypeValidator
+
+	sync.RWMutex
+}
+
+func (tm *customTypeTagMap) Get(name string) (CustomTypeValidator, bool) {
+	tm.RLock()
+	defer tm.RUnlock()
+	v, ok := tm.validators[name]
+	return v, ok
+}
+
+func (tm *customTypeTagMap) Set(name string, ctv CustomTypeValidator) {
+	tm.Lock()
+	defer tm.Unlock()
+	tm.validators[name] = ctv
+}
+
+// CustomTypeTagMap is a map of functions that can be used as tags for ValidateStruct function.
+// Use this to validate compound or custom types that need to be handled as a whole, e.g.
+// `type UUID [16]byte` (this would be handled as an array of bytes).
+var CustomTypeTagMap = &customTypeTagMap{validators: make(map[string]CustomTypeValidator)}
+
+// TagMap is a map of functions, that can be used as tags for ValidateStruct function.
+var TagMap = map[string]Validator{
+	"email":              IsEmail,
+	"url":                IsURL,
+	"dialstring":         IsDialString,
+	"requrl":             IsRequestURL,
+	"requri":             IsRequestURI,
+	"alpha":              IsAlpha,
+	"utfletter":          IsUTFLetter,
+	"alphanum":           IsAlphanumeric,
+	"utfletternum":       IsUTFLetterNumeric,
+	"numeric":            IsNumeric,
+	"utfnumeric":         IsUTFNumeric,
+	"utfdigit":           IsUTFDigit,
+	"hexadecimal":        IsHexadecimal,
+	"hexcolor":           IsHexcolor,
+	"rgbcolor":           IsRGBcolor,
+	"lowercase":          IsLowerCase,
+	"uppercase":          IsUpperCase,
+	"int":                IsInt,
+	"float":              IsFloat,
+	"null":               IsNull,
+	"notnull":            IsNotNull,
+	"uuid":               IsUUID,
+	"uuidv3":             IsUUIDv3,
+	"uuidv4":             IsUUIDv4,
+	"uuidv5":             IsUUIDv5,
+	"creditcard":         IsCreditCard,
+	"isbn10":             IsISBN10,
+	"isbn13":             IsISBN13,
+	"json":               IsJSON,
+	"multibyte":          IsMultibyte,
+	"ascii":              IsASCII,
+	"printableascii":     IsPrintableASCII,
+	"fullwidth":          IsFullWidth,
+	"halfwidth":          IsHalfWidth,
+	"variablewidth":      IsVariableWidth,
+	"base64":             IsBase64,
+	"datauri":            IsDataURI,
+	"ip":                 IsIP,
+	"port":               IsPort,
+	"ipv4":               IsIPv4,
+	"ipv6":               IsIPv6,
+	"dns":                IsDNSName,
+	"host":               IsHost,
+	"mac":                IsMAC,
+	"latitude":           IsLatitude,
+	"longitude":          IsLongitude,
+	"ssn":                IsSSN,
+	"semver":             IsSemver,
+	"rfc3339":            IsRFC3339,
+	"rfc3339WithoutZone": IsRFC3339WithoutZone,
+	"ISO3166Alpha2":      IsISO3166Alpha2,
+	"ISO3166Alpha3":      IsISO3166Alpha3,
+	"ISO4217":            IsISO4217,
+	"IMEI":               IsIMEI,
+}
+
+// ISO3166Entry stores country codes
+type ISO3166Entry struct {
+	EnglishShortName string
+	FrenchShortName  string
+	Alpha2Code       string
+	Alpha3Code       string
+	Numeric          string
+}
+
+//ISO3166List based on https://www.iso.org/obp/ui/#search/code/ Code Type "Officially Assigned Codes"
+var ISO3166List = []ISO3166Entry{
+	{"Afghanistan", "Afghanistan (l')", "AF", "AFG", "004"},
+	{"Albania", "Albanie (l')", "AL", "ALB", "008"},
+	{"Antarctica", "Antarctique (l')", "AQ", "ATA", "010"},
+	{"Algeria", "Algérie (l')", "DZ", "DZA", "012"},
+	{"American Samoa", "Samoa américaines (les)", "AS", "ASM", "016"},
+	{"Andorra", "Andorre (l')", "AD", "AND", "020"},
+	{"Angola", "Angola (l')", "AO", "AGO", "024"},
+	{"Antigua and Barbuda", "Antigua-et-Barbuda", "AG", "ATG", "028"},
+	{"Azerbaijan", "Azerbaïdjan (l')", "AZ", "AZE", "031"},
+	{"Argentina", "Argentine (l')", "AR", "ARG", "032"},
+	{"Australia", "Australie (l')", "AU", "AUS", "036"},
+	{"Austria", "Autriche (l')", "AT", "AUT", "040"},
+	{"Bahamas (the)", "Bahamas (les)", "BS", "BHS", "044"},
+	{"Bahrain", "Bahreïn", "BH", "BHR", "048"},
+	{"Bangladesh", "Bangladesh (le)", "BD", "BGD", "050"},
+	{"Armenia", "Arménie (l')", "AM", "ARM", "051"},
+	{"Barbados", "Barbade (la)", "BB", "BRB", "052"},
+	{"Belgium", "Belgique (la)", "BE", "BEL", "056"},
+	{"Bermuda", "Bermudes (les)", "BM", "BMU", "060"},
+	{"Bhutan", "Bhoutan (le)", "BT", "BTN", "064"},
+	{"Bolivia (Plurinational State of)", "Bolivie (État plurinational de)", "BO", "BOL", "068"},
+	{"Bosnia and Herzegovina", "Bosnie-Herzégovine (la)", "BA", "BIH", "070"},
+	{"Botswana", "Botswana (le)", "BW", "BWA", "072"},
+	{"Bouvet Island", "Bouvet (l'Île)", "BV", "BVT", "074"},
+	{"Brazil", "Brésil (le)", "BR", "BRA", "076"},
+	{"Belize", "Belize (le)", "BZ", "BLZ", "084"},
+	{"British Indian Ocean Territory (the)", "Indien (le Territoire britannique de l'océan)", "IO", "IOT", "086"},
+	{"Solomon Islands", "Salomon (Îles)", "SB", "SLB", "090"},
+	{"Virgin Islands (British)", "Vierges britanniques (les Îles)", "VG", "VGB", "092"},
+	{"Brunei Darussalam", "Brunéi Darussalam (le)", "BN", "BRN", "096"},
+	{"Bulgaria", "Bulgarie (la)", "BG", "BGR", "100"},
+	{"Myanmar", "Myanmar (le)", "MM", "MMR", "104"},
+	{"Burundi", "Burundi (le)", "BI", "BDI", "108"},
+	{"Belarus", "Bélarus (le)", "BY", "BLR", "112"},
+	{"Cambodia", "Cambodge (le)", "KH", "KHM", "116"},
+	{"Cameroon", "Cameroun (le)", "CM", "CMR", "120"},
+	{"Canada", "Canada (le)", "CA", "CAN", "124"},
+	{"Cabo Verde", "Cabo Verde", "CV", "CPV", "132"},
+	{"Cayman Islands (the)", "Caïmans (les Îles)", "KY", "CYM", "136"},
+	{"Central African Republic (the)", "République centrafricaine (la)", "CF", "CAF", "140"},
+	{"Sri Lanka", "Sri Lanka", "LK", "LKA", "144"},
+	{"Chad", "Tchad (le)", "TD", "TCD", "148"},
+	{"Chile", "Chili (le)", "CL", "CHL", "152"},
+	{"China", "Chine (la)", "CN", "CHN", "156"},
+	{"Taiwan (Province of China)", "Taïwan (Province de Chine)", "TW", "TWN", "158"},
+	{"Christmas Island", "Christmas (l'Île)", "CX", "CXR", "162"},
+	{"Cocos (Keeling) Islands (the)", "Cocos (les Îles)/ Keeling (les Îles)", "CC", "CCK", "166"},
+	{"Colombia", "Colombie (la)", "CO", "COL", "170"},
+	{"Comoros (the)", "Comores (les)", "KM", "COM", "174"},
+	{"Mayotte", "Mayotte", "YT", "MYT", "175"},
+	{"Congo (the)", "Congo (le)", "CG", "COG", "178"},
+	{"Congo (the Democratic Republic of the)", "Congo (la République démocratique du)", "CD", "COD", "180"},
+	{"Cook Islands (the)", "Cook (les Îles)", "CK", "COK", "184"},
+	{"Costa Rica", "Costa Rica (le)", "CR", "CRI", "188"},
+	{"Croatia", "Croatie (la)", "HR", "HRV", "191"},
+	{"Cuba", "Cuba", "CU", "CUB", "192"},
+	{"Cyprus", "Chypre", "CY", "CYP", "196"},
+	{"Czech Republic (the)", "tchèque (la République)", "CZ", "CZE", "203"},
+	{"Benin", "Bénin (le)", "BJ", "BEN", "204"},
+	{"Denmark", "Danemark (le)", "DK", "DNK", "208"},
+	{"Dominica", "Dominique (la)", "DM", "DMA", "212"},
+	{"Dominican Republic (the)", "dominicaine (la République)", "DO", "DOM", "214"},
+	{"Ecuador", "Équateur (l')", "EC", "ECU", "218"},
+	{"El Salvador", "El Salvador", "SV", "SLV", "222"},
+	{"Equatorial Guinea", "Guinée équatoriale (la)", "GQ", "GNQ", "226"},
+	{"Ethiopia", "Éthiopie (l')", "ET", "ETH", "231"},
+	{"Eritrea", "Érythrée (l')", "ER", "ERI", "232"},
+	{"Estonia", "Estonie (l')", "EE", "EST", "233"},
+	{"Faroe Islands (the)", "Féroé (les Îles)", "FO", "FRO", "234"},
+	{"Falkland Islands (the) [Malvinas]", "Falkland (les Îles)/Malouines (les Îles)", "FK", "FLK", "238"},
+	{"South Georgia and the South Sandwich Islands", "Géorgie du Sud-et-les Îles Sandwich du Sud (la)", "GS", "SGS", "239"},
+	{"Fiji", "Fidji (les)", "FJ", "FJI", "242"},
+	{"Finland", "Finlande (la)", "FI", "FIN", "246"},
+	{"Åland Islands", "Åland(les Îles)", "AX", "ALA", "248"},
+	{"France", "France (la)", "FR", "FRA", "250"},
+	{"French Guiana", "Guyane française (la )", "GF", "GUF", "254"},
+	{"French Polynesia", "Polynésie française (la)", "PF", "PYF", "258"},
+	{"French Southern Territories (the)", "Terres australes françaises (les)", "TF", "ATF", "260"},
+	{"Djibouti", "Djibouti", "DJ", "DJI", "262"},
+	{"Gabon", "Gabon (le)", "GA", "GAB", "266"},
+	{"Georgia", "Géorgie (la)", "GE", "GEO", "268"},
+	{"Gambia (the)", "Gambie (la)", "GM", "GMB", "270"},
+	{"Palestine, State of", "Palestine, État de", "PS", "PSE", "275"},
+	{"Germany", "Allemagne (l')", "DE", "DEU", "276"},
+	{"Ghana", "Ghana (le)", "GH", "GHA", "288"},
+	{"Gibraltar", "Gibraltar", "GI", "GIB", "292"},
+	{"Kiribati", "Kiribati", "KI", "KIR", "296"},
+	{"Greece", "Grèce (la)", "GR", "GRC", "300"},
+	{"Greenland", "Groenland (le)", "GL", "GRL", "304"},
+	{"Grenada", "Grenade (la)", "GD", "GRD", "308"},
+	{"Guadeloupe", "Guadeloupe (la)", "GP", "GLP", "312"},
+	{"Guam", "Guam", "GU", "GUM", "316"},
+	{"Guatemala", "Guatemala (le)", "GT", "GTM", "320"},
+	{"Guinea", "Guinée (la)", "GN", "GIN", "324"},
+	{"Guyana", "Guyana (le)", "GY", "GUY", "328"},
+	{"Haiti", "Haïti", "HT", "HTI", "332"},
+	{"Heard Island and McDonald Islands", "Heard-et-Îles MacDonald (l'Île)", "HM", "HMD", "334"},
+	{"Holy See (the)", "Saint-Siège (le)", "VA", "VAT", "336"},
+	{"Honduras", "Honduras (le)", "HN", "HND", "340"},
+	{"Hong Kong", "Hong Kong", "HK", "HKG", "344"},
+	{"Hungary", "Hongrie (la)", "HU", "HUN", "348"},
+	{"Iceland", "Islande (l')", "IS", "ISL", "352"},
+	{"India", "Inde (l')", "IN", "IND", "356"},
+	{"Indonesia", "Indonésie (l')", "ID", "IDN", "360"},
+	{"Iran (Islamic Republic of)", "Iran (République Islamique d')", "IR", "IRN", "364"},
+	{"Iraq", "Iraq (l')", "IQ", "IRQ", "368"},
+	{"Ireland", "Irlande (l')", "IE", "IRL", "372"},
+	{"Israel", "Israël", "IL", "ISR", "376"},
+	{"Italy", "Italie (l')", "IT", "ITA", "380"},
+	{"Côte d'Ivoire", "Côte d'Ivoire (la)", "CI", "CIV", "384"},
+	{"Jamaica", "Jamaïque (la)", "JM", "JAM", "388"},
+	{"Japan", "Japon (le)", "JP", "JPN", "392"},
+	{"Kazakhstan", "Kazakhstan (le)", "KZ", "KAZ", "398"},
+	{"Jordan", "Jordanie (la)", "JO", "JOR", "400"},
+	{"Kenya", "Kenya (le)", "KE", "KEN", "404"},
+	{"Korea (the Democratic People's Republic of)", "Corée (la République populaire démocratique de)", "KP", "PRK", "408"},
+	{"Korea (the Republic of)", "Corée (la République de)", "KR", "KOR", "410"},
+	{"Kuwait", "Koweït (le)", "KW", "KWT", "414"},
+	{"Kyrgyzstan", "Kirghizistan (le)", "KG", "KGZ", "417"},
+	{"Lao People's Democratic Republic (the)", "Lao, République démocratique populaire", "LA", "LAO", "418"},
+	{"Lebanon", "Liban (le)", "LB", "LBN", "422"},
+	{"Lesotho", "Lesotho (le)", "LS", "LSO", "426"},
+	{"Latvia", "Lettonie (la)", "LV", "LVA", "428"},
+	{"Liberia", "Libéria (le)", "LR", "LBR", "430"},
+	{"Libya", "Libye (la)", "LY", "LBY", "434"},
+	{"Liechtenstein", "Liechtenstein (le)", "LI", "LIE", "438"},
+	{"Lithuania", "Lituanie (la)", "LT", "LTU", "440"},
+	{"Luxembourg", "Luxembourg (le)", "LU", "LUX", "442"},
+	{"Macao", "Macao", "MO", "MAC", "446"},
+	{"Madagascar", "Madagascar", "MG", "MDG", "450"},
+	{"Malawi", "Malawi (le)", "MW", "MWI", "454"},
+	{"Malaysia", "Malaisie (la)", "MY", "MYS", "458"},
+	{"Maldives", "Maldives (les)", "MV", "MDV", "462"},
+	{"Mali", "Mali (le)", "ML", "MLI", "466"},
+	{"Malta", "Malte", "MT", "MLT", "470"},
+	{"Martinique", "Martinique (la)", "MQ", "MTQ", "474"},
+	{"Mauritania", "Mauritanie (la)", "MR", "MRT", "478"},
+	{"Mauritius", "Maurice", "MU", "MUS", "480"},
+	{"Mexico", "Mexique (le)", "MX", "MEX", "484"},
+	{"Monaco", "Monaco", "MC", "MCO", "492"},
+	{"Mongolia", "Mongolie (la)", "MN", "MNG", "496"},
+	{"Moldova (the Republic of)", "Moldova , République de", "MD", "MDA", "498"},
+	{"Montenegro", "Monténégro (le)", "ME", "MNE", "499"},
+	{"Montserrat", "Montserrat", "MS", "MSR", "500"},
+	{"Morocco", "Maroc (le)", "MA", "MAR", "504"},
+	{"Mozambique", "Mozambique (le)", "MZ", "MOZ", "508"},
+	{"Oman", "Oman", "OM", "OMN", "512"},
+	{"Namibia", "Namibie (la)", "NA", "NAM", "516"},
+	{"Nauru", "Nauru", "NR", "NRU", "520"},
+	{"Nepal", "Népal (le)", "NP", "NPL", "524"},
+	{"Netherlands (the)", "Pays-Bas (les)", "NL", "NLD", "528"},
+	{"Curaçao", "Curaçao", "CW", "CUW", "531"},
+	{"Aruba", "Aruba", "AW", "ABW", "533"},
+	{"Sint Maarten (Dutch part)", "Saint-Martin (partie néerlandaise)", "SX", "SXM", "534"},
+	{"Bonaire, Sint Eustatius and Saba", "Bonaire, Saint-Eustache et Saba", "BQ", "BES", "535"},
+	{"New Caledonia", "Nouvelle-Calédonie (la)", "NC", "NCL", "540"},
+	{"Vanuatu", "Vanuatu (le)", "VU", "VUT", "548"},
+	{"New Zealand", "Nouvelle-Zélande (la)", "NZ", "NZL", "554"},
+	{"Nicaragua", "Nicaragua (le)", "NI", "NIC", "558"},
+	{"Niger (the)", "Niger (le)", "NE", "NER", "562"},
+	{"Nigeria", "Nigéria (le)", "NG", "NGA", "566"},
+	{"Niue", "Niue", "NU", "NIU", "570"},
+	{"Norfolk Island", "Norfolk (l'Île)", "NF", "NFK", "574"},
+	{"Norway", "Norvège (la)", "NO", "NOR", "578"},
+	{"Northern Mariana Islands (the)", "Mariannes du Nord (les Îles)", "MP", "MNP", "580"},
+	{"United States Minor Outlying Islands (the)", "Îles mineures éloignées des États-Unis (les)", "UM", "UMI", "581"},
+	{"Micronesia (Federated States of)", "Micronésie (États fédérés de)", "FM", "FSM", "583"},
+	{"Marshall Islands (the)", "Marshall (Îles)", "MH", "MHL", "584"},
+	{"Palau", "Palaos (les)", "PW", "PLW", "585"},
+	{"Pakistan", "Pakistan (le)", "PK", "PAK", "586"},
+	{"Panama", "Panama (le)", "PA", "PAN", "591"},
+	{"Papua New Guinea", "Papouasie-Nouvelle-Guinée (la)", "PG", "PNG", "598"},
+	{"Paraguay", "Paraguay (le)", "PY", "PRY", "600"},
+	{"Peru", "Pérou (le)", "PE", "PER", "604"},
+	{"Philippines (the)", "Philippines (les)", "PH", "PHL", "608"},
+	{"Pitcairn", "Pitcairn", "PN", "PCN", "612"},
+	{"Poland", "Pologne (la)", "PL", "POL", "616"},
+	{"Portugal", "Portugal (le)", "PT", "PRT", "620"},
+	{"Guinea-Bissau", "Guinée-Bissau (la)", "GW", "GNB", "624"},
+	{"Timor-Leste", "Timor-Leste (le)", "TL", "TLS", "626"},
+	{"Puerto Rico", "Porto Rico", "PR", "PRI", "630"},
+	{"Qatar", "Qatar (le)", "QA", "QAT", "634"},
+	{"Réunion", "Réunion (La)", "RE", "REU", "638"},
+	{"Romania", "Roumanie (la)", "RO", "ROU", "642"},
+	{"Russian Federation (the)", "Russie (la Fédération de)", "RU", "RUS", "643"},
+	{"Rwanda", "Rwanda (le)", "RW", "RWA", "646"},
+	{"Saint Barthélemy", "Saint-Barthélemy", "BL", "BLM", "652"},
+	{"Saint Helena, Ascension and Tristan da Cunha", "Sainte-Hélène, Ascension et Tristan da Cunha", "SH", "SHN", "654"},
+	{"Saint Kitts and Nevis", "Saint-Kitts-et-Nevis", "KN", "KNA", "659"},
+	{"Anguilla", "Anguilla", "AI", "AIA", "660"},
+	{"Saint Lucia", "Sainte-Lucie", "LC", "LCA", "662"},
+	{"Saint Martin (French part)", "Saint-Martin (partie française)", "MF", "MAF", "663"},
+	{"Saint Pierre and Miquelon", "Saint-Pierre-et-Miquelon", "PM", "SPM", "666"},
+	{"Saint Vincent and the Grenadines", "Saint-Vincent-et-les Grenadines", "VC", "VCT", "670"},
+	{"San Marino", "Saint-Marin", "SM", "SMR", "674"},
+	{"Sao Tome and Principe", "Sao Tomé-et-Principe", "ST", "STP", "678"},
+	{"Saudi Arabia", "Arabie saoudite (l')", "SA", "SAU", "682"},
+	{"Senegal", "Sénégal (le)", "SN", "SEN", "686"},
+	{"Serbia", "Serbie (la)", "RS", "SRB", "688"},
+	{"Seychelles", "Seychelles (les)", "SC", "SYC", "690"},
+	{"Sierra Leone", "Sierra Leone (la)", "SL", "SLE", "694"},
+	{"Singapore", "Singapour", "SG", "SGP", "702"},
+	{"Slovakia", "Slovaquie (la)", "SK", "SVK", "703"},
+	{"Viet Nam", "Viet Nam (le)", "VN", "VNM", "704"},
+	{"Slovenia", "Slovénie (la)", "SI", "SVN", "705"},
+	{"Somalia", "Somalie (la)", "SO", "SOM", "706"},
+	{"South Africa", "Afrique du Sud (l')", "ZA", "ZAF", "710"},
+	{"Zimbabwe", "Zimbabwe (le)", "ZW", "ZWE", "716"},
+	{"Spain", "Espagne (l')", "ES", "ESP", "724"},
+	{"South Sudan", "Soudan du Sud (le)", "SS", "SSD", "728"},
+	{"Sudan (the)", "Soudan (le)", "SD", "SDN", "729"},
+	{"Western Sahara*", "Sahara occidental (le)*", "EH", "ESH", "732"},
+	{"Suriname", "Suriname (le)", "SR", "SUR", "740"},
+	{"Svalbard and Jan Mayen", "Svalbard et l'Île Jan Mayen (le)", "SJ", "SJM", "744"},
+	{"Swaziland", "Swaziland (le)", "SZ", "SWZ", "748"},
+	{"Sweden", "Suède (la)", "SE", "SWE", "752"},
+	{"Switzerland", "Suisse (la)", "CH", "CHE", "756"},
+	{"Syrian Arab Republic", "République arabe syrienne (la)", "SY", "SYR", "760"},
+	{"Tajikistan", "Tadjikistan (le)", "TJ", "TJK", "762"},
+	{"Thailand", "Thaïlande (la)", "TH", "THA", "764"},
+	{"Togo", "Togo (le)", "TG", "TGO", "768"},
+	{"Tokelau", "Tokelau (les)", "TK", "TKL", "772"},
+	{"Tonga", "Tonga (les)", "TO", "TON", "776"},
+	{"Trinidad and Tobago", "Trinité-et-Tobago (la)", "TT", "TTO", "780"},
+	{"United Arab Emirates (the)", "Émirats arabes unis (les)", "AE", "ARE", "784"},
+	{"Tunisia", "Tunisie (la)", "TN", "TUN", "788"},
+	{"Turkey", "Turquie (la)", "TR", "TUR", "792"},
+	{"Turkmenistan", "Turkménistan (le)", "TM", "TKM", "795"},
+	{"Turks and Caicos Islands (the)", "Turks-et-Caïcos (les Îles)", "TC", "TCA", "796"},
+	{"Tuvalu", "Tuvalu (les)", "TV", "TUV", "798"},
+	{"Uganda", "Ouganda (l')", "UG", "UGA", "800"},
+	{"Ukraine", "Ukraine (l')", "UA", "UKR", "804"},
+	{"Macedonia (the former Yugoslav Republic of)", "Macédoine (l'ex‑République yougoslave de)", "MK", "MKD", "807"},
+	{"Egypt", "Égypte (l')", "EG", "EGY", "818"},
+	{"United Kingdom of Great Britain and Northern Ireland (the)", "Royaume-Uni de Grande-Bretagne et d'Irlande du Nord (le)", "GB", "GBR", "826"},
+	{"Guernsey", "Guernesey", "GG", "GGY", "831"},
+	{"Jersey", "Jersey", "JE", "JEY", "832"},
+	{"Isle of Man", "Île de Man", "IM", "IMN", "833"},
+	{"Tanzania, United Republic of", "Tanzanie, République-Unie de", "TZ", "TZA", "834"},
+	{"United States of America (the)", "États-Unis d'Amérique (les)", "US", "USA", "840"},
+	{"Virgin Islands (U.S.)", "Vierges des États-Unis (les Îles)", "VI", "VIR", "850"},
+	{"Burkina Faso", "Burkina Faso (le)", "BF", "BFA", "854"},
+	{"Uruguay", "Uruguay (l')", "UY", "URY", "858"},
+	{"Uzbekistan", "Ouzbékistan (l')", "UZ", "UZB", "860"},
+	{"Venezuela (Bolivarian Republic of)", "Venezuela (République bolivarienne du)", "VE", "VEN", "862"},
+	{"Wallis and Futuna", "Wallis-et-Futuna", "WF", "WLF", "876"},
+	{"Samoa", "Samoa (le)", "WS", "WSM", "882"},
+	{"Yemen", "Yémen (le)", "YE", "YEM", "887"},
+	{"Zambia", "Zambie (la)", "ZM", "ZMB", "894"},
+}
+
+// ISO4217List is the list of ISO currency codes
+var ISO4217List = []string{
+	"AED", "AFN", "ALL", "AMD", "ANG", "AOA", "ARS", "AUD", "AWG", "AZN",
+	"BAM", "BBD", "BDT", "BGN", "BHD", "BIF", "BMD", "BND", "BOB", "BOV", "BRL", "BSD", "BTN", "BWP", "BYN", "BZD",
+	"CAD", "CDF", "CHE", "CHF", "CHW", "CLF", "CLP", "CNY", "COP", "COU", "CRC", "CUC", "CUP", "CVE", "CZK",
+	"DJF", "DKK", "DOP", "DZD",
+	"EGP", "ERN", "ETB", "EUR",
+	"FJD", "FKP",
+	"GBP", "GEL", "GHS", "GIP", "GMD", "GNF", "GTQ", "GYD",
+	"HKD", "HNL", "HRK", "HTG", "HUF",
+	"IDR", "ILS", "INR", "IQD", "IRR", "ISK",
+	"JMD", "JOD", "JPY",
+	"KES", "KGS", "KHR", "KMF", "KPW", "KRW", "KWD", "KYD", "KZT",
+	"LAK", "LBP", "LKR", "LRD", "LSL", "LYD",
+	"MAD", "MDL", "MGA", "MKD", "MMK", "MNT", "MOP", "MRO", "MUR", "MVR", "MWK", "MXN", "MXV", "MYR", "MZN",
+	"NAD", "NGN", "NIO", "NOK", "NPR", "NZD",
+	"OMR",
+	"PAB", "PEN", "PGK", "PHP", "PKR", "PLN", "PYG",
+	"QAR",
+	"RON", "RSD", "RUB", "RWF",
+	"SAR", "SBD", "SCR", "SDG", "SEK", "SGD", "SHP", "SLL", "SOS", "SRD", "SSP", "STD", "STN", "SVC", "SYP", "SZL",
+	"THB", "TJS", "TMT", "TND", "TOP", "TRY", "TTD", "TWD", "TZS",
+	"UAH", "UGX", "USD", "USN", "UYI", "UYU", "UYW", "UZS",
+	"VEF", "VES", "VND", "VUV",
+	"WST",
+	"XAF", "XAG", "XAU", "XBA", "XBB", "XBC", "XBD", "XCD", "XDR", "XOF", "XPD", "XPF", "XPT", "XSU", "XTS", "XUA", "XXX",
+	"YER",
+	"ZAR", "ZMW", "ZWL",
+}
+
+// ISO693Entry stores ISO language codes
+type ISO693Entry struct {
+	Alpha3bCode string
+	Alpha2Code  string
+	English     string
+}
+
+//ISO693List based on http://data.okfn.org/data/core/language-codes/r/language-codes-3b2.json
+var ISO693List = []ISO693Entry{
+	{Alpha3bCode: "aar", Alpha2Code: "aa", English: "Afar"},
+	{Alpha3bCode: "abk", Alpha2Code: "ab", English: "Abkhazian"},
+	{Alpha3bCode: "afr", Alpha2Code: "af", English: "Afrikaans"},
+	{Alpha3bCode: "aka", Alpha2Code: "ak", English: "Akan"},
+	{Alpha3bCode: "alb", Alpha2Code: "sq", English: "Albanian"},
+	{Alpha3bCode: "amh", Alpha2Code: "am", English: "Amharic"},
+	{Alpha3bCode: "ara", Alpha2Code: "ar", English: "Arabic"},
+	{Alpha3bCode: "arg", Alpha2Code: "an", English: "Aragonese"},
+	{Alpha3bCode: "arm", Alpha2Code: "hy", English: "Armenian"},
+	{Alpha3bCode: "asm", Alpha2Code: "as", English: "Assamese"},
+	{Alpha3bCode: "ava", Alpha2Code: "av", English: "Avaric"},
+	{Alpha3bCode: "ave", Alpha2Code: "ae", English: "Avestan"},
+	{Alpha3bCode: "aym", Alpha2Code: "ay", English: "Aymara"},
+	{Alpha3bCode: "aze", Alpha2Code: "az", English: "Azerbaijani"},
+	{Alpha3bCode: "bak", Alpha2Code: "ba", English: "Bashkir"},
+	{Alpha3bCode: "bam", Alpha2Code: "bm", English: "Bambara"},
+	{Alpha3bCode: "baq", Alpha2Code: "eu", English: "Basque"},
+	{Alpha3bCode: "bel", Alpha2Code: "be", English: "Belarusian"},
+	{Alpha3bCode: "ben", Alpha2Code: "bn", English: "Bengali"},
+	{Alpha3bCode: "bih", Alpha2Code: "bh", English: "Bihari languages"},
+	{Alpha3bCode: "bis", Alpha2Code: "bi", English: "Bislama"},
+	{Alpha3bCode: "bos", Alpha2Code: "bs", English: "Bosnian"},
+	{Alpha3bCode: "bre", Alpha2Code: "br", English: "Breton"},
+	{Alpha3bCode: "bul", Alpha2Code: "bg", English: "Bulgarian"},
+	{Alpha3bCode: "bur", Alpha2Code: "my", English: "Burmese"},
+	{Alpha3bCode: "cat", Alpha2Code: "ca", English: "Catalan; Valencian"},
+	{Alpha3bCode: "cha", Alpha2Code: "ch", English: "Chamorro"},
+	{Alpha3bCode: "che", Alpha2Code: "ce", English: "Chechen"},
+	{Alpha3bCode: "chi", Alpha2Code: "zh", English: "Chinese"},
+	{Alpha3bCode: "chu", Alpha2Code: "cu", English: "Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic"},
+	{Alpha3bCode: "chv", Alpha2Code: "cv", English: "Chuvash"},
+	{Alpha3bCode: "cor", Alpha2Code: "kw", English: "Cornish"},
+	{Alpha3bCode: "cos", Alpha2Code: "co", English: "Corsican"},
+	{Alpha3bCode: "cre", Alpha2Code: "cr", English: "Cree"},
+	{Alpha3bCode: "cze", Alpha2Code: "cs", English: "Czech"},
+	{Alpha3bCode: "dan", Alpha2Code: "da", English: "Danish"},
+	{Alpha3bCode: "div", Alpha2Code: "dv", English: "Divehi; Dhivehi; Maldivian"},
+	{Alpha3bCode: "dut", Alpha2Code: "nl", English: "Dutch; Flemish"},
+	{Alpha3bCode: "dzo", Alpha2Code: "dz", English: "Dzongkha"},
+	{Alpha3bCode: "eng", Alpha2Code: "en", English: "English"},
+	{Alpha3bCode: "epo", Alpha2Code: "eo", English: "Esperanto"},
+	{Alpha3bCode: "est", Alpha2Code: "et", English: "Estonian"},
+	{Alpha3bCode: "ewe", Alpha2Code: "ee", English: "Ewe"},
+	{Alpha3bCode: "fao", Alpha2Code: "fo", English: "Faroese"},
+	{Alpha3bCode: "fij", Alpha2Code: "fj", English: "Fijian"},
+	{Alpha3bCode: "fin", Alpha2Code: "fi", English: "Finnish"},
+	{Alpha3bCode: "fre", Alpha2Code: "fr", English: "French"},
+	{Alpha3bCode: "fry", Alpha2Code: "fy", English: "Western Frisian"},
+	{Alpha3bCode: "ful", Alpha2Code: "ff", English: "Fulah"},
+	{Alpha3bCode: "geo", Alpha2Code: "ka", English: "Georgian"},
+	{Alpha3bCode: "ger", Alpha2Code: "de", English: "German"},
+	{Alpha3bCode: "gla", Alpha2Code: "gd", English: "Gaelic; Scottish Gaelic"},
+	{Alpha3bCode: "gle", Alpha2Code: "ga", English: "Irish"},
+	{Alpha3bCode: "glg", Alpha2Code: "gl", English: "Galician"},
+	{Alpha3bCode: "glv", Alpha2Code: "gv", English: "Manx"},
+	{Alpha3bCode: "gre", Alpha2Code: "el", English: "Greek, Modern (1453-)"},
+	{Alpha3bCode: "grn", Alpha2Code: "gn", English: "Guarani"},
+	{Alpha3bCode: "guj", Alpha2Code: "gu", English: "Gujarati"},
+	{Alpha3bCode: "hat", Alpha2Code: "ht", English: "Haitian; Haitian Creole"},
+	{Alpha3bCode: "hau", Alpha2Code: "ha", English: "Hausa"},
+	{Alpha3bCode: "heb", Alpha2Code: "he", English: "Hebrew"},
+	{Alpha3bCode: "her", Alpha2Code: "hz", English: "Herero"},
+	{Alpha3bCode: "hin", Alpha2Code: "hi", English: "Hindi"},
+	{Alpha3bCode: "hmo", Alpha2Code: "ho", English: "Hiri Motu"},
+	{Alpha3bCode: "hrv", Alpha2Code: "hr", English: "Croatian"},
+	{Alpha3bCode: "hun", Alpha2Code: "hu", English: "Hungarian"},
+	{Alpha3bCode: "ibo", Alpha2Code: "ig", English: "Igbo"},
+	{Alpha3bCode: "ice", Alpha2Code: "is", English: "Icelandic"},
+	{Alpha3bCode: "ido", Alpha2Code: "io", English: "Ido"},
+	{Alpha3bCode: "iii", Alpha2Code: "ii", English: "Sichuan Yi; Nuosu"},
+	{Alpha3bCode: "iku", Alpha2Code: "iu", English: "Inuktitut"},
+	{Alpha3bCode: "ile", Alpha2Code: "ie", English: "Interlingue; Occidental"},
+	{Alpha3bCode: "ina", Alpha2Code: "ia", English: "Interlingua (International Auxiliary Language Association)"},
+	{Alpha3bCode: "ind", Alpha2Code: "id", English: "Indonesian"},
+	{Alpha3bCode: "ipk", Alpha2Code: "ik", English: "Inupiaq"},
+	{Alpha3bCode: "ita", Alpha2Code: "it", English: "Italian"},
+	{Alpha3bCode: "jav", Alpha2Code: "jv", English: "Javanese"},
+	{Alpha3bCode: "jpn", Alpha2Code: "ja", English: "Japanese"},
+	{Alpha3bCode: "kal", Alpha2Code: "kl", English: "Kalaallisut; Greenlandic"},
+	{Alpha3bCode: "kan", Alpha2Code: "kn", English: "Kannada"},
+	{Alpha3bCode: "kas", Alpha2Code: "ks", English: "Kashmiri"},
+	{Alpha3bCode: "kau", Alpha2Code: "kr", English: "Kanuri"},
+	{Alpha3bCode: "kaz", Alpha2Code: "kk", English: "Kazakh"},
+	{Alpha3bCode: "khm", Alpha2Code: "km", English: "Central Khmer"},
+	{Alpha3bCode: "kik", Alpha2Code: "ki", English: "Kikuyu; Gikuyu"},
+	{Alpha3bCode: "kin", Alpha2Code: "rw", English: "Kinyarwanda"},
+	{Alpha3bCode: "kir", Alpha2Code: "ky", English: "Kirghiz; Kyrgyz"},
+	{Alpha3bCode: "kom", Alpha2Code: "kv", English: "Komi"},
+	{Alpha3bCode: "kon", Alpha2Code: "kg", English: "Kongo"},
+	{Alpha3bCode: "kor", Alpha2Code: "ko", English: "Korean"},
+	{Alpha3bCode: "kua", Alpha2Code: "kj", English: "Kuanyama; Kwanyama"},
+	{Alpha3bCode: "kur", Alpha2Code: "ku", English: "Kurdish"},
+	{Alpha3bCode: "lao", Alpha2Code: "lo", English: "Lao"},
+	{Alpha3bCode: "lat", Alpha2Code: "la", English: "Latin"},
+	{Alpha3bCode: "lav", Alpha2Code: "lv", English: "Latvian"},
+	{Alpha3bCode: "lim", Alpha2Code: "li", English: "Limburgan; Limburger; Limburgish"},
+	{Alpha3bCode: "lin", Alpha2Code: "ln", English: "Lingala"},
+	{Alpha3bCode: "lit", Alpha2Code: "lt", English: "Lithuanian"},
+	{Alpha3bCode: "ltz", Alpha2Code: "lb", English: "Luxembourgish; Letzeburgesch"},
+	{Alpha3bCode: "lub", Alpha2Code: "lu", English: "Luba-Katanga"},
+	{Alpha3bCode: "lug", Alpha2Code: "lg", English: "Ganda"},
+	{Alpha3bCode: "mac", Alpha2Code: "mk", English: "Macedonian"},
+	{Alpha3bCode: "mah", Alpha2Code: "mh", English: "Marshallese"},
+	{Alpha3bCode: "mal", Alpha2Code: "ml", English: "Malayalam"},
+	{Alpha3bCode: "mao", Alpha2Code: "mi", English: "Maori"},
+	{Alpha3bCode: "mar", Alpha2Code: "mr", English: "Marathi"},
+	{Alpha3bCode: "may", Alpha2Code: "ms", English: "Malay"},
+	{Alpha3bCode: "mlg", Alpha2Code: "mg", English: "Malagasy"},
+	{Alpha3bCode: "mlt", Alpha2Code: "mt", English: "Maltese"},
+	{Alpha3bCode: "mon", Alpha2Code: "mn", English: "Mongolian"},
+	{Alpha3bCode: "nau", Alpha2Code: "na", English: "Nauru"},
+	{Alpha3bCode: "nav", Alpha2Code: "nv", English: "Navajo; Navaho"},
+	{Alpha3bCode: "nbl", Alpha2Code: "nr", English: "Ndebele, South; South Ndebele"},
+	{Alpha3bCode: "nde", Alpha2Code: "nd", English: "Ndebele, North; North Ndebele"},
+	{Alpha3bCode: "ndo", Alpha2Code: "ng", English: "Ndonga"},
+	{Alpha3bCode: "nep", Alpha2Code: "ne", English: "Nepali"},
+	{Alpha3bCode: "nno", Alpha2Code: "nn", English: "Norwegian Nynorsk; Nynorsk, Norwegian"},
+	{Alpha3bCode: "nob", Alpha2Code: "nb", English: "Bokmål, Norwegian; Norwegian Bokmål"},
+	{Alpha3bCode: "nor", Alpha2Code: "no", English: "Norwegian"},
+	{Alpha3bCode: "nya", Alpha2Code: "ny", English: "Chichewa; Chewa; Nyanja"},
+	{Alpha3bCode: "oci", Alpha2Code: "oc", English: "Occitan (post 1500); Provençal"},
+	{Alpha3bCode: "oji", Alpha2Code: "oj", English: "Ojibwa"},
+	{Alpha3bCode: "ori", Alpha2Code: "or", English: "Oriya"},
+	{Alpha3bCode: "orm", Alpha2Code: "om", English: "Oromo"},
+	{Alpha3bCode: "oss", Alpha2Code: "os", English: "Ossetian; Ossetic"},
+	{Alpha3bCode: "pan", Alpha2Code: "pa", English: "Panjabi; Punjabi"},
+	{Alpha3bCode: "per", Alpha2Code: "fa", English: "Persian"},
+	{Alpha3bCode: "pli", Alpha2Code: "pi", English: "Pali"},
+	{Alpha3bCode: "pol", Alpha2Code: "pl", English: "Polish"},
+	{Alpha3bCode: "por", Alpha2Code: "pt", English: "Portuguese"},
+	{Alpha3bCode: "pus", Alpha2Code: "ps", English: "Pushto; Pashto"},
+	{Alpha3bCode: "que", Alpha2Code: "qu", English: "Quechua"},
+	{Alpha3bCode: "roh", Alpha2Code: "rm", English: "Romansh"},
+	{Alpha3bCode: "rum", Alpha2Code: "ro", English: "Romanian; Moldavian; Moldovan"},
+	{Alpha3bCode: "run", Alpha2Code: "rn", English: "Rundi"},
+	{Alpha3bCode: "rus", Alpha2Code: "ru", English: "Russian"},
+	{Alpha3bCode: "sag", Alpha2Code: "sg", English: "Sango"},
+	{Alpha3bCode: "san", Alpha2Code: "sa", English: "Sanskrit"},
+	{Alpha3bCode: "sin", Alpha2Code: "si", English: "Sinhala; Sinhalese"},
+	{Alpha3bCode: "slo", Alpha2Code: "sk", English: "Slovak"},
+	{Alpha3bCode: "slv", Alpha2Code: "sl", English: "Slovenian"},
+	{Alpha3bCode: "sme", Alpha2Code: "se", English: "Northern Sami"},
+	{Alpha3bCode: "smo", Alpha2Code: "sm", English: "Samoan"},
+	{Alpha3bCode: "sna", Alpha2Code: "sn", English: "Shona"},
+	{Alpha3bCode: "snd", Alpha2Code: "sd", English: "Sindhi"},
+	{Alpha3bCode: "som", Alpha2Code: "so", English: "Somali"},
+	{Alpha3bCode: "sot", Alpha2Code: "st", English: "Sotho, Southern"},
+	{Alpha3bCode: "spa", Alpha2Code: "es", English: "Spanish; Castilian"},
+	{Alpha3bCode: "srd", Alpha2Code: "sc", English: "Sardinian"},
+	{Alpha3bCode: "srp", Alpha2Code: "sr", English: "Serbian"},
+	{Alpha3bCode: "ssw", Alpha2Code: "ss", English: "Swati"},
+	{Alpha3bCode: "sun", Alpha2Code: "su", English: "Sundanese"},
+	{Alpha3bCode: "swa", Alpha2Code: "sw", English: "Swahili"},
+	{Alpha3bCode: "swe", Alpha2Code: "sv", English: "Swedish"},
+	{Alpha3bCode: "tah", Alpha2Code: "ty", English: "Tahitian"},
+	{Alpha3bCode: "tam", Alpha2Code: "ta", English: "Tamil"},
+	{Alpha3bCode: "tat", Alpha2Code: "tt", English: "Tatar"},
+	{Alpha3bCode: "tel", Alpha2Code: "te", English: "Telugu"},
+	{Alpha3bCode: "tgk", Alpha2Code: "tg", English: "Tajik"},
+	{Alpha3bCode: "tgl", Alpha2Code: "tl", English: "Tagalog"},
+	{Alpha3bCode: "tha", Alpha2Code: "th", English: "Thai"},
+	{Alpha3bCode: "tib", Alpha2Code: "bo", English: "Tibetan"},
+	{Alpha3bCode: "tir", Alpha2Code: "ti", English: "Tigrinya"},
+	{Alpha3bCode: "ton", Alpha2Code: "to", English: "Tonga (Tonga Islands)"},
+	{Alpha3bCode: "tsn", Alpha2Code: "tn", English: "Tswana"},
+	{Alpha3bCode: "tso", Alpha2Code: "ts", English: "Tsonga"},
+	{Alpha3bCode: "tuk", Alpha2Code: "tk", English: "Turkmen"},
+	{Alpha3bCode: "tur", Alpha2Code: "tr", English: "Turkish"},
+	{Alpha3bCode: "twi", Alpha2Code: "tw", English: "Twi"},
+	{Alpha3bCode: "uig", Alpha2Code: "ug", English: "Uighur; Uyghur"},
+	{Alpha3bCode: "ukr", Alpha2Code: "uk", English: "Ukrainian"},
+	{Alpha3bCode: "urd", Alpha2Code: "ur", English: "Urdu"},
+	{Alpha3bCode: "uzb", Alpha2Code: "uz", English: "Uzbek"},
+	{Alpha3bCode: "ven", Alpha2Code: "ve", English: "Venda"},
+	{Alpha3bCode: "vie", Alpha2Code: "vi", English: "Vietnamese"},
+	{Alpha3bCode: "vol", Alpha2Code: "vo", English: "Volapük"},
+	{Alpha3bCode: "wel", Alpha2Code: "cy", English: "Welsh"},
+	{Alpha3bCode: "wln", Alpha2Code: "wa", English: "Walloon"},
+	{Alpha3bCode: "wol", Alpha2Code: "wo", English: "Wolof"},
+	{Alpha3bCode: "xho", Alpha2Code: "xh", English: "Xhosa"},
+	{Alpha3bCode: "yid", Alpha2Code: "yi", English: "Yiddish"},
+	{Alpha3bCode: "yor", Alpha2Code: "yo", English: "Yoruba"},
+	{Alpha3bCode: "zha", Alpha2Code: "za", English: "Zhuang; Chuang"},
+	{Alpha3bCode: "zul", Alpha2Code: "zu", English: "Zulu"},
+}

common_utils/validator/utils.go

@@ -0,0 +1,270 @@
+package govalidator
+
+import (
+	"errors"
+	"fmt"
+	"html"
+	"math"
+	"path"
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// Contains checks if the string contains the substring.
+func Contains(str, substring string) bool {
+	return strings.Contains(str, substring)
+}
+
+// Matches checks if string matches the pattern (pattern is regular expression)
+// In case of error return false
+func Matches(str, pattern string) bool {
+	match, _ := regexp.MatchString(pattern, str)
+	return match
+}
+
+// LeftTrim trims characters from the left side of the input.
+// If second argument is empty, it will remove leading spaces.
+func LeftTrim(str, chars string) string {
+	if chars == "" {
+		return strings.TrimLeftFunc(str, unicode.IsSpace)
+	}
+	r, _ := regexp.Compile("^[" + chars + "]+")
+	return r.ReplaceAllString(str, "")
+}
+
+// RightTrim trims characters from the right side of the input.
+// If second argument is empty, it will remove trailing spaces.
+func RightTrim(str, chars string) string {
+	if chars == "" {
+		return strings.TrimRightFunc(str, unicode.IsSpace)
+	}
+	r, _ := regexp.Compile("[" + chars + "]+$")
+	return r.ReplaceAllString(str, "")
+}
+
+// Trim trims characters from both sides of the input.
+// If second argument is empty, it will remove spaces.
+func Trim(str, chars string) string {
+	return LeftTrim(RightTrim(str, chars), chars)
+}
+
+// WhiteList removes characters that do not appear in the whitelist.
+func WhiteList(str, chars string) string {
+	pattern := "[^" + chars + "]+"
+	r, _ := regexp.Compile(pattern)
+	return r.ReplaceAllString(str, "")
+}
+
+// BlackList removes characters that appear in the blacklist.
+func BlackList(str, chars string) string {
+	pattern := "[" + chars + "]+"
+	r, _ := regexp.Compile(pattern)
+	return r.ReplaceAllString(str, "")
+}
+
+// StripLow removes characters with a numerical value < 32 and 127, mostly control characters.
+// If keep_new_lines is true, newline characters are preserved (\n and \r, hex 0xA and 0xD).
+func StripLow(str string, keepNewLines bool) string {
+	chars := ""
+	if keepNewLines {
+		chars = "\x00-\x09\x0B\x0C\x0E-\x1F\x7F"
+	} else {
+		chars = "\x00-\x1F\x7F"
+	}
+	return BlackList(str, chars)
+}
+
+// ReplacePattern replaces regular expression pattern in string
+func ReplacePattern(str, pattern, replace string) string {
+	r, _ := regexp.Compile(pattern)
+	return r.ReplaceAllString(str, replace)
+}
+
+// Escape replaces <, >, & and " with HTML entities.
+var Escape = html.EscapeString
+
+func addSegment(inrune, segment []rune) []rune {
+	if len(segment) == 0 {
+		return inrune
+	}
+	if len(inrune) != 0 {
+		inrune = append(inrune, '_')
+	}
+	inrune = append(inrune, segment...)
+	return inrune
+}
+
+// UnderscoreToCamelCase converts from underscore separated form to camel case form.
+// Ex.: my_func => MyFunc
+func UnderscoreToCamelCase(s string) string {
+	return strings.Replace(strings.Title(strings.Replace(strings.ToLower(s), "_", " ", -1)), " ", "", -1)
+}
+
+// CamelCaseToUnderscore converts from camel case form to underscore separated form.
+// Ex.: MyFunc => my_func
+func CamelCaseToUnderscore(str string) string {
+	var output []rune
+	var segment []rune
+	for _, r := range str {
+
+		// not treat number as separate segment
+		if !unicode.IsLower(r) && string(r) != "_" && !unicode.IsNumber(r) {
+			output = addSegment(output, segment)
+			segment = nil
+		}
+		segment = append(segment, unicode.ToLower(r))
+	}
+	output = addSegment(output, segment)
+	return string(output)
+}
+
+// Reverse returns reversed string
+func Reverse(s string) string {
+	r := []rune(s)
+	for i, j := 0, len(r)-1; i < j; i, j = i+1, j-1 {
+		r[i], r[j] = r[j], r[i]
+	}
+	return string(r)
+}
+
+// GetLines splits string by "\n" and return array of lines
+func GetLines(s string) []string {
+	return strings.Split(s, "\n")
+}
+
+// GetLine returns specified line of multiline string
+func GetLine(s string, index int) (string, error) {
+	lines := GetLines(s)
+	if index < 0 || index >= len(lines) {
+		return "", errors.New("line index out of bounds")
+	}
+	return lines[index], nil
+}
+
+// RemoveTags removes all tags from HTML string
+func RemoveTags(s string) string {
+	return ReplacePattern(s, "<[^>]*>", "")
+}
+
+// SafeFileName returns safe string that can be used in file names
+func SafeFileName(str string) string {
+	name := strings.ToLower(str)
+	name = path.Clean(path.Base(name))
+	name = strings.Trim(name, " ")
+	separators, err := regexp.Compile(`[ &_=+:]`)
+	if err == nil {
+		name = separators.ReplaceAllString(name, "-")
+	}
+	legal, err := regexp.Compile(`[^[:alnum:]-.]`)
+	if err == nil {
+		name = legal.ReplaceAllString(name, "")
+	}
+	for strings.Contains(name, "--") {
+		name = strings.Replace(name, "--", "-", -1)
+	}
+	return name
+}
+
+// NormalizeEmail canonicalize an email address.
+// The local part of the email address is lowercased for all domains; the hostname is always lowercased and
+// the local part of the email address is always lowercased for hosts that are known to be case-insensitive (currently only GMail).
+// Normalization follows special rules for known providers: currently, GMail addresses have dots removed in the local part and
+// are stripped of tags (e.g. some.one+tag@gmail.com becomes someone@gmail.com) and all @googlemail.com addresses are
+// normalized to @gmail.com.
+func NormalizeEmail(str string) (string, error) {
+	if !IsEmail(str) {
+		return "", fmt.Errorf("%s is not an email", str)
+	}
+	parts := strings.Split(str, "@")
+	parts[0] = strings.ToLower(parts[0])
+	parts[1] = strings.ToLower(parts[1])
+	if parts[1] == "gmail.com" || parts[1] == "googlemail.com" {
+		parts[1] = "gmail.com"
+		parts[0] = strings.Split(ReplacePattern(parts[0], `\.`, ""), "+")[0]
+	}
+	return strings.Join(parts, "@"), nil
+}
+
+// Truncate a string to the closest length without breaking words.
+func Truncate(str string, length int, ending string) string {
+	var aftstr, befstr string
+	if len(str) > length {
+		words := strings.Fields(str)
+		before, present := 0, 0
+		for i := range words {
+			befstr = aftstr
+			before = present
+			aftstr = aftstr + words[i] + " "
+			present = len(aftstr)
+			if present > length && i != 0 {
+				if (length - before) < (present - length) {
+					return Trim(befstr, " /\\.,\"'#!?&@+-") + ending
+				}
+				return Trim(aftstr, " /\\.,\"'#!?&@+-") + ending
+			}
+		}
+	}
+
+	return str
+}
+
+// PadLeft pads left side of a string if size of string is less then indicated pad length
+func PadLeft(str string, padStr string, padLen int) string {
+	return buildPadStr(str, padStr, padLen, true, false)
+}
+
+// PadRight pads right side of a string if size of string is less then indicated pad length
+func PadRight(str string, padStr string, padLen int) string {
+	return buildPadStr(str, padStr, padLen, false, true)
+}
+
+// PadBoth pads both sides of a string if size of string is less then indicated pad length
+func PadBoth(str string, padStr string, padLen int) string {
+	return buildPadStr(str, padStr, padLen, true, true)
+}
+
+// PadString either left, right or both sides.
+// Note that padding string can be unicode and more then one character
+func buildPadStr(str string, padStr string, padLen int, padLeft bool, padRight bool) string {
+
+	// When padded length is less then the current string size
+	if padLen < utf8.RuneCountInString(str) {
+		return str
+	}
+
+	padLen -= utf8.RuneCountInString(str)
+
+	targetLen := padLen
+
+	targetLenLeft := targetLen
+	targetLenRight := targetLen
+	if padLeft && padRight {
+		targetLenLeft = padLen / 2
+		targetLenRight = padLen - targetLenLeft
+	}
+
+	strToRepeatLen := utf8.RuneCountInString(padStr)
+
+	repeatTimes := int(math.Ceil(float64(targetLen) / float64(strToRepeatLen)))
+	repeatedString := strings.Repeat(padStr, repeatTimes)
+
+	leftSide := ""
+	if padLeft {
+		leftSide = repeatedString[0:targetLenLeft]
+	}
+
+	rightSide := ""
+	if padRight {
+		rightSide = repeatedString[0:targetLenRight]
+	}
+
+	return leftSide + str + rightSide
+}
+
+// TruncatingErrorf removes extra args from fmt.Errorf if not formatted in the str object
+func TruncatingErrorf(str string, args ...interface{}) error {
+	n := strings.Count(str, "%s")
+	return fmt.Errorf(str, args[:n]...)
+}

common_utils/validator/validator.go

@@ -0,0 +1,1622 @@
+// Package govalidator is package of validators and sanitizers for strings, structs and collections.
+package govalidator
+
+import (
+	"bytes"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/url"
+	"reflect"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+	"unicode/utf8"
+)
+
+var (
+	fieldsRequiredByDefault bool
+	nilPtrAllowedByRequired = false
+	notNumberRegexp         = regexp.MustCompile("[^0-9]+")
+	whiteSpacesAndMinus     = regexp.MustCompile(`[\s-]+`)
+	paramsRegexp            = regexp.MustCompile(`\(.*\)$`)
+)
+
+const maxURLRuneCount = 2083
+const minURLRuneCount = 3
+const rfc3339WithoutZone = "2006-01-02T15:04:05"
+
+// SetFieldsRequiredByDefault causes validation to fail when struct fields
+// do not include validations or are not explicitly marked as exempt (using `valid:"-"` or `valid:"email,optional"`).
+// This struct definition will fail govalidator.ValidateStruct() (and the field values do not matter):
+//     type exampleStruct struct {
+//         Name  string ``
+//         Email string `valid:"email"`
+// This, however, will only fail when Email is empty or an invalid email address:
+//     type exampleStruct2 struct {
+//         Name  string `valid:"-"`
+//         Email string `valid:"email"`
+// Lastly, this will only fail when Email is an invalid email address but not when it's empty:
+//     type exampleStruct2 struct {
+//         Name  string `valid:"-"`
+//         Email string `valid:"email,optional"`
+func SetFieldsRequiredByDefault(value bool) {
+	fieldsRequiredByDefault = value
+}
+
+// SetNilPtrAllowedByRequired causes validation to pass for nil ptrs when a field is set to required.
+// The validation will still reject ptr fields in their zero value state. Example with this enabled:
+//     type exampleStruct struct {
+//         Name  *string `valid:"required"`
+// With `Name` set to "", this will be considered invalid input and will cause a validation error.
+// With `Name` set to nil, this will be considered valid by validation.
+// By default this is disabled.
+func SetNilPtrAllowedByRequired(value bool) {
+	nilPtrAllowedByRequired = value
+}
+
+// IsEmail checks if the string is an email.
+func IsEmail(str string) bool {
+	// TODO uppercase letters are not supported
+	return rxEmail.MatchString(str)
+}
+
+// IsExistingEmail checks if the string is an email of existing domain
+func IsExistingEmail(email string) bool {
+
+	if len(email) < 6 || len(email) > 254 {
+		return false
+	}
+	at := strings.LastIndex(email, "@")
+	if at <= 0 || at > len(email)-3 {
+		return false
+	}
+	user := email[:at]
+	host := email[at+1:]
+	if len(user) > 64 {
+		return false
+	}
+	switch host {
+	case "localhost", "example.com":
+		return true
+	}
+	if userDotRegexp.MatchString(user) || !userRegexp.MatchString(user) || !hostRegexp.MatchString(host) {
+		return false
+	}
+	if _, err := net.LookupMX(host); err != nil {
+		if _, err := net.LookupIP(host); err != nil {
+			return false
+		}
+	}
+
+	return true
+}
+
+// IsURL checks if the string is an URL.
+func IsURL(str string) bool {
+	if str == "" || utf8.RuneCountInString(str) >= maxURLRuneCount || len(str) <= minURLRuneCount || strings.HasPrefix(str, ".") {
+		return false
+	}
+	strTemp := str
+	if strings.Contains(str, ":") && !strings.Contains(str, "://") {
+		// support no indicated urlscheme but with colon for port number
+		// http:// is appended so url.Parse will succeed, strTemp used so it does not impact rxURL.MatchString
+		strTemp = "http://" + str
+	}
+	u, err := url.Parse(strTemp)
+	if err != nil {
+		return false
+	}
+	if strings.HasPrefix(u.Host, ".") {
+		return false
+	}
+	if u.Host == "" && (u.Path != "" && !strings.Contains(u.Path, ".")) {
+		return false
+	}
+	return rxURL.MatchString(str)
+}
+
+// IsRequestURL checks if the string rawurl, assuming
+// it was received in an HTTP request, is a valid
+// URL confirm to RFC 3986
+func IsRequestURL(rawurl string) bool {
+	url, err := url.ParseRequestURI(rawurl)
+	if err != nil {
+		return false //Couldn't even parse the rawurl
+	}
+	if len(url.Scheme) == 0 {
+		return false //No Scheme found
+	}
+	return true
+}
+
+// IsRequestURI checks if the string rawurl, assuming
+// it was received in an HTTP request, is an
+// absolute URI or an absolute path.
+func IsRequestURI(rawurl string) bool {
+	_, err := url.ParseRequestURI(rawurl)
+	return err == nil
+}
+
+// IsAlpha checks if the string contains only letters (a-zA-Z). Empty string is valid.
+func IsAlpha(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxAlpha.MatchString(str)
+}
+
+//IsUTFLetter checks if the string contains only unicode letter characters.
+//Similar to IsAlpha but for all languages. Empty string is valid.
+func IsUTFLetter(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+
+	for _, c := range str {
+		if !unicode.IsLetter(c) {
+			return false
+		}
+	}
+	return true
+
+}
+
+// IsAlphanumeric checks if the string contains only letters and numbers. Empty string is valid.
+func IsAlphanumeric(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxAlphanumeric.MatchString(str)
+}
+
+// IsUTFLetterNumeric checks if the string contains only unicode letters and numbers. Empty string is valid.
+func IsUTFLetterNumeric(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	for _, c := range str {
+		if !unicode.IsLetter(c) && !unicode.IsNumber(c) { //letters && numbers are ok
+			return false
+		}
+	}
+	return true
+
+}
+
+// IsNumeric checks if the string contains only numbers. Empty string is valid.
+func IsNumeric(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxNumeric.MatchString(str)
+}
+
+// IsUTFNumeric checks if the string contains only unicode numbers of any kind.
+// Numbers can be 0-9 but also Fractions ¾,Roman Ⅸ and Hangzhou 〩. Empty string is valid.
+func IsUTFNumeric(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	if strings.IndexAny(str, "+-") > 0 {
+		return false
+	}
+	if len(str) > 1 {
+		str = strings.TrimPrefix(str, "-")
+		str = strings.TrimPrefix(str, "+")
+	}
+	for _, c := range str {
+		if !unicode.IsNumber(c) { //numbers && minus sign are ok
+			return false
+		}
+	}
+	return true
+
+}
+
+// IsUTFDigit checks if the string contains only unicode radix-10 decimal digits. Empty string is valid.
+func IsUTFDigit(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	if strings.IndexAny(str, "+-") > 0 {
+		return false
+	}
+	if len(str) > 1 {
+		str = strings.TrimPrefix(str, "-")
+		str = strings.TrimPrefix(str, "+")
+	}
+	for _, c := range str {
+		if !unicode.IsDigit(c) { //digits && minus sign are ok
+			return false
+		}
+	}
+	return true
+
+}
+
+// IsHexadecimal checks if the string is a hexadecimal number.
+func IsHexadecimal(str string) bool {
+	return rxHexadecimal.MatchString(str)
+}
+
+// IsHexcolor checks if the string is a hexadecimal color.
+func IsHexcolor(str string) bool {
+	return rxHexcolor.MatchString(str)
+}
+
+// IsRGBcolor checks if the string is a valid RGB color in form rgb(RRR, GGG, BBB).
+func IsRGBcolor(str string) bool {
+	return rxRGBcolor.MatchString(str)
+}
+
+// IsLowerCase checks if the string is lowercase. Empty string is valid.
+func IsLowerCase(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return str == strings.ToLower(str)
+}
+
+// IsUpperCase checks if the string is uppercase. Empty string is valid.
+func IsUpperCase(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return str == strings.ToUpper(str)
+}
+
+// HasLowerCase checks if the string contains at least 1 lowercase. Empty string is valid.
+func HasLowerCase(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxHasLowerCase.MatchString(str)
+}
+
+// HasUpperCase checks if the string contains as least 1 uppercase. Empty string is valid.
+func HasUpperCase(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxHasUpperCase.MatchString(str)
+}
+
+// IsInt checks if the string is an integer. Empty string is valid.
+func IsInt(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxInt.MatchString(str)
+}
+
+// IsFloat checks if the string is a float.
+func IsFloat(str string) bool {
+	return str != "" && rxFloat.MatchString(str)
+}
+
+// IsDivisibleBy checks if the string is a number that's divisible by another.
+// If second argument is not valid integer or zero, it's return false.
+// Otherwise, if first argument is not valid integer or zero, it's return true (Invalid string converts to zero).
+func IsDivisibleBy(str, num string) bool {
+	f, _ := ToFloat(str)
+	p := int64(f)
+	q, _ := ToInt(num)
+	if q == 0 {
+		return false
+	}
+	return (p == 0) || (p%q == 0)
+}
+
+// IsNull checks if the string is null.
+func IsNull(str string) bool {
+	return len(str) == 0
+}
+
+// IsNotNull checks if the string is not null.
+func IsNotNull(str string) bool {
+	return !IsNull(str)
+}
+
+// HasWhitespaceOnly checks the string only contains whitespace
+func HasWhitespaceOnly(str string) bool {
+	return len(str) > 0 && rxHasWhitespaceOnly.MatchString(str)
+}
+
+// HasWhitespace checks if the string contains any whitespace
+func HasWhitespace(str string) bool {
+	return len(str) > 0 && rxHasWhitespace.MatchString(str)
+}
+
+// IsByteLength checks if the string's length (in bytes) falls in a range.
+func IsByteLength(str string, min, max int) bool {
+	return len(str) >= min && len(str) <= max
+}
+
+// IsUUIDv3 checks if the string is a UUID version 3.
+func IsUUIDv3(str string) bool {
+	return rxUUID3.MatchString(str)
+}
+
+// IsUUIDv4 checks if the string is a UUID version 4.
+func IsUUIDv4(str string) bool {
+	return rxUUID4.MatchString(str)
+}
+
+// IsUUIDv5 checks if the string is a UUID version 5.
+func IsUUIDv5(str string) bool {
+	return rxUUID5.MatchString(str)
+}
+
+// IsUUID checks if the string is a UUID (version 3, 4 or 5).
+func IsUUID(str string) bool {
+	return rxUUID.MatchString(str)
+}
+
+// IsCreditCard checks if the string is a credit card.
+func IsCreditCard(str string) bool {
+	sanitized := notNumberRegexp.ReplaceAllString(str, "")
+	if !rxCreditCard.MatchString(sanitized) {
+		return false
+	}
+	var sum int64
+	var digit string
+	var tmpNum int64
+	var shouldDouble bool
+	for i := len(sanitized) - 1; i >= 0; i-- {
+		digit = sanitized[i:(i + 1)]
+		tmpNum, _ = ToInt(digit)
+		if shouldDouble {
+			tmpNum *= 2
+			if tmpNum >= 10 {
+				sum += (tmpNum % 10) + 1
+			} else {
+				sum += tmpNum
+			}
+		} else {
+			sum += tmpNum
+		}
+		shouldDouble = !shouldDouble
+	}
+
+	return sum%10 == 0
+}
+
+// IsISBN10 checks if the string is an ISBN version 10.
+func IsISBN10(str string) bool {
+	return IsISBN(str, 10)
+}
+
+// IsISBN13 checks if the string is an ISBN version 13.
+func IsISBN13(str string) bool {
+	return IsISBN(str, 13)
+}
+
+// IsISBN checks if the string is an ISBN (version 10 or 13).
+// If version value is not equal to 10 or 13, it will be checks both variants.
+func IsISBN(str string, version int) bool {
+	sanitized := whiteSpacesAndMinus.ReplaceAllString(str, "")
+	var checksum int32
+	var i int32
+	if version == 10 {
+		if !rxISBN10.MatchString(sanitized) {
+			return false
+		}
+		for i = 0; i < 9; i++ {
+			checksum += (i + 1) * int32(sanitized[i]-'0')
+		}
+		if sanitized[9] == 'X' {
+			checksum += 10 * 10
+		} else {
+			checksum += 10 * int32(sanitized[9]-'0')
+		}
+		if checksum%11 == 0 {
+			return true
+		}
+		return false
+	} else if version == 13 {
+		if !rxISBN13.MatchString(sanitized) {
+			return false
+		}
+		factor := []int32{1, 3}
+		for i = 0; i < 12; i++ {
+			checksum += factor[i%2] * int32(sanitized[i]-'0')
+		}
+		return (int32(sanitized[12]-'0'))-((10-(checksum%10))%10) == 0
+	}
+	return IsISBN(str, 10) || IsISBN(str, 13)
+}
+
+// IsJSON checks if the string is valid JSON (note: uses json.Unmarshal).
+func IsJSON(str string) bool {
+	var js json.RawMessage
+	return json.Unmarshal([]byte(str), &js) == nil
+}
+
+// IsMultibyte checks if the string contains one or more multibyte chars. Empty string is valid.
+func IsMultibyte(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxMultibyte.MatchString(str)
+}
+
+// IsASCII checks if the string contains ASCII chars only. Empty string is valid.
+func IsASCII(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxASCII.MatchString(str)
+}
+
+// IsPrintableASCII checks if the string contains printable ASCII chars only. Empty string is valid.
+func IsPrintableASCII(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxPrintableASCII.MatchString(str)
+}
+
+// IsFullWidth checks if the string contains any full-width chars. Empty string is valid.
+func IsFullWidth(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxFullWidth.MatchString(str)
+}
+
+// IsHalfWidth checks if the string contains any half-width chars. Empty string is valid.
+func IsHalfWidth(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxHalfWidth.MatchString(str)
+}
+
+// IsVariableWidth checks if the string contains a mixture of full and half-width chars. Empty string is valid.
+func IsVariableWidth(str string) bool {
+	if IsNull(str) {
+		return true
+	}
+	return rxHalfWidth.MatchString(str) && rxFullWidth.MatchString(str)
+}
+
+// IsBase64 checks if a string is base64 encoded.
+func IsBase64(str string) bool {
+	return rxBase64.MatchString(str)
+}
+
+// IsFilePath checks is a string is Win or Unix file path and returns it's type.
+func IsFilePath(str string) (bool, int) {
+	if rxWinPath.MatchString(str) {
+		//check windows path limit see:
+		//  http://msdn.microsoft.com/en-us/library/aa365247(VS.85).aspx#maxpath
+		if len(str[3:]) > 32767 {
+			return false, Win
+		}
+		return true, Win
+	} else if rxUnixPath.MatchString(str) {
+		return true, Unix
+	}
+	return false, Unknown
+}
+
+// IsDataURI checks if a string is base64 encoded data URI such as an image
+func IsDataURI(str string) bool {
+	dataURI := strings.Split(str, ",")
+	if !rxDataURI.MatchString(dataURI[0]) {
+		return false
+	}
+	return IsBase64(dataURI[1])
+}
+
+// IsMagnetURI checks if a string is valid magnet URI
+func IsMagnetURI(str string) bool {
+	return rxMagnetURI.MatchString(str)
+}
+
+// IsISO3166Alpha2 checks if a string is valid two-letter country code
+func IsISO3166Alpha2(str string) bool {
+	for _, entry := range ISO3166List {
+		if str == entry.Alpha2Code {
+			return true
+		}
+	}
+	return false
+}
+
+// IsISO3166Alpha3 checks if a string is valid three-letter country code
+func IsISO3166Alpha3(str string) bool {
+	for _, entry := range ISO3166List {
+		if str == entry.Alpha3Code {
+			return true
+		}
+	}
+	return false
+}
+
+// IsISO693Alpha2 checks if a string is valid two-letter language code
+func IsISO693Alpha2(str string) bool {
+	for _, entry := range ISO693List {
+		if str == entry.Alpha2Code {
+			return true
+		}
+	}
+	return false
+}
+
+// IsISO693Alpha3b checks if a string is valid three-letter language code
+func IsISO693Alpha3b(str string) bool {
+	for _, entry := range ISO693List {
+		if str == entry.Alpha3bCode {
+			return true
+		}
+	}
+	return false
+}
+
+// IsDNSName will validate the given string as a DNS name
+func IsDNSName(str string) bool {
+	if str == "" || len(strings.Replace(str, ".", "", -1)) > 255 {
+		// constraints already violated
+		return false
+	}
+	return !IsIP(str) && rxDNSName.MatchString(str)
+}
+
+// IsHash checks if a string is a hash of type algorithm.
+// Algorithm is one of ['md4', 'md5', 'sha1', 'sha256', 'sha384', 'sha512', 'ripemd128', 'ripemd160', 'tiger128', 'tiger160', 'tiger192', 'crc32', 'crc32b']
+func IsHash(str string, algorithm string) bool {
+	var len string
+	algo := strings.ToLower(algorithm)
+
+	if algo == "crc32" || algo == "crc32b" {
+		len = "8"
+	} else if algo == "md5" || algo == "md4" || algo == "ripemd128" || algo == "tiger128" {
+		len = "32"
+	} else if algo == "sha1" || algo == "ripemd160" || algo == "tiger160" {
+		len = "40"
+	} else if algo == "tiger192" {
+		len = "48"
+	} else if algo == "sha256" {
+		len = "64"
+	} else if algo == "sha384" {
+		len = "96"
+	} else if algo == "sha512" {
+		len = "128"
+	} else {
+		return false
+	}
+
+	return Matches(str, "^[a-f0-9]{"+len+"}$")
+}
+
+// IsSHA512 checks is a string is a SHA512 hash. Alias for `IsHash(str, "sha512")`
+func IsSHA512(str string) bool {
+	return IsHash(str, "sha512")
+}
+
+// IsSHA384 checks is a string is a SHA384 hash. Alias for `IsHash(str, "sha384")`
+func IsSHA384(str string) bool {
+	return IsHash(str, "sha384")
+}
+
+// IsSHA256 checks is a string is a SHA256 hash. Alias for `IsHash(str, "sha256")`
+func IsSHA256(str string) bool {
+	return IsHash(str, "sha256")
+}
+
+// IsTiger192 checks is a string is a Tiger192 hash. Alias for `IsHash(str, "tiger192")`
+func IsTiger192(str string) bool {
+	return IsHash(str, "tiger192")
+}
+
+// IsTiger160 checks is a string is a Tiger160 hash. Alias for `IsHash(str, "tiger160")`
+func IsTiger160(str string) bool {
+	return IsHash(str, "tiger160")
+}
+
+// IsRipeMD160 checks is a string is a RipeMD160 hash. Alias for `IsHash(str, "ripemd160")`
+func IsRipeMD160(str string) bool {
+	return IsHash(str, "ripemd160")
+}
+
+// IsSHA1 checks is a string is a SHA-1 hash. Alias for `IsHash(str, "sha1")`
+func IsSHA1(str string) bool {
+	return IsHash(str, "sha1")
+}
+
+// IsTiger128 checks is a string is a Tiger128 hash. Alias for `IsHash(str, "tiger128")`
+func IsTiger128(str string) bool {
+	return IsHash(str, "tiger128")
+}
+
+// IsRipeMD128 checks is a string is a RipeMD128 hash. Alias for `IsHash(str, "ripemd128")`
+func IsRipeMD128(str string) bool {
+	return IsHash(str, "ripemd128")
+}
+
+// IsCRC32 checks is a string is a CRC32 hash. Alias for `IsHash(str, "crc32")`
+func IsCRC32(str string) bool {
+	return IsHash(str, "crc32")
+}
+
+// IsCRC32b checks is a string is a CRC32b hash. Alias for `IsHash(str, "crc32b")`
+func IsCRC32b(str string) bool {
+	return IsHash(str, "crc32b")
+}
+
+// IsMD5 checks is a string is a MD5 hash. Alias for `IsHash(str, "md5")`
+func IsMD5(str string) bool {
+	return IsHash(str, "md5")
+}
+
+// IsMD4 checks is a string is a MD4 hash. Alias for `IsHash(str, "md4")`
+func IsMD4(str string) bool {
+	return IsHash(str, "md4")
+}
+
+// IsDialString validates the given string for usage with the various Dial() functions
+func IsDialString(str string) bool {
+	if h, p, err := net.SplitHostPort(str); err == nil && h != "" && p != "" && (IsDNSName(h) || IsIP(h)) && IsPort(p) {
+		return true
+	}
+
+	return false
+}
+
+// IsIP checks if a string is either IP version 4 or 6. Alias for `net.ParseIP`
+func IsIP(str string) bool {
+	return net.ParseIP(str) != nil
+}
+
+// IsPort checks if a string represents a valid port
+func IsPort(str string) bool {
+	if i, err := strconv.Atoi(str); err == nil && i > 0 && i < 65536 {
+		return true
+	}
+	return false
+}
+
+// IsIPv4 checks if the string is an IP version 4.
+func IsIPv4(str string) bool {
+	ip := net.ParseIP(str)
+	return ip != nil && strings.Contains(str, ".")
+}
+
+// IsIPv6 checks if the string is an IP version 6.
+func IsIPv6(str string) bool {
+	ip := net.ParseIP(str)
+	return ip != nil && strings.Contains(str, ":")
+}
+
+// IsCIDR checks if the string is an valid CIDR notiation (IPV4 & IPV6)
+func IsCIDR(str string) bool {
+	_, _, err := net.ParseCIDR(str)
+	return err == nil
+}
+
+// IsMAC checks if a string is valid MAC address.
+// Possible MAC formats:
+// 01:23:45:67:89:ab
+// 01:23:45:67:89:ab:cd:ef
+// 01-23-45-67-89-ab
+// 01-23-45-67-89-ab-cd-ef
+// 0123.4567.89ab
+// 0123.4567.89ab.cdef
+func IsMAC(str string) bool {
+	_, err := net.ParseMAC(str)
+	return err == nil
+}
+
+// IsHost checks if the string is a valid IP (both v4 and v6) or a valid DNS name
+func IsHost(str string) bool {
+	return IsIP(str) || IsDNSName(str)
+}
+
+// IsMongoID checks if the string is a valid hex-encoded representation of a MongoDB ObjectId.
+func IsMongoID(str string) bool {
+	return rxHexadecimal.MatchString(str) && (len(str) == 24)
+}
+
+// IsLatitude checks if a string is valid latitude.
+func IsLatitude(str string) bool {
+	return rxLatitude.MatchString(str)
+}
+
+// IsLongitude checks if a string is valid longitude.
+func IsLongitude(str string) bool {
+	return rxLongitude.MatchString(str)
+}
+
+// IsIMEI checks if a string is valid IMEI
+func IsIMEI(str string) bool {
+	return rxIMEI.MatchString(str)
+}
+
+// IsIMSI checks if a string is valid IMSI
+func IsIMSI(str string) bool {
+	if !rxIMSI.MatchString(str) {
+		return false
+	}
+
+	mcc, err := strconv.ParseInt(str[0:3], 10, 32)
+	if err != nil {
+		return false
+	}
+
+	switch mcc {
+	case 202, 204, 206, 208, 212, 213, 214, 216, 218, 219:
+	case 220, 221, 222, 226, 228, 230, 231, 232, 234, 235:
+	case 238, 240, 242, 244, 246, 247, 248, 250, 255, 257:
+	case 259, 260, 262, 266, 268, 270, 272, 274, 276, 278:
+	case 280, 282, 283, 284, 286, 288, 289, 290, 292, 293:
+	case 294, 295, 297, 302, 308, 310, 311, 312, 313, 314:
+	case 315, 316, 330, 332, 334, 338, 340, 342, 344, 346:
+	case 348, 350, 352, 354, 356, 358, 360, 362, 363, 364:
+	case 365, 366, 368, 370, 372, 374, 376, 400, 401, 402:
+	case 404, 405, 406, 410, 412, 413, 414, 415, 416, 417:
+	case 418, 419, 420, 421, 422, 424, 425, 426, 427, 428:
+	case 429, 430, 431, 432, 434, 436, 437, 438, 440, 441:
+	case 450, 452, 454, 455, 456, 457, 460, 461, 466, 467:
+	case 470, 472, 502, 505, 510, 514, 515, 520, 525, 528:
+	case 530, 536, 537, 539, 540, 541, 542, 543, 544, 545:
+	case 546, 547, 548, 549, 550, 551, 552, 553, 554, 555:
+	case 602, 603, 604, 605, 606, 607, 608, 609, 610, 611:
+	case 612, 613, 614, 615, 616, 617, 618, 619, 620, 621:
+	case 622, 623, 624, 625, 626, 627, 628, 629, 630, 631:
+	case 632, 633, 634, 635, 636, 637, 638, 639, 640, 641:
+	case 642, 643, 645, 646, 647, 648, 649, 650, 651, 652:
+	case 653, 654, 655, 657, 658, 659, 702, 704, 706, 708:
+	case 710, 712, 714, 716, 722, 724, 730, 732, 734, 736:
+	case 738, 740, 742, 744, 746, 748, 750, 995:
+		return true
+	default:
+		return false
+	}
+	return true
+}
+
+// IsRsaPublicKey checks if a string is valid public key with provided length
+func IsRsaPublicKey(str string, keylen int) bool {
+	bb := bytes.NewBufferString(str)
+	pemBytes, err := ioutil.ReadAll(bb)
+	if err != nil {
+		return false
+	}
+	block, _ := pem.Decode(pemBytes)
+	if block != nil && block.Type != "PUBLIC KEY" {
+		return false
+	}
+	var der []byte
+
+	if block != nil {
+		der = block.Bytes
+	} else {
+		der, err = base64.StdEncoding.DecodeString(str)
+		if err != nil {
+			return false
+		}
+	}
+
+	key, err := x509.ParsePKIXPublicKey(der)
+	if err != nil {
+		return false
+	}
+	pubkey, ok := key.(*rsa.PublicKey)
+	if !ok {
+		return false
+	}
+	bitlen := len(pubkey.N.Bytes()) * 8
+	return bitlen == int(keylen)
+}
+
+func toJSONName(tag string) string {
+	if tag == "" {
+		return ""
+	}
+
+	// JSON name always comes first. If there's no options then split[0] is
+	// JSON name, if JSON name is not set, then split[0] is an empty string.
+	split := strings.SplitN(tag, ",", 2)
+
+	name := split[0]
+
+	// However it is possible that the field is skipped when
+	// (de-)serializing from/to JSON, in which case assume that there is no
+	// tag name to use
+	if name == "-" {
+		return ""
+	}
+	return name
+}
+
+func prependPathToErrors(err error, path string) error {
+	switch err2 := err.(type) {
+	case Error:
+		err2.Path = append([]string{path}, err2.Path...)
+		return err2
+	case Errors:
+		errors := err2.Errors()
+		for i, err3 := range errors {
+			errors[i] = prependPathToErrors(err3, path)
+		}
+		return err2
+	}
+	return err
+}
+
+// ValidateMap use validation map for fields.
+// result will be equal to `false` if there are any errors.
+// s is the map containing the data to be validated.
+// m is the validation map in the form:
+//   map[string]interface{}{"name":"required,alpha","address":map[string]interface{}{"line1":"required,alphanum"}}
+func ValidateMap(s map[string]interface{}, m map[string]interface{}) (bool, error) {
+	if s == nil {
+		return true, nil
+	}
+	result := true
+	var err error
+	var errs Errors
+	var index int
+	val := reflect.ValueOf(s)
+	for key, value := range s {
+		presentResult := true
+		validator, ok := m[key]
+		if !ok {
+			presentResult = false
+			var err error
+			err = fmt.Errorf("all map keys has to be present in the validation map; got %s", key)
+			err = prependPathToErrors(err, key)
+			errs = append(errs, err)
+		}
+		valueField := reflect.ValueOf(value)
+		mapResult := true
+		typeResult := true
+		structResult := true
+		resultField := true
+		switch subValidator := validator.(type) {
+		case map[string]interface{}:
+			var err error
+			if v, ok := value.(map[string]interface{}); !ok {
+				mapResult = false
+				err = fmt.Errorf("map validator has to be for the map type only; got %s", valueField.Type().String())
+				err = prependPathToErrors(err, key)
+				errs = append(errs, err)
+			} else {
+				mapResult, err = ValidateMap(v, subValidator)
+				if err != nil {
+					mapResult = false
+					err = prependPathToErrors(err, key)
+					errs = append(errs, err)
+				}
+			}
+		case string:
+			if (valueField.Kind() == reflect.Struct ||
+				(valueField.Kind() == reflect.Ptr && valueField.Elem().Kind() == reflect.Struct)) &&
+				subValidator != "-" {
+				var err error
+				structResult, err = ValidateStruct(valueField.Interface())
+				if err != nil {
+					err = prependPathToErrors(err, key)
+					errs = append(errs, err)
+				}
+			}
+			resultField, err = typeCheck(valueField, reflect.StructField{
+				Name:      key,
+				PkgPath:   "",
+				Type:      val.Type(),
+				Tag:       reflect.StructTag(fmt.Sprintf("%s:%q", tagName, subValidator)),
+				Offset:    0,
+				Index:     []int{index},
+				Anonymous: false,
+			}, val, nil)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		case nil:
+			// already handlerd when checked before
+		default:
+			typeResult = false
+			err = fmt.Errorf("map validator has to be either map[string]interface{} or string; got %s", valueField.Type().String())
+			err = prependPathToErrors(err, key)
+			errs = append(errs, err)
+		}
+		result = result && presentResult && typeResult && resultField && structResult && mapResult
+		index++
+	}
+	// checks required keys
+	requiredResult := true
+	for key, value := range m {
+		if schema, ok := value.(string); ok {
+			tags := parseTagIntoMap(schema)
+			if required, ok := tags["required"]; ok {
+				if _, ok := s[key]; !ok {
+					requiredResult = false
+					if required.customErrorMessage != "" {
+						err = Error{key, fmt.Errorf(required.customErrorMessage), true, "required", []string{}}
+					} else {
+						err = Error{key, fmt.Errorf("required field missing"), false, "required", []string{}}
+					}
+					errs = append(errs, err)
+				}
+			}
+		}
+	}
+
+	if len(errs) > 0 {
+		err = errs
+	}
+	return result && requiredResult, err
+}
+
+// ValidateStruct use tags for fields.
+// result will be equal to `false` if there are any errors.
+// todo currently there is no guarantee that errors will be returned in predictable order (tests may to fail)
+func ValidateStruct(s interface{}) (bool, error) {
+	if s == nil {
+		return true, nil
+	}
+	result := true
+	var err error
+	val := reflect.ValueOf(s)
+	if val.Kind() == reflect.Interface || val.Kind() == reflect.Ptr {
+		val = val.Elem()
+	}
+	// we only accept structs
+	if val.Kind() != reflect.Struct {
+		return false, fmt.Errorf("function only accepts structs; got %s", val.Kind())
+	}
+	var errs Errors
+	for i := 0; i < val.NumField(); i++ {
+		valueField := val.Field(i)
+		typeField := val.Type().Field(i)
+		if typeField.PkgPath != "" {
+			continue // Private field
+		}
+		structResult := true
+		if valueField.Kind() == reflect.Interface {
+			valueField = valueField.Elem()
+		}
+		if (valueField.Kind() == reflect.Struct ||
+			(valueField.Kind() == reflect.Ptr && valueField.Elem().Kind() == reflect.Struct)) &&
+			typeField.Tag.Get(tagName) != "-" {
+			var err error
+			structResult, err = ValidateStruct(valueField.Interface())
+			if err != nil {
+				err = prependPathToErrors(err, typeField.Name)
+				errs = append(errs, err)
+			}
+		}
+		resultField, err2 := typeCheck(valueField, typeField, val, nil)
+		if err2 != nil {
+
+			// Replace structure name with JSON name if there is a tag on the variable
+			jsonTag := toJSONName(typeField.Tag.Get("json"))
+			if jsonTag != "" {
+				switch jsonError := err2.(type) {
+				case Error:
+					jsonError.Name = jsonTag
+					err2 = jsonError
+				case Errors:
+					for i2, err3 := range jsonError {
+						switch customErr := err3.(type) {
+						case Error:
+							customErr.Name = jsonTag
+							jsonError[i2] = customErr
+						}
+					}
+
+					err2 = jsonError
+				}
+			}
+
+			errs = append(errs, err2)
+		}
+		result = result && resultField && structResult
+	}
+	if len(errs) > 0 {
+		err = errs
+	}
+	return result, err
+}
+
+// ValidateStructAsync performs async validation of the struct and returns results through the channels
+func ValidateStructAsync(s interface{}) (<-chan bool, <-chan error) {
+	res := make(chan bool)
+	errors := make(chan error)
+
+	go func() {
+		defer close(res)
+		defer close(errors)
+
+		isValid, isFailed := ValidateStruct(s)
+
+		res <- isValid
+		errors <- isFailed
+	}()
+
+	return res, errors
+}
+
+// ValidateMapAsync performs async validation of the map and returns results through the channels
+func ValidateMapAsync(s map[string]interface{}, m map[string]interface{}) (<-chan bool, <-chan error) {
+	res := make(chan bool)
+	errors := make(chan error)
+
+	go func() {
+		defer close(res)
+		defer close(errors)
+
+		isValid, isFailed := ValidateMap(s, m)
+
+		res <- isValid
+		errors <- isFailed
+	}()
+
+	return res, errors
+}
+
+// parseTagIntoMap parses a struct tag `valid:required~Some error message,length(2|3)` into map[string]string{"required": "Some error message", "length(2|3)": ""}
+func parseTagIntoMap(tag string) tagOptionsMap {
+	optionsMap := make(tagOptionsMap)
+	options := strings.Split(tag, ",")
+
+	for i, option := range options {
+		option = strings.TrimSpace(option)
+
+		validationOptions := strings.Split(option, "~")
+		if !isValidTag(validationOptions[0]) {
+			continue
+		}
+		if len(validationOptions) == 2 {
+			optionsMap[validationOptions[0]] = tagOption{validationOptions[0], validationOptions[1], i}
+		} else {
+			optionsMap[validationOptions[0]] = tagOption{validationOptions[0], "", i}
+		}
+	}
+	return optionsMap
+}
+
+func isValidTag(s string) bool {
+	if s == "" {
+		return false
+	}
+	for _, c := range s {
+		switch {
+		case strings.ContainsRune("\\'\"!#$%&()*+-./:<=>?@[]^_{|}~ ", c):
+			// Backslash and quote chars are reserved, but
+			// otherwise any punctuation chars are allowed
+			// in a tag name.
+		default:
+			if !unicode.IsLetter(c) && !unicode.IsDigit(c) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+// IsSSN will validate the given string as a U.S. Social Security Number
+func IsSSN(str string) bool {
+	if str == "" || len(str) != 11 {
+		return false
+	}
+	return rxSSN.MatchString(str)
+}
+
+// IsSemver checks if string is valid semantic version
+func IsSemver(str string) bool {
+	return rxSemver.MatchString(str)
+}
+
+// IsType checks if interface is of some type
+func IsType(v interface{}, params ...string) bool {
+	if len(params) == 1 {
+		typ := params[0]
+		return strings.Replace(reflect.TypeOf(v).String(), " ", "", -1) == strings.Replace(typ, " ", "", -1)
+	}
+	return false
+}
+
+// IsTime checks if string is valid according to given format
+func IsTime(str string, format string) bool {
+	_, err := time.Parse(format, str)
+	return err == nil
+}
+
+// IsUnixTime checks if string is valid unix timestamp value
+func IsUnixTime(str string) bool {
+	if _, err := strconv.Atoi(str); err == nil {
+		return true
+	}
+	return false
+}
+
+// IsRFC3339 checks if string is valid timestamp value according to RFC3339
+func IsRFC3339(str string) bool {
+	return IsTime(str, time.RFC3339)
+}
+
+// IsRFC3339WithoutZone checks if string is valid timestamp value according to RFC3339 which excludes the timezone.
+func IsRFC3339WithoutZone(str string) bool {
+	return IsTime(str, rfc3339WithoutZone)
+}
+
+// IsISO4217 checks if string is valid ISO currency code
+func IsISO4217(str string) bool {
+	for _, currency := range ISO4217List {
+		if str == currency {
+			return true
+		}
+	}
+
+	return false
+}
+
+// ByteLength checks string's length
+func ByteLength(str string, params ...string) bool {
+	if len(params) == 2 {
+		min, _ := ToInt(params[0])
+		max, _ := ToInt(params[1])
+		return len(str) >= int(min) && len(str) <= int(max)
+	}
+
+	return false
+}
+
+// RuneLength checks string's length
+// Alias for StringLength
+func RuneLength(str string, params ...string) bool {
+	return StringLength(str, params...)
+}
+
+// IsRsaPub checks whether string is valid RSA key
+// Alias for IsRsaPublicKey
+func IsRsaPub(str string, params ...string) bool {
+	if len(params) == 1 {
+		len, _ := ToInt(params[0])
+		return IsRsaPublicKey(str, int(len))
+	}
+
+	return false
+}
+
+// StringMatches checks if a string matches a given pattern.
+func StringMatches(s string, params ...string) bool {
+	if len(params) == 1 {
+		pattern := params[0]
+		return Matches(s, pattern)
+	}
+	return false
+}
+
+// StringLength checks string's length (including multi byte strings)
+func StringLength(str string, params ...string) bool {
+
+	if len(params) == 2 {
+		strLength := utf8.RuneCountInString(str)
+		min, _ := ToInt(params[0])
+		max, _ := ToInt(params[1])
+		return strLength >= int(min) && strLength <= int(max)
+	}
+
+	return false
+}
+
+// MinStringLength checks string's minimum length (including multi byte strings)
+func MinStringLength(str string, params ...string) bool {
+
+	if len(params) == 1 {
+		strLength := utf8.RuneCountInString(str)
+		min, _ := ToInt(params[0])
+		return strLength >= int(min)
+	}
+
+	return false
+}
+
+// MaxStringLength checks string's maximum length (including multi byte strings)
+func MaxStringLength(str string, params ...string) bool {
+
+	if len(params) == 1 {
+		strLength := utf8.RuneCountInString(str)
+		max, _ := ToInt(params[0])
+		return strLength <= int(max)
+	}
+
+	return false
+}
+
+// Range checks string's length
+func Range(str string, params ...string) bool {
+	if len(params) == 2 {
+		value, _ := ToFloat(str)
+		min, _ := ToFloat(params[0])
+		max, _ := ToFloat(params[1])
+		return InRange(value, min, max)
+	}
+
+	return false
+}
+
+// IsInRaw checks if string is in list of allowed values
+func IsInRaw(str string, params ...string) bool {
+	if len(params) == 1 {
+		rawParams := params[0]
+
+		parsedParams := strings.Split(rawParams, "|")
+
+		return IsIn(str, parsedParams...)
+	}
+
+	return false
+}
+
+// IsIn checks if string str is a member of the set of strings params
+func IsIn(str string, params ...string) bool {
+	for _, param := range params {
+		if str == param {
+			return true
+		}
+	}
+
+	return false
+}
+
+func checkRequired(v reflect.Value, t reflect.StructField, options tagOptionsMap) (bool, error) {
+	if nilPtrAllowedByRequired {
+		k := v.Kind()
+		if (k == reflect.Ptr || k == reflect.Interface) && v.IsNil() {
+			return true, nil
+		}
+	}
+
+	if requiredOption, isRequired := options["required"]; isRequired {
+		if len(requiredOption.customErrorMessage) > 0 {
+			return false, Error{t.Name, fmt.Errorf(requiredOption.customErrorMessage), true, "required", []string{}}
+		}
+		return false, Error{t.Name, fmt.Errorf("non zero value required"), false, "required", []string{}}
+	} else if _, isOptional := options["optional"]; fieldsRequiredByDefault && !isOptional {
+		return false, Error{t.Name, fmt.Errorf("Missing required field"), false, "required", []string{}}
+	}
+	// not required and empty is valid
+	return true, nil
+}
+
+func typeCheck(v reflect.Value, t reflect.StructField, o reflect.Value, options tagOptionsMap) (isValid bool, resultErr error) {
+	if !v.IsValid() {
+		return false, nil
+	}
+
+	tag := t.Tag.Get(tagName)
+
+	// checks if the field should be ignored
+	switch tag {
+	case "":
+		if v.Kind() != reflect.Slice && v.Kind() != reflect.Map {
+			if !fieldsRequiredByDefault {
+				return true, nil
+			}
+			return false, Error{t.Name, fmt.Errorf("All fields are required to at least have one validation defined"), false, "required", []string{}}
+		}
+	case "-":
+		return true, nil
+	}
+
+	isRootType := false
+	if options == nil {
+		isRootType = true
+		options = parseTagIntoMap(tag)
+	}
+
+	if isEmptyValue(v) {
+		// an empty value is not validated, checks only required
+		isValid, resultErr = checkRequired(v, t, options)
+		for key := range options {
+			delete(options, key)
+		}
+		return isValid, resultErr
+	}
+
+	var customTypeErrors Errors
+	optionsOrder := options.orderedKeys()
+	for _, validatorName := range optionsOrder {
+		validatorStruct := options[validatorName]
+		if validatefunc, ok := CustomTypeTagMap.Get(validatorName); ok {
+			delete(options, validatorName)
+
+			if result := validatefunc(v.Interface(), o.Interface()); !result {
+				if len(validatorStruct.customErrorMessage) > 0 {
+					customTypeErrors = append(customTypeErrors, Error{Name: t.Name, Err: TruncatingErrorf(validatorStruct.customErrorMessage, fmt.Sprint(v), validatorName), CustomErrorMessageExists: true, Validator: stripParams(validatorName)})
+					continue
+				}
+				customTypeErrors = append(customTypeErrors, Error{Name: t.Name, Err: fmt.Errorf("%s does not validate as %s", fmt.Sprint(v), validatorName), CustomErrorMessageExists: false, Validator: stripParams(validatorName)})
+			}
+		}
+	}
+
+	if len(customTypeErrors.Errors()) > 0 {
+		return false, customTypeErrors
+	}
+
+	if isRootType {
+		// Ensure that we've checked the value by all specified validators before report that the value is valid
+		defer func() {
+			delete(options, "optional")
+			delete(options, "required")
+
+			if isValid && resultErr == nil && len(options) != 0 {
+				optionsOrder := options.orderedKeys()
+				for _, validator := range optionsOrder {
+					isValid = false
+					resultErr = Error{t.Name, fmt.Errorf(
+						"The following validator is invalid or can't be applied to the field: %q", validator), false, stripParams(validator), []string{}}
+					return
+				}
+			}
+		}()
+	}
+
+	for _, validatorSpec := range optionsOrder {
+		validatorStruct := options[validatorSpec]
+		var negate bool
+		validator := validatorSpec
+		customMsgExists := len(validatorStruct.customErrorMessage) > 0
+
+		// checks whether the tag looks like '!something' or 'something'
+		if validator[0] == '!' {
+			validator = validator[1:]
+			negate = true
+		}
+
+		// checks for interface param validators
+		for key, value := range InterfaceParamTagRegexMap {
+			ps := value.FindStringSubmatch(validator)
+			if len(ps) == 0 {
+				continue
+			}
+
+			validatefunc, ok := InterfaceParamTagMap[key]
+			if !ok {
+				continue
+			}
+
+			delete(options, validatorSpec)
+
+			field := fmt.Sprint(v)
+			if result := validatefunc(v.Interface(), ps[1:]...); (!result && !negate) || (result && negate) {
+				if customMsgExists {
+					return false, Error{t.Name, TruncatingErrorf(validatorStruct.customErrorMessage, field, validator), customMsgExists, stripParams(validatorSpec), []string{}}
+				}
+				if negate {
+					return false, Error{t.Name, fmt.Errorf("%s does validate as %s", field, validator), customMsgExists, stripParams(validatorSpec), []string{}}
+				}
+				return false, Error{t.Name, fmt.Errorf("%s does not validate as %s", field, validator), customMsgExists, stripParams(validatorSpec), []string{}}
+			}
+		}
+	}
+
+	switch v.Kind() {
+	case reflect.Bool,
+		reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
+		reflect.Float32, reflect.Float64,
+		reflect.String:
+		// for each tag option checks the map of validator functions
+		for _, validatorSpec := range optionsOrder {
+			validatorStruct := options[validatorSpec]
+			var negate bool
+			validator := validatorSpec
+			customMsgExists := len(validatorStruct.customErrorMessage) > 0
+
+			// checks whether the tag looks like '!something' or 'something'
+			if validator[0] == '!' {
+				validator = validator[1:]
+				negate = true
+			}
+
+			// checks for param validators
+			for key, value := range ParamTagRegexMap {
+				ps := value.FindStringSubmatch(validator)
+				if len(ps) == 0 {
+					continue
+				}
+
+				validatefunc, ok := ParamTagMap[key]
+				if !ok {
+					continue
+				}
+
+				delete(options, validatorSpec)
+
+				switch v.Kind() {
+				case reflect.String,
+					reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+					reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
+					reflect.Float32, reflect.Float64:
+
+					field := fmt.Sprint(v) // make value into string, then validate with regex
+					if result := validatefunc(field, ps[1:]...); (!result && !negate) || (result && negate) {
+						if customMsgExists {
+							return false, Error{t.Name, TruncatingErrorf(validatorStruct.customErrorMessage, field, validator), customMsgExists, stripParams(validatorSpec), []string{}}
+						}
+						if negate {
+							return false, Error{t.Name, fmt.Errorf("%s does validate as %s", field, validator), customMsgExists, stripParams(validatorSpec), []string{}}
+						}
+						return false, Error{t.Name, fmt.Errorf("%s does not validate as %s", field, validator), customMsgExists, stripParams(validatorSpec), []string{}}
+					}
+				default:
+					// type not yet supported, fail
+					return false, Error{t.Name, fmt.Errorf("Validator %s doesn't support kind %s", validator, v.Kind()), false, stripParams(validatorSpec), []string{}}
+				}
+			}
+
+			if validatefunc, ok := TagMap[validator]; ok {
+				delete(options, validatorSpec)
+
+				switch v.Kind() {
+				case reflect.String,
+					reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+					reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
+					reflect.Float32, reflect.Float64:
+					field := fmt.Sprint(v) // make value into string, then validate with regex
+					if result := validatefunc(field); !result && !negate || result && negate {
+						if customMsgExists {
+							return false, Error{t.Name, TruncatingErrorf(validatorStruct.customErrorMessage, field, validator), customMsgExists, stripParams(validatorSpec), []string{}}
+						}
+						if negate {
+							return false, Error{t.Name, fmt.Errorf("%s does validate as %s", field, validator), customMsgExists, stripParams(validatorSpec), []string{}}
+						}
+						return false, Error{t.Name, fmt.Errorf("%s does not validate as %s", field, validator), customMsgExists, stripParams(validatorSpec), []string{}}
+					}
+				default:
+					//Not Yet Supported Types (Fail here!)
+					err := fmt.Errorf("Validator %s doesn't support kind %s for value %v", validator, v.Kind(), v)
+					return false, Error{t.Name, err, false, stripParams(validatorSpec), []string{}}
+				}
+			}
+		}
+		return true, nil
+	case reflect.Map:
+		if v.Type().Key().Kind() != reflect.String {
+			return false, &UnsupportedTypeError{v.Type()}
+		}
+		var sv stringValues
+		sv = v.MapKeys()
+		sort.Sort(sv)
+		result := true
+		for i, k := range sv {
+			var resultItem bool
+			var err error
+			if v.MapIndex(k).Kind() != reflect.Struct {
+				resultItem, err = typeCheck(v.MapIndex(k), t, o, options)
+				if err != nil {
+					return false, err
+				}
+			} else {
+				resultItem, err = ValidateStruct(v.MapIndex(k).Interface())
+				if err != nil {
+					err = prependPathToErrors(err, t.Name+"."+sv[i].Interface().(string))
+					return false, err
+				}
+			}
+			result = result && resultItem
+		}
+		return result, nil
+	case reflect.Slice, reflect.Array:
+		result := true
+		for i := 0; i < v.Len(); i++ {
+			var resultItem bool
+			var err error
+			if v.Index(i).Kind() != reflect.Struct {
+				resultItem, err = typeCheck(v.Index(i), t, o, options)
+				if err != nil {
+					return false, err
+				}
+			} else {
+				resultItem, err = ValidateStruct(v.Index(i).Interface())
+				if err != nil {
+					err = prependPathToErrors(err, t.Name+"."+strconv.Itoa(i))
+					return false, err
+				}
+			}
+			result = result && resultItem
+		}
+		return result, nil
+	case reflect.Interface:
+		// If the value is an interface then encode its element
+		if v.IsNil() {
+			return true, nil
+		}
+		return ValidateStruct(v.Interface())
+	case reflect.Ptr:
+		// If the value is a pointer then checks its element
+		if v.IsNil() {
+			return true, nil
+		}
+		return typeCheck(v.Elem(), t, o, options)
+	case reflect.Struct:
+		return true, nil
+	default:
+		return false, &UnsupportedTypeError{v.Type()}
+	}
+}
+
+func stripParams(validatorString string) string {
+	return paramsRegexp.ReplaceAllString(validatorString, "")
+}
+
+// isEmptyValue checks whether value empty or not
+func isEmptyValue(v reflect.Value) bool {
+	switch v.Kind() {
+	case reflect.String, reflect.Array:
+		return v.Len() == 0
+	case reflect.Map, reflect.Slice:
+		return v.Len() == 0 || v.IsNil()
+	case reflect.Bool:
+		return !v.Bool()
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return v.Int() == 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return v.Uint() == 0
+	case reflect.Float32, reflect.Float64:
+		return v.Float() == 0
+	case reflect.Interface, reflect.Ptr:
+		return v.IsNil()
+	}
+
+	return reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface())
+}
+
+// ErrorByField returns error for specified field of the struct
+// validated by ValidateStruct or empty string if there are no errors
+// or this field doesn't exists or doesn't have any errors.
+func ErrorByField(e error, field string) string {
+	if e == nil {
+		return ""
+	}
+	return ErrorsByField(e)[field]
+}
+
+// ErrorsByField returns map of errors of the struct validated
+// by ValidateStruct or empty map if there are no errors.
+func ErrorsByField(e error) map[string]string {
+	m := make(map[string]string)
+	if e == nil {
+		return m
+	}
+	// prototype for ValidateStruct
+
+	switch e := e.(type) {
+	case Error:
+		m[e.Name] = e.Err.Error()
+	case Errors:
+		for _, item := range e.Errors() {
+			n := ErrorsByField(item)
+			for k, v := range n {
+				m[k] = v
+			}
+		}
+	}
+
+	return m
+}
+
+// Error returns string equivalent for reflect.Type
+func (e *UnsupportedTypeError) Error() string {
+	return "validator: unsupported type: " + e.Type.String()
+}
+
+func (sv stringValues) Len() int           { return len(sv) }
+func (sv stringValues) Swap(i, j int)      { sv[i], sv[j] = sv[j], sv[i] }
+func (sv stringValues) Less(i, j int) bool { return sv.get(i) < sv.get(j) }
+func (sv stringValues) get(i int) string   { return sv[i].String() }

udpcreateindex/biddingall.go

@@ -0,0 +1,328 @@
+package main
+
+import (
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"regexp"
+	"strings"
+	"sync"
+	util "utils"
+	"utils/mongodb"
+)
+
+func (t *TaskInfo) biddingAllTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	var mpool = make(chan bool, t.thread)
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	} else {
+		idMap := q["_id"].(map[string]interface{})
+		tmpQ := map[string]interface{}{}
+		for c, id := range idMap {
+			if idStr, ok := id.(string); ok && id != "" {
+				tmpQ[c] = mongodb.StringTOBsonId(idStr)
+			}
+		}
+		q["_id"] = tmpQ
+	}
+	//bidding库
+	biddingConn := biddingMgo.GetMgoConn()
+	defer biddingMgo.DestoryMongoConn(biddingConn)
+	//extract库
+	extractConn := extractMgo.GetMgoConn()
+	defer extractMgo.DestoryMongoConn(extractConn)
+	//连接信息
+	c, _ := mapInfo["coll"].(string)
+	if c == "" {
+		c, _ = bidding["collect"].(string)
+	} else {
+		currentColl = c
+	}
+
+	extractc, _ := extract["collect"].(string)
+	count, _ := biddingConn.DB(biddingMgo.DbName).C(c).Find(&q).Count()
+	//线程池
+	UpdatesLock := sync.Mutex{}
+
+	util.Debug("查询语句:", q, "同步总数:", count)
+	//查询招标数据
+	query := biddingConn.DB(biddingMgo.DbName).C(c).Find(q).Select(bson.M{
+		"projectinfo.attachment": 0,
+		"contenthtml":            0,
+		"publishdept":            0, // 6.30		迭代报错，字段值乱码
+	}).Sort("_id").Iter()
+	//查询抽取结果
+	extractResult := extractConn.DB(extractMgo.DbName).C(extractc).Find(q).Sort("_id").Iter()
+	n := 0
+	//对比两张表数据，减少查询次数
+	var compare map[string]interface{}
+	bnil := false
+	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+		if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
+			tmp = make(map[string]interface{})
+			continue
+		}
+		update := map[string]interface{}{}
+		del := map[string]interface{}{} //记录extract没有值而bidding中有值的字段
+		//对比方法----------------
+		for {
+			if compare == nil {
+				compare = make(map[string]interface{})
+				if !extractResult.Next(compare) {
+					break
+				}
+			}
+			if compare != nil {
+				//对比
+				cid := mongodb.BsonIdToSId(compare["_id"])
+				tid := mongodb.BsonIdToSId(tmp["_id"])
+				if cid == tid {
+					bnil = false
+					//更新bidding表，生成索引;bidding表modifyinfo中的字段不更新
+					modifyinfo := make(map[string]bool)
+					if tmpmodifyinfo, ok := tmp["modifyinfo"].(map[string]interface{}); ok && tmpmodifyinfo != nil {
+						for k, _ := range tmpmodifyinfo {
+							modifyinfo[k] = true
+						}
+					}
+					for _, k := range biddingMgoFields { //fields更新到mongo的字段
+						v1 := compare[k] //extract
+						v2 := tmp[k]     //bidding
+						if v2 == nil && v1 != nil {
+							update[k] = v1
+						} else if v2 != nil && v1 != nil && !modifyinfo[k] {
+							update[k] = v1
+						} else if v2 != nil && v1 == nil && !modifyinfo[k] { //
+							if k == "s_subscopeclass" && del["subscopeclass"] == nil {
+								continue
+							} else if k == "s_topscopeclass" && del["topscopeclass"] == nil {
+								continue
+							}
+							del[k] = 1
+							//qutil.Debug("抽取结果没有值,bidding有值：field--", k, "val--", v2)
+						}
+					}
+					if util.IntAll(compare["repeat"]) == 1 {
+						update["extracttype"] = -1
+					} else {
+						update["extracttype"] = 1
+					}
+					break
+				} else {
+					if cid < tid {
+						bnil = false
+						compare = nil
+						continue
+					} else {
+						bnil = true
+						break
+					}
+				}
+			} else {
+				bnil = false
+				break
+			}
+		}
+		//下面可以多线程跑的--->
+		//处理分类
+		mpool <- true
+		_id := tmp["_id"]
+		go func(tmp, update, compare, del map[string]interface{}, bnil bool) {
+			defer func() {
+				<-mpool
+			}()
+			if !bnil && compare != nil && len(compare) > 0 {
+				FieldMethod(compare, update)
+				compare = nil
+			} else {
+				area := util.ObjToString(tmp["area"])
+				city := util.ObjToString(tmp["city"])
+				district := util.ObjToString(tmp["district"])
+				UpdatesLock.Lock()
+				rdata := standardCheckCity(area, city, district)
+				UpdatesLock.Unlock()
+				if len(rdata) > 0 {
+					for k, v := range rdata {
+						update[k] = v
+					}
+				}
+			}
+			//------------------对比结束
+			//同时保存到elastic
+			for tk, tv := range update {
+				tmp[tk] = tv
+			}
+
+			if tmp["s_winner"] != "" {
+				cid := FieldFun(tmp)
+				if len(cid) > 0 {
+					tmp["entidlist"] = cid
+					update["entidlist"] = cid
+					tmp_up := []map[string]interface{}{}
+					tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
+					tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
+					updateExtractPool <- tmp_up
+				}
+			}
+
+			clearMap(tmp)
+			//go IS.Add("bidding")
+			UpdatesLock.Lock()
+			if util.IntAll(update["extracttype"]) != -1 {
+				newTmp := GetEsField(tmp, update, t.stype)
+				saveEsPool <- newTmp
+			}
+			if len(update) > 0 {
+				delete(update, "winnerorder") //winnerorder不需要更新到bindding表，删除
+				if len(del) > 0 {             //删除的数据
+					updateBiddingPool <- []map[string]interface{}{{
+						"_id": tmp["_id"],
+					},
+						{"$set": update, "$unset": del},
+					}
+				} else {
+					updateBiddingPool <- []map[string]interface{}{{
+						"_id": tmp["_id"],
+					},
+						{"$set": update},
+					}
+				}
+			}
+
+			UpdatesLock.Unlock()
+		}(tmp, update, compare, del, bnil)
+		if n%20000 == 0 {
+			util.Debug("current:", n, _id)
+		}
+		tmp = make(map[string]interface{})
+	}
+	util.Debug(mapInfo, "create bidding index...over", n)
+}
+
+//城市标准校验
+func standardCheckCity(area string, city string, district string) map[string]string {
+	rdata := make(map[string]string)
+	if area == "香港" || area == "澳门" || area == "台湾" || (area == "全国" && (city == "" && district == "")) {
+		return rdata
+	}
+	//第一步：区校验
+	if district != "" {
+		districtArr := DistrictDict[district]
+		if districtArr == nil { //涉及了 个别别名相关的数据
+			trim_arr := aliasDataDistrict(district) //拆分后缀
+			if len(trim_arr) > 0 {
+				for _, alias_district := range trim_arr {
+					alias_districtArr := DistrictDict[alias_district]
+					for _, v := range alias_districtArr {
+						if city == v.C_Name && area == v.P_Name {
+							rdata["district"] = alias_district
+							return rdata
+						}
+					}
+				}
+			}
+			rdata["district"] = ""
+		} else {
+			isTrue := false
+			for _, v := range districtArr {
+				if city == v.C_Name && area == v.P_Name {
+					isTrue = true
+					break
+				}
+			}
+			if isTrue { //完全匹配
+				return rdata
+			} else { //未完全匹配
+				if len(districtArr) == 1 {
+					rdata["area"] = districtArr[0].P_Name
+					rdata["city"] = districtArr[0].C_Name
+					rdata["district"] = districtArr[0].D_Name
+					return rdata
+				} else {
+					rdata["district"] = ""
+				}
+			}
+		}
+	}
+
+	//第二步：区校验-失败   市-校验
+	if city != "" {
+		cityArr := CityDict[city]
+		if cityArr == nil {
+			//把市当成区，匹配三级   - 存在优化空间- city:郑州  别名
+			districtArr := DistrictDict[city]
+			for _, v := range districtArr {
+				if city == v.C_Name && area == v.P_Name {
+					rdata["area"] = districtArr[0].P_Name
+					rdata["city"] = districtArr[0].C_Name
+					rdata["district"] = districtArr[0].D_Name
+					return rdata
+				}
+			}
+			rdata["city"] = ""
+		} else {
+			isTrue := false
+			for _, v := range cityArr {
+				if area == v.P_Name {
+					isTrue = true
+					break
+				}
+			}
+			if isTrue { //完全匹配
+				return rdata
+			} else { //未完全匹配
+				if len(cityArr) == 1 {
+					rdata["area"] = cityArr[0].P_Name
+					rdata["city"] = cityArr[0].C_Name
+					rdata["district"] = ""
+					return rdata
+				} else {
+					rdata["city"] = ""
+				}
+			}
+		}
+	}
+
+	//第三步：省份校验
+	if ProvinceDict[area] == nil {
+		rdata["area"] = "全国"
+		rdata["city"] = ""
+		rdata["district"] = ""
+	}
+
+	return rdata
+}
+
+var cityEndReg = regexp.MustCompile("(区|县|市)$")
+
+//拆分三级县
+func aliasDataDistrict(district string) []string {
+	arr := []string{}
+	if cityEndReg.MatchString(district) {
+		str := cityEndReg.FindString(district)
+		strings.TrimRight(district, str)
+		if str == "县" {
+			arr = append(arr, fmt.Sprintf("%s区", strings.TrimRight(district, str)))
+			arr = append(arr, fmt.Sprintf("%s市", strings.TrimRight(district, str)))
+		} else if str == "区" {
+			arr = append(arr, fmt.Sprintf("%s县", strings.TrimRight(district, str)))
+			arr = append(arr, fmt.Sprintf("%s市", strings.TrimRight(district, str)))
+		} else if str == "市" {
+			arr = append(arr, fmt.Sprintf("%s县", strings.TrimRight(district, str)))
+			arr = append(arr, fmt.Sprintf("%s区", strings.TrimRight(district, str)))
+		} else {
+
+		}
+	} else { //未找到 district- 区县市  例： district : 金水
+		arr = append(arr, fmt.Sprintf("%s区", district))
+		arr = append(arr, fmt.Sprintf("%s县", district))
+		arr = append(arr, fmt.Sprintf("%s市", district))
+	}
+	return arr
+}

udpcreateindex/biddingdata.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	util "utils"
+	"utils/mongodb"
+)
+
+func (t *TaskInfo) biddingDataTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	} else {
+		idMap := q["_id"].(map[string]interface{})
+		tmpQ := map[string]interface{}{}
+		for c, id := range idMap {
+			if idStr, ok := id.(string); ok && id != "" {
+				tmpQ[c] = mongodb.StringTOBsonId(idStr)
+			}
+		}
+		q["_id"] = tmpQ
+	}
+	//bidding库
+	biddingConn := biddingMgo.GetMgoConn()
+	defer biddingMgo.DestoryMongoConn(biddingConn)
+	//连接信息
+	c, _ := mapInfo["coll"].(string)
+	if c == "" {
+		c, _ = bidding["collect"].(string)
+	} else {
+		currentColl = c
+	}
+	count, _ := biddingConn.DB(biddingMgo.DbName).C(c).Find(&q).Count()
+	util.Debug("查询语句:", q, "同步总数:", count, "elastic库:")
+	//查询招标数据
+	query := biddingConn.DB(biddingMgo.DbName).C(c).Find(q).Select(bson.M{
+		"projectname": 1,
+		"title":       1,
+		"site":        1,
+		"href":        1,
+		"publishtime": 1,
+		"spidercode":  1,
+		"extracttype": 1,
+	}).Sort("_id").Iter()
+	n := 0
+	//更新数组
+	var mpool = make(chan bool, t.thread)
+	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+		if n%20000 == 0 {
+			util.Debug("current:", n, tmp["_id"])
+		}
+		mpool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-mpool
+			}()
+			newTmp := GetEsField(tmp, nil, t.stype)
+			newTmp["extracttype"] = util.IntAll(tmp["extracttype"])
+			saveEsAllPool <- newTmp
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	util.Debug(mapInfo, "create biddingdata index...over", n)
+}

udpcreateindex/biddingdeletebyextract.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	"log"
+	util "utils"
+	"utils/elastic"
+	"utils/mongodb"
+)
+
+//根据抽取表repeat=1，删除es中重复数据
+func biddingDelByExtract(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+				// "$gte": mongodb.StringTOBsonId("5eb18d86511b1203376bd742"),
+				// "$lte": mongodb.StringTOBsonId("5ebb43c5f2c1a7850bb1337c"),
+			},
+			"repeat": 1,
+		}
+	}
+	extractc, _ := extract["collect"].(string)
+	index, _ := bidding["index"].(string)
+	itype, _ := bidding["type"].(string)
+
+	//extract
+	extractConn := extractMgo.GetMgoConn()
+	defer extractMgo.DestoryMongoConn(extractConn)
+	//查询数据
+	count, _ := extractConn.DB(extractMgo.DbName).C(extractc).Find(&q).Count()
+	log.Println("查询语句:", q, "删除同步总数:", count, "elastic库:", index)
+	extractquery := extractConn.DB(extractMgo.DbName).C(extractc).Find(q).Select(
+		bson.M{"_id": 1},
+	).Sort("_id").Iter()
+	log.Println("开始迭代...")
+	i := 0
+	var n int
+	var dnum int
+	for tmp := make(map[string]interface{}); extractquery.Next(tmp); i = i + 1 {
+		n++
+		_id := mongodb.BsonIdToSId(tmp["_id"])
+		if elastic.DelById(index, itype, _id) { //删除
+			dnum++
+		}
+		//if other_index != "" && other_itype != "" {
+		//	bidding_other_es.DelById(other_index, other_itype, _id)
+		//}
+
+		if n%200 == 0 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	log.Println("共删除：", dnum)
+}

udpcreateindex/biddingdeletebyextracttype.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	"log"
+	util "utils"
+	"utils/mongodb"
+)
+
+//根据bidding表extracttype=-1，删除es中重复数据
+func biddingDelByExtracttype(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+				// "$gte": mongodb.StringTOBsonId("5eb18d86511b1203376bd742"),
+				// "$lte": mongodb.StringTOBsonId("5ebb43c5f2c1a7850bb1337c"),
+			},
+			"extracttype": -1,
+		}
+	}
+	c, _ := bidding["collect"].(string)
+
+	//bidding
+	conn := biddingMgo.GetMgoConn()
+	defer biddingMgo.DestoryMongoConn(conn)
+	//查询数据
+	//count, _ := session.DB(db).C(c).Find(&q).Count()
+	//log.Println("查询语句:", q, "删除同步总数:", count, "elastic库:", index)
+	biddingquery := conn.DB(biddingMgo.DbName).C(c).Find(q).Select(
+		bson.M{"_id": 1},
+	).Sort("_id").Iter()
+	log.Println("开始迭代...")
+	i := 0
+	var n int
+	var dnum int
+	for tmp := make(map[string]interface{}); biddingquery.Next(tmp); i = i + 1 {
+		n++
+		_id := mongodb.BsonIdToSId(tmp["_id"])
+		if Es2.DelById(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), _id) { //删除
+			//Es2.DelById(qutil.ObjToString(biddingIndex["idnex"]), qutil.ObjToString(biddingIndex["type"]), _id)
+			dnum++
+		}
+		if n%200 == 0 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	log.Println("共删除：", dnum)
+}

udpcreateindex/biddingindex.go

@@ -0,0 +1,502 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+	util "utils"
+	"utils/mfw"
+	"utils/mongodb"
+	"utils/udp"
+)
+
+//招标数据表和抽取表一一对应开始更新
+
+func (t *TaskInfo) biddingTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+
+	q, _ := mapInfo["query"].(map[string]interface{})
+	bkey, _ := mapInfo["bkey"].(string)
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": map[string]interface{}{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	//extract库
+	extractConn := extractMgo.GetMgoConn()
+	defer extractMgo.DestoryMongoConn(extractConn)
+	extractc, _ := extract["collect"].(string)
+	extractResult := extractConn.DB(extractMgo.DbName).C(extractc).Find(q).Sort("_id").Iter()
+	eMap := map[string]map[string]interface{}{}
+	extCount, repeatCount := 0, 0
+	for tmp := make(map[string]interface{}); extractResult.Next(tmp); extCount++ {
+		if util.IntAll(tmp["repeat"]) == 1 {
+			repeatCount++
+		}
+		tid := mongodb.BsonIdToSId(tmp["_id"])
+		eMap[tid] = tmp
+		tmp = make(map[string]interface{})
+	}
+	util.Debug("抽取表 重复数据量：", extCount, repeatCount)
+
+	//bidding库
+	biddingConn := biddingMgo.GetMgoConn()
+	c, _ := bidding["collect"].(string)
+	count, _ := biddingConn.DB(biddingMgo.DbName).C(c).Find(&q).Count()
+	util.Debug("查询语句:", q, "同步总数:", count)
+	n1, n2 := 0, 0
+	if count < 200000 {
+		var res []map[string]interface{}
+		result := biddingConn.DB(biddingMgo.DbName).C(c).Find(q).Select(map[string]interface{}{
+			"contenthtml": 0,
+		}).Iter()
+		for tmp := make(map[string]interface{}); result.Next(tmp); {
+			res = append(res, tmp)
+			tmp = make(map[string]interface{})
+		}
+		biddingMgo.DestoryMongoConn(biddingConn)
+		util.Debug("查询结果", "bidding:", count, "抽取:", extCount)
+		//if int64(len(res)) != count {
+		//	time.Sleep(20 * time.Second)
+		//	toadd := &net.UDPAddr{
+		//		IP:   net.ParseIP("127.0.0.1"),
+		//		Port: util.IntAll(Sysconfig["udpport"]),
+		//	}
+		//	udpclient.WriteUdp(data, mu.OP_TYPE_DATA, toadd)
+		//}
+		n1, n2 = t.doIndex(res, eMap, bkey)
+	} else {
+		util.Debug("数据量太大，放弃！", count)
+		biddingMgo.DestoryMongoConn(biddingConn)
+	}
+	util.Debug(mapInfo, "create bidding index...over", "all:", count, "bidding size:", n1, "，es size:", n2)
+	if t.stype == "bidding_history" {
+		// 历史判重id段结束之后 生全量数据索引
+		t.stype = "biddingdata"
+		t.thread = 30
+		t.biddingDataTask(data, mapInfo)
+	}
+}
+
+func (t *TaskInfo) doIndex(infos []map[string]interface{}, eMap map[string]map[string]interface{}, bkey string) (int, int) {
+	n1, n2 := 0, 0 //bidding数量，索引数量
+	//对比两张表数据，减少查询次数
+	var compare map[string]interface{}
+	util.Debug("start ...")
+	for n, tmp := range infos {
+		n1++
+		if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
+			tmp = make(map[string]interface{})
+			continue
+		}
+		tid := mongodb.BsonIdToSId(tmp["_id"])
+		update := map[string]interface{}{} //要更新的mongo数据
+		//对比方法----------------
+		if eMap[tid] != nil {
+			compare = eMap[tid]
+			if t.stype == "bidding" {
+				// 增量id段 正常数据
+				if num := util.IntAll(compare["dataging"]); num == 1 { //extract中dataging=1跳过
+					tmp = make(map[string]interface{})
+					compare = nil
+					continue
+				}
+				delete(eMap, tid)
+			}
+			if t.stype == "bidding_history" {
+				//增量id段 历史数据
+				if compare["history_updatetime"] == nil { //extract中history_updatetime不存在跳过
+					tmp = make(map[string]interface{})
+					compare = nil
+					continue
+				}
+				delete(eMap, tid)
+			}
+			//更新bidding表，生成索引;bidding表modifyinfo中的字段不更新
+			modifyinfo := make(map[string]bool)
+			if tmpmodifyinfo, ok := tmp["modifyinfo"].(map[string]interface{}); ok && tmpmodifyinfo != nil {
+				for k, _ := range tmpmodifyinfo {
+					modifyinfo[k] = true
+				}
+			}
+			//更新bidding表，生成索引
+			for _, k := range biddingMgoFields {
+				v1 := compare[k] //extract
+				v2 := tmp[k]     //bidding
+				if v2 == nil && v1 != nil && !modifyinfo[k] {
+					update[k] = v1
+				} else if v2 != nil && v1 != nil && !modifyinfo[k] {
+					//update[k+"_b"] = v2
+					update[k] = v1
+				} else if v2 != nil && v1 == nil {
+					//update[k+"_b"] = v2
+					if k == "area" || k == "city" || k == "district" {
+						update[k] = ""
+					}
+				}
+			}
+			if util.IntAll(compare["repeat"]) == 1 {
+				update["extracttype"] = -1
+			} else {
+				update["extracttype"] = 1
+			}
+		} else {
+			compare = nil
+			if util.IntAll(tmp["dataging"]) == 1 { //修改未抽取的bidding数据的dataging
+				update["dataging"] = 0
+			}
+		}
+		//下面可以多线程跑的--->
+		//处理分类
+		if compare != nil { //extract
+			FieldMethod(compare, update)
+			compare = nil
+		} else {
+			area := util.ObjToString(tmp["area"])
+			city := util.ObjToString(tmp["city"])
+			district := util.ObjToString(tmp["district"])
+			rdata := standardCheckCity(area, city, district)
+			if len(rdata) > 0 {
+				for k, v := range rdata {
+					update[k] = v
+				}
+			}
+		}
+		//------------------对比结束
+
+		//处理key descript
+		if bkey == "" {
+			DealInfo(&tmp, &update)
+		}
+		//同时保存到elastic
+		for tk, tv := range update {
+			tmp[tk] = tv
+		}
+
+		extractMap := make(map[string]interface{})
+		if tmp["s_winner"] != "" {
+			cid := FieldFun(tmp)
+			if len(cid) > 0 {
+				tmp["entidlist"] = cid
+				update["entidlist"] = cid
+				extractMap["entidlist"] = cid
+			}
+		}
+		// 6.10 剑鱼发布信息分类处理, 写在这里是为了修改抽取表
+		TypeMethod(tmp, update, extractMap)
+		if len(extractMap) > 0 {
+			if extractMap["toptype"] != nil && extractMap["subtype"] == nil {
+				updateExtractPool <- []map[string]interface{}{
+					{"_id": tmp["_id"]},
+					{"$set": extractMap, "$unset": map[string]interface{}{"subtype": ""}},
+				}
+			} else {
+				updateExtractPool <- []map[string]interface{}{
+					{"_id": tmp["_id"]},
+					{"$set": extractMap},
+				}
+			}
+		}
+
+		// 附件有效字段
+		if i := validFile(tmp); i != 0 {
+			if i == -1 {
+				tmp["isValidFile"] = false
+				update["isValidFile"] = false
+			} else {
+				tmp["isValidFile"] = true
+				update["isValidFile"] = true
+			}
+		}
+
+		clearMap(tmp)
+		//go IS.Add("bidding")
+		if util.IntAll(update["extracttype"]) != -1 {
+			n2++
+			newTmp := GetEsField(tmp, update, t.stype)
+			newTmp["dataweight"] = 0 //索引数据新增 jy置顶字段
+			if util.ObjToString(newTmp["spidercode"]) == "a_jyxxfbpt_gg" {
+				// 剑鱼信息发布数据 通过udp通知信息发布程序
+				go UdpMethod(mongodb.BsonIdToSId(newTmp["_id"]))
+			}
+			saveEsPool <- newTmp
+
+		}
+		if len(update) > 0 {
+			delete(update, "winnerorder") //winnerorder不需要更新到bindding表，删除
+			updateBiddingPool <- []map[string]interface{}{{
+				"_id": tmp["_id"],
+			},
+				{"$set": update},
+			}
+		}
+		if n%200 == 0 {
+			util.Debug("current:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	return n1, n2
+}
+
+var client *mfw.Client
+var reg = regexp.MustCompile("^[0-9a-zA-Z-.]+$")
+var reg_space = regexp.MustCompile("(?ism)(<style.*?>.*?</style>)|([.#]?\\w{1,20}\\{.*?\\})|(<.*?>)|(\\\\t)+|\\t|( +)|(　+)|(" + string(rune(160)) + "+)")
+var reg_row = regexp.MustCompile("(?i)<(tr|div|p)[^>]*?>|(\\n)+")
+var reg_dh = regexp.MustCompile("[,]+")
+var reg_newdb = regexp.MustCompile("([:,、：，。.;])[,]")
+var reg_no = regexp.MustCompile("^[0-9]*$")
+var reg_letter = regexp.MustCompile("[a-z]*")
+var MSG_SERVER = "123.56.236.148:7070"
+var DesLen = 120
+
+func inits() {
+	ser := util.ObjToString(Sysconfig["msg_server"])
+	if ser != "" {
+		MSG_SERVER = ser
+	}
+	cf := &mfw.ClientConfig{
+		ClientName:      "剑鱼抽关键词",
+		EventHandler:    func(p *mfw.Packet) {},
+		MsgServerAddr:   MSG_SERVER,
+		CanHandleEvents: []int{},
+		OnConnectSuccess: func() {
+			util.Debug("剑鱼关键词 client")
+		},
+		ReadBufferSize:  10,
+		WriteBufferSize: 10,
+	}
+	client, _ = mfw.NewClient(cf)
+
+}
+
+var keypool = make(chan bool, 1)
+
+func DealInfo(obj, update *map[string]interface{}) {
+	defer util.Catch()
+	if (*obj)["keywords"] != nil && (*obj)["description"] != nil {
+		return
+	} else {
+		(*update)["keywords"] = ""
+		(*update)["description"] = ""
+	}
+	title := util.ObjToString((*obj)["title"])
+	var m [][]string
+	select {
+	case <-func() <-chan bool {
+		ch := make(chan bool, 1)
+		go func(chan bool) {
+			select {
+			case keypool <- true:
+				defer func() {
+					<-keypool
+				}()
+				ret, _ := client.Call("", mfw.UUID(8), 4010, mfw.SENDTO_TYPE_RAND_RECIVER, title, 1)
+				json.Unmarshal(ret, &m)
+			case <-time.After(10 * time.Millisecond):
+			}
+			ch <- true
+		}(ch)
+		return ch
+	}():
+	case <-time.After(40 * time.Millisecond):
+	}
+	arr := []string{}
+	keyword := []string{}
+	keywordnew := []string{}
+	for _, tmp := range m {
+		if reg.MatchString(tmp[0]) {
+			arr = append(arr, tmp[0])
+		} else {
+			if len(arr) > 0 {
+				str := strings.Join(arr, "")
+				keyword = append(keyword, str)
+				arr = []string{}
+			}
+			if len(tmp[0]) > 3 && (strings.HasPrefix(tmp[1], "n") || tmp[1] == "v" || tmp[1] == "vn" || strings.HasPrefix(tmp[1], "g")) {
+				keyword = append(keyword, tmp[0])
+			}
+		}
+	}
+	for _, v := range keyword {
+		v = reg_no.ReplaceAllString(v, "")
+		if len(v) > 0 {
+			keywordnew = append(keywordnew, v)
+		}
+	}
+	keywords := strings.Join(keywordnew, ",")
+	(*update)["keywords"] = keywords
+	content := ""
+	if (*obj)["detail_bak"] != nil {
+		content = util.ObjToString((*obj)["detail_bak"])
+	} else {
+		content = util.ObjToString((*obj)["detail"])
+	}
+	//内容替换
+	content = strings.Replace(content, " ", "", -1)
+	content = reg_space.ReplaceAllString(content, "")
+	content = reg_row.ReplaceAllString(content, ",")
+	content = reg_dh.ReplaceAllString(content, ",")
+	content = reg_newdb.ReplaceAllString(content, "$1")
+	if strings.HasPrefix(content, ",") {
+		content = content[1:]
+	}
+	tc := []rune(content)
+	ltc := len(tc)
+	description := content
+	if ltc > DesLen {
+		description = string(tc[:DesLen])
+	}
+	(*update)["description"] = description
+	//保存到数据库
+	return
+}
+
+// @Description tmp修改索引，update 修改bidding表，extractM修改抽取表
+// @Author J 2022/6/10 10:29 AM
+func TypeMethod(tmp, update, extractM map[string]interface{}) {
+	if jyData, ok := tmp["jyfb_data"].(map[string]interface{}); ok {
+		if t := util.ObjToString(jyData["type"]); t != "" {
+			switch t {
+			//case "采购信息":
+			case "招标公告":
+				if util.ObjToString(tmp["toptype"]) != "招标" {
+					tmp["toptype"] = "招标"
+					update["toptype"] = "招标"
+					extractM["toptype"] = "招标"
+					delete(tmp, "subtype")
+					delete(update, "subtype")
+				}
+			case "采购意向":
+				if util.ObjToString(tmp["toptype"]) != "采购意向" {
+					tmp["toptype"] = "采购意向"
+					tmp["subtype"] = "采购意向"
+					update["toptype"] = "采购意向"
+					update["subtype"] = "采购意向"
+					extractM["toptype"] = "采购意向"
+					extractM["subtype"] = "采购意向"
+				}
+			case "招标预告":
+				if util.ObjToString(tmp["toptype"]) != "预告" {
+					tmp["toptype"] = "预告"
+					update["toptype"] = "预告"
+					extractM["toptype"] = "预告"
+					delete(tmp, "subtype")
+					delete(update, "subtype")
+				}
+			case "招标结果":
+				if util.ObjToString(tmp["toptype"]) != "结果" {
+					tmp["toptype"] = "结果"
+					update["toptype"] = "结果"
+					extractM["toptype"] = "结果"
+					delete(tmp, "subtype")
+					delete(update, "subtype")
+				}
+			}
+		}
+	}
+}
+
+// @Description rpc调用信息发布程序接口
+// @Author J 2022/4/13 9:13 AM
+func UdpMethod(id string) {
+	mapinfo := map[string]interface{}{
+		"infoid": id,
+		"stype":  "jyfb_data_over",
+	}
+	datas, _ := json.Marshal(mapinfo)
+	util.Debug(JyUdpAddr, string(datas))
+	_ = udpclient.WriteUdp(datas, udp.OP_TYPE_DATA, JyUdpAddr)
+}
+
+// @Description id不变，内容变化 重新索引数据
+// @Author J 2022/8/10 13:29
+func taskinfo(id string) {
+	tmp, _ := biddingMgo.FindById("bidding", id, nil)
+	if tmp == nil || len(*tmp) == 0 {
+		util.Debug(fmt.Sprintf("taskinfo bidding id=%s 未查询到数据", id))
+		return
+	}
+	extractM, _ := extractMgo.FindById(util.ObjToString(extract["collect"]), id, nil)
+	if extractM == nil || len(*extractM) == 0 {
+		extractM, _ = extractMgo.FindById(util.ObjToString(extract["collect1"]), id, nil)
+		if extractM == nil || len(*extractM) == 0 {
+			util.Debug(fmt.Sprintf("taskinfo extract id=%s 未查询到数据", id))
+			return
+		}
+	}
+	update := map[string]interface{}{} //要更新的mongo数据
+	//更新bidding表字段
+	for _, k := range biddingMgoFields {
+		v1 := (*extractM)[k] //extract
+		v2 := (*tmp)[k]      //bidding
+		if v2 == nil && v1 != nil {
+			update[k] = v1
+		} else if v2 != nil && v1 != nil {
+			update[k] = v1
+		} else if v2 != nil && v1 == nil {
+			if k == "area" || k == "city" || k == "district" {
+				update[k] = ""
+			}
+		}
+	}
+	if util.IntAll((*extractM)["repeat"]) == 1 {
+		update["extracttype"] = -1
+	} else {
+		update["extracttype"] = 1
+	}
+	if util.IntAll((*tmp)["dataging"]) == 1 { //修改未抽取的bidding数据的dataging
+		update["dataging"] = 0
+	}
+	//处理分类
+	FieldMethod(*extractM, update)
+
+	//同时保存到elastic
+	for tk, tv := range update {
+		(*tmp)[tk] = tv
+	}
+
+	extractMap := make(map[string]interface{})
+	if util.ObjToString((*tmp)["s_winner"]) != "" {
+		cid := FieldFun(*tmp)
+		if len(cid) > 0 {
+			(*tmp)["entidlist"] = cid
+			update["entidlist"] = cid
+			extractMap["entidlist"] = cid
+		}
+		updateExtractPool <- []map[string]interface{}{
+			{"_id": mongodb.StringTOBsonId(id)},
+			{"$set": extractMap},
+		}
+	}
+
+	// 附件有效字段
+	if i := validFile(*tmp); i != 0 {
+		if i == -1 {
+			(*tmp)["isValidFile"] = false
+			update["isValidFile"] = false
+		} else {
+			(*tmp)["isValidFile"] = true
+			update["isValidFile"] = true
+		}
+	}
+	clearMap(*tmp)
+	if util.IntAll(update["extracttype"]) != -1 {
+		newTmp := GetEsField(*tmp, update, "")
+		newTmp["dataweight"] = 0 //索引数据新增 jy置顶字段
+		saveEsPool <- newTmp
+
+	}
+	if len(update) > 0 {
+		delete(update, "winnerorder") //winnerorder不需要更新到bindding表，删除
+		updateBiddingPool <- []map[string]interface{}{{
+			"_id": mongodb.StringTOBsonId(id),
+		},
+			{"$set": update},
+		}
+	}
+}

udpcreateindex/biddingindexback.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	util "utils"
+	"utils/mongodb"
+)
+
+func (t *TaskInfo) biddingBackTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	//bidding库
+	biddingConn := biddingMgo.GetMgoConn()
+	defer biddingMgo.DestoryMongoConn(biddingConn)
+	c := util.ObjToString(mapInfo["coll"])
+	if c == "" {
+		c = bidding["collect"].(string)
+	} else {
+		currentColl = c
+	}
+	count, _ := biddingConn.DB(biddingMgo.DbName).C(c).Find(&q).Count()
+	util.Debug("查询语句:", q, "同步总数:", count)
+	//查询招标数据
+	query := biddingConn.DB(biddingMgo.DbName).C(c).Find(q).Select(bson.M{
+		"projectinfo.attachment": 0,
+		"contenthtml":            0,
+		"publishdept":            0,
+	}).Sort("_id").Iter()
+	//查询抽取结果
+	n := 0
+
+	var mpool = make(chan bool, t.thread)
+	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+		if n%20000 == 0 {
+			util.Debug("current:", n, tmp["_id"])
+		}
+		if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
+			tmp = make(map[string]interface{})
+			continue
+		}
+		if util.IntAll(tmp["extracttype"]) == -1 {
+			tmp = make(map[string]interface{})
+			continue
+		}
+		mpool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-mpool
+			}()
+			newTmp := GetEsField(tmp, nil, t.stype)
+			saveEsElsePool <- newTmp
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	util.Debug(mapInfo, "create biddingback index...over", n)
+}

udpcreateindex/biddingmerge.go

@@ -0,0 +1,224 @@
+package main
+
+import (
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"log"
+	"strings"
+	"sync"
+	util "utils"
+	"utils/mongodb"
+)
+
+//对字段处理 bidamount  budget
+//招标数据表和抽取表一一对应开始更新
+
+func biddingMergeTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	thread := 40
+	var mpool = make(chan bool, thread)
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	//bidding库
+	session := biddingMgo.GetMgoConn()
+	defer biddingMgo.DestoryMongoConn(session)
+	//extract库
+	extractsession := extractMgo.GetMgoConn()
+	defer extractMgo.DestoryMongoConn(extractsession)
+	//连接信息
+	c, _ := mapInfo["coll"].(string)
+	if c == "" {
+		c, _ = bidding["collect"].(string)
+	}
+
+	extractc, _ := extract["collect"].(string)
+	count, _ := session.DB(biddingMgo.DbName).C(c).Find(&q).Count()
+	//线程池
+	UpdatesLock := sync.Mutex{}
+
+	log.Println("查询语句:", q, "同步总数:", count)
+	//查询招标数据
+	query := session.DB(biddingMgo.DbName).C(c).Find(q).Select(bson.M{
+		"projectinfo.attachment": 0,
+		"contenthtml":            0,
+	}).Sort("_id").Iter()
+	//查询抽取结果
+	extractquery := extractsession.DB(extractMgo.DbName).C(extractc).Find(q).Sort("_id").Iter()
+
+	n := 0
+
+	//更新数组
+	arr := [][]map[string]interface{}{}
+	//对比两张表数据，减少查询次数
+	var compare bson.M
+	bnil := false
+	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+		update := map[string]interface{}{}
+		//对比方法----------------
+		for {
+			if compare == nil {
+				compare = make(bson.M)
+				if !extractquery.Next(compare) {
+					break
+				}
+			}
+			if compare != nil {
+				//对比
+				cid := mongodb.BsonIdToSId(compare["_id"])
+				tid := mongodb.BsonIdToSId(tmp["_id"])
+				if cid == tid {
+					bnil = false
+					//更新bidding表，生成索引
+					for _, k := range biddingMgoFields {
+						v1 := compare[k]
+						v2 := tmp[k]
+						if v2 == nil && v1 != nil {
+							update[k] = v1
+						} else if v2 != nil && v1 != nil {
+							//update[k+"_b"] = v2
+							update[k] = v1
+						} else if v2 != nil && v1 == nil {
+							//update[k+"_b"] = v2
+						}
+					}
+					if util.IntAll(compare["repeat"]) == 1 {
+						update["extracttype"] = -1
+						//} else if qutil.IntAll(tmp["extracttype"]) == -1 {
+					} else {
+						update["extracttype"] = 1
+					}
+					break
+				} else {
+					if cid < tid {
+						bnil = false
+						compare = nil
+						continue
+					} else {
+						bnil = true
+						break
+					}
+				}
+			} else {
+				bnil = false
+				break
+			}
+		}
+		//下面可以多线程跑的--->
+		//处理分类
+		mpool <- true
+		go func(tmp, update, compare map[string]interface{}, bnil bool) {
+			defer func() {
+				<-mpool
+			}()
+			if !bnil && compare != nil {
+				subscopeclass, _ := compare["subscopeclass"].([]interface{})
+				if subscopeclass != nil {
+					//str := ","
+					m1 := map[string]bool{}
+					newclass := []string{}
+					for _, sc := range subscopeclass {
+						sclass, _ := sc.(string)
+						if !m1[sclass] {
+							m1[sclass] = true
+							//str += sclass + ","
+							newclass = append(newclass, sclass)
+						}
+					}
+					update["s_subscopeclass"] = strings.Join(newclass, ",")
+					update["subscopeclass"] = newclass
+				}
+				//处理中标企业
+				winner, _ := compare["winner"].(string)
+				m1 := map[string]bool{}
+				if winner != "" {
+					m1[winner] = true
+				}
+				package1 := compare["package"]
+				if package1 != nil {
+					packageM, _ := package1.(map[string]interface{})
+					for _, p := range packageM {
+						pm, _ := p.(map[string]interface{})
+						pw, _ := pm["winner"].(string)
+						if pw != "" {
+							m1[pw] = true
+						}
+					}
+				}
+				compare = nil
+				if len(m1) > 0 {
+					//str := ","
+					winnerarr := []string{}
+					for k, _ := range m1 {
+						//str += k + ","
+						winnerarr = append(winnerarr, k)
+					}
+					update["s_winner"] = strings.Join(winnerarr, ",")
+				}
+			}
+			//------------------对比结束
+
+			//处理key descript
+			//		if bkey == "" {
+			//			DealInfo(&tmp, &update)
+			//		}
+			//同时保存到elastic
+			for tk, tv := range update {
+				tmp[tk] = tv
+			}
+
+			if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
+				tmp["budget"] = nil
+			} else if sbd, ok := tmp["budget"].(string); ok {
+				tmp["budget"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+			}
+			if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
+				tmp["bidamount"] = nil
+			} else if sbd, ok := tmp["bidamount"].(string); ok {
+				tmp["bidamount"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+			}
+			//		for k1, _ := range tmp {
+			//			if strings.HasSuffix(k1, "_b") || k1 == "contenthtml" {
+			//				delete(tmp, k1)
+			//			}
+			//		}
+			//go IS.Add("bidding")
+			UpdatesLock.Lock()
+			if len(update) > 0 {
+				arr = append(arr, []map[string]interface{}{
+					map[string]interface{}{
+						"_id": tmp["_id"],
+					},
+					map[string]interface{}{
+						"$set": update,
+					},
+				})
+			}
+			//if len(arr) >= BulkSize-1 {
+			//	mgo.UpdateBulkAll(db, c, arr...)
+			//	arr = [][]map[string]interface{}{}
+			//}
+
+			UpdatesLock.Unlock()
+		}(tmp, update, compare, bnil)
+		if n%100 == 0 {
+			log.Println("current:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	for i := 0; i < thread; i++ {
+		mpool <- true
+	}
+	//UpdatesLock.Lock()
+	//if len(arr) > 0 {
+	//	mgo.UpdateBulkAll(db, c, arr...)
+	//}
+	//UpdatesLock.Unlock()
+	log.Println(mapInfo, "merge bidding...over", n)
+}

udpcreateindex/biddingtask.go

@@ -0,0 +1,606 @@
+package main
+
+import (
+	"createindex/oss"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+	util "utils"
+	"utils/mongodb"
+	"utils/redis"
+)
+
+var date1 = regexp.MustCompile("20[0-2][0-9][年|\\-/.][0-9]{1,2}[月|\\-/.][0-9]{1,2}[日]?")
+
+// @Description 合并extract 字段到bidding表
+// @Author J 2022/6/7 2:25 PM
+func MergeExtract(tmp, compare, update map[string]interface{}, extractMap map[string]map[string]interface{}, tasktype string) (map[string]interface{}, map[string]interface{}) {
+	tid := mongodb.BsonIdToSId(tmp["_id"])
+	if extractMap[tid] != nil {
+		compare = extractMap[tid]
+		if tasktype == "bidding" {
+			// 增量id段 正常数据
+			if num := util.IntAll(compare["dataging"]); num == 1 { //extract中dataging=1跳过
+				tmp = make(map[string]interface{})
+				compare = nil
+				return compare, update
+			}
+			delete(extractMap, tid)
+		}
+		if tasktype == "bidding_history" {
+			//增量id段 历史数据
+			if compare["history_updatetime"] == nil { //extract中history_updatetime不存在跳过
+				tmp = make(map[string]interface{})
+				compare = nil
+				return compare, update
+			}
+			delete(extractMap, tid)
+		}
+		//更新bidding表，生成索引;bidding表modifyinfo中的字段不更新
+		modifyinfo := make(map[string]bool)
+		if tmpmodifyinfo, ok := tmp["modifyinfo"].(map[string]interface{}); ok && tmpmodifyinfo != nil {
+			for k, _ := range tmpmodifyinfo {
+				modifyinfo[k] = true
+			}
+		}
+		//更新bidding表，生成索引
+		for _, k := range biddingMgoFields {
+			v1 := compare[k] //extract
+			v2 := tmp[k]     //bidding
+			if v2 == nil && v1 != nil && !modifyinfo[k] {
+				update[k] = v1
+			} else if v2 != nil && v1 != nil && !modifyinfo[k] {
+				//update[k+"_b"] = v2
+				update[k] = v1
+			} else if v2 != nil && v1 == nil {
+				//update[k+"_b"] = v2
+				if k == "area" || k == "city" || k == "district" {
+					update[k] = ""
+				}
+			}
+		}
+		if util.IntAll(compare["repeat"]) == 1 {
+			update["extracttype"] = -1
+		} else {
+			update["extracttype"] = 1
+		}
+	} else {
+		compare = nil
+		if util.IntAll(tmp["dataging"]) == 1 { //修改未抽取的bidding数据的dataging
+			update["dataging"] = 0
+		}
+	}
+	return compare, update
+}
+
+// @Description subscopeclass、topscopeclass、package
+// @Author J 2022/6/7 5:54 PM
+func FieldMethod(compare, update map[string]interface{}) {
+	subscopeclass, _ := compare["subscopeclass"].([]interface{}) //subscopeclass
+	if subscopeclass != nil {
+		m1 := map[string]bool{}
+		newclass := []string{}
+		for _, sc := range subscopeclass {
+			sclass, _ := sc.(string)
+			if !m1[sclass] {
+				m1[sclass] = true
+				newclass = append(newclass, sclass)
+			}
+		}
+		update["s_subscopeclass"] = strings.Join(newclass, ",")
+		update["subscopeclass"] = newclass
+	}
+	topscopeclass, _ := compare["topscopeclass"].([]interface{}) //topscopeclass
+	if topscopeclass != nil {
+		m2 := map[string]bool{}
+		newclass := []string{}
+		for _, tc := range topscopeclass {
+			tclass, _ := tc.(string)
+			tclass = reg_letter.ReplaceAllString(tclass, "") // 去除字母
+			if !m2[tclass] {
+				m2[tclass] = true
+				newclass = append(newclass, tclass)
+			}
+		}
+		update["s_topscopeclass"] = strings.Join(newclass, ",")
+	}
+	if package1 := compare["package"]; package1 != nil {
+		packageM, _ := package1.(map[string]interface{})
+		for _, p := range packageM {
+			pm, _ := p.(map[string]interface{})
+			if util.ObjToString(pm["winner"]) != "" || util.Float64All(pm["budget"]) > 0 ||
+				util.Float64All(pm["bidamount"]) > 0 {
+				update["multipackage"] = 1
+				break
+			}
+		}
+	} else {
+		update["multipackage"] = 0
+	}
+}
+
+// @Description ES保存字段
+// @Author J 2022/6/7 11:34 AM
+func GetEsField(tmp, update map[string]interface{}, stype string) map[string]interface{} {
+	newTmp := make(map[string]interface{})
+	for field, ftype := range biddingEsFields {
+		if tmp[field] != nil { //
+			if field == "projectinfo" {
+				mp, _ := tmp[field].(map[string]interface{})
+				if mp != nil {
+					newmap := map[string]interface{}{}
+					for k, ktype := range projectinfoFields {
+						mpv := mp[k]
+						if mpv != nil && reflect.TypeOf(mpv).String() == ktype {
+							newmap[k] = mp[k]
+						}
+					}
+					if len(newmap) > 0 {
+						newTmp[field] = newmap
+					}
+				}
+			} else if field == "purchasinglist" { //标的物处理
+				purchasinglist_new := []map[string]interface{}{}
+				if pcl, _ := tmp[field].([]interface{}); len(pcl) > 0 {
+					for _, ls := range pcl {
+						lsm_new := make(map[string]interface{})
+						lsm := ls.(map[string]interface{})
+						for pf, pftype := range purchasinglistFields {
+							lsmv := lsm[pf]
+							if lsmv != nil && reflect.TypeOf(lsmv).String() == pftype {
+								lsm_new[pf] = lsm[pf]
+							}
+						}
+						if lsm_new != nil && len(lsm_new) > 0 {
+							purchasinglist_new = append(purchasinglist_new, lsm_new)
+						}
+					}
+				}
+				if len(purchasinglist_new) > 0 {
+					newTmp[field] = purchasinglist_new
+				}
+			} else if field == "procurementlist" {
+				if tmp["procurementlist"] != nil {
+					var arr []interface{}
+					plist := tmp["procurementlist"].([]interface{})
+					for _, p := range plist {
+						p1 := p.(map[string]interface{})
+						p2 := make(map[string]interface{})
+						for k, v := range procurementlisFields {
+							if k == "projectname" && util.ObjToString(p1[k]) == "" {
+								p2[k] = util.ObjToString(tmp["projectname"])
+							} else if k == "buyer" && util.ObjToString(p1[k]) == "" && util.ObjToString(tmp["buyer"]) != "" {
+								p2[k] = util.ObjToString(tmp["buyer"])
+							} else if k == "expurasingtime" && util.ObjToString(p1[k]) != "" {
+								res := getMethod(util.ObjToString(p1[k]))
+								if res != 0 {
+									p2[k] = res
+								}
+							} else if p1[k] != nil && reflect.TypeOf(p1[k]).String() == v {
+								p2[k] = p1[k]
+							}
+						}
+						arr = append(arr, p2)
+					}
+					if len(arr) > 0 {
+						newTmp[field] = arr
+					}
+				}
+			} else if field == "projectscope" {
+				ps, _ := tmp["projectscope"].(string)
+				if len(ps) > pscopeLength {
+					newTmp["projectscope"] = string(([]rune(ps))[:pscopeLength])
+				} else {
+					newTmp["projectscope"] = ps
+				}
+			} else if field == "winnerorder" { //中标候选
+				winnerorder_new := []map[string]interface{}{}
+				if winnerorder, _ := tmp[field].([]interface{}); len(winnerorder) > 0 {
+					for _, win := range winnerorder {
+						winMap_new := make(map[string]interface{})
+						winMap := win.(map[string]interface{})
+						for wf, wftype := range winnerorderlistFields {
+							wfv := winMap[wf]
+							if wfv != nil && reflect.TypeOf(wfv).String() == wftype {
+								if wf == "sort" && util.Int64All(wfv) > 100 {
+									continue
+								}
+								winMap_new[wf] = winMap[wf]
+							}
+						}
+						if winMap_new != nil && len(winMap_new) > 0 {
+							winnerorder_new = append(winnerorder_new, winMap_new)
+						}
+					}
+				}
+				if len(winnerorder_new) > 0 {
+					newTmp[field] = winnerorder_new
+				}
+			} else if field == "qualifies" {
+				//项目资质
+				qs := []string{}
+				if q, _ := tmp[field].([]interface{}); len(q) > 0 {
+					for _, v := range q {
+						v1 := v.(map[string]interface{})
+						qs = append(qs, util.ObjToString(v1["key"]))
+					}
+				}
+				if len(qs) > 0 {
+					newTmp[field] = strings.Join(qs, ",")
+				}
+			} else if field == "review_experts" {
+				// 评审专家
+				if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
+					arr1 := util.ObjArrToStringArr(arr)
+					newTmp[field] = strings.Join(arr1, ",")
+				}
+			} else if field == "bidopentime" {
+				if tmp[field] != nil && tmp["bidendtime"] == nil {
+					newTmp["bidendtime"] = tmp[field]
+					newTmp[field] = tmp[field]
+				} else if tmp[field] == nil && tmp["bidendtime"] != nil {
+					newTmp["bidendtime"] = tmp[field]
+					newTmp[field] = tmp["bidendtime"]
+				} else {
+					if tmp["bidopentime"] != nil {
+						newTmp[field] = tmp["bidopentime"]
+					}
+				}
+			} else if field == "detail" { //过滤
+				detail, _ := tmp[field].(string)
+				if len([]rune(detail)) > detailLength {
+					detail = detail[:detailLength]
+				}
+				detail = filterSpace.ReplaceAllString(detail, "")
+				if stype == "bidding" || stype == "bidding_history" {
+					text, b := FilterDetail(detail)
+					newTmp[field] = util.ObjToString(tmp["title"]) + " " + text
+					update["cleartag"] = b
+				} else {
+					if tmp["cleartag"] != nil && tmp["cleartag"].(bool) {
+						text, _ := FilterDetail(detail)
+						newTmp[field] = util.ObjToString(tmp["title"]) + " " + text
+					} else {
+						newTmp[field] = util.ObjToString(tmp["title"]) + " " + detail
+					}
+				}
+			} else if field == "_id" || field == "topscopeclass" || field == "entidlist" { //不做处理
+				newTmp[field] = tmp[field]
+			} else if field == "publishtime" || field == "comeintime" {
+				//字段类型不正确，特别处理
+				if tmp[field] != nil && util.Int64All(tmp[field]) > 0 {
+					newTmp[field] = util.Int64All(tmp[field])
+				}
+			} else { //其它字段判断数据类型，不正确舍弃
+				if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {
+					continue
+				} else {
+					if fieldval != "" {
+						newTmp[field] = fieldval
+					}
+				}
+			}
+		}
+	}
+	filetext := getFileText(tmp)
+	if len([]rune(filetext)) > 10 {
+		newTmp["filetext"] = filetext
+	}
+	//else {
+	//	// 附件未识别到内容，有附件且附件能够下载	filetext=""
+	//	if pinfo, o1 := tmp["projectinfo"].(map[string]interface{}); o1 {
+	//		if atts, o2 := pinfo["attachments"].(map[string]interface{}); o2 {
+	//			tag := false
+	//			for _, at := range atts {
+	//				at1 := at.(map[string]interface{})
+	//				if at1["fid"] != nil {
+	//					tag = true
+	//					break
+	//				}
+	//			}
+	//			if tag {
+	//				newTmp["filetext"] = ""
+	//			}
+	//		}
+	//	}
+	//}
+	YuceEndtime(newTmp) // 预测结果时间
+	if stype == "bidding" || stype == "bidding_history" {
+		newTmp["createtime"] = time.Now().Unix() // es库数据创建时间，只有增量数据有
+	}
+	return newTmp
+}
+
+// @Description 附件内容
+// @Author J 2022/6/7 1:54 PM
+func getFileText(tmp map[string]interface{}) (filetext string) {
+	if attchMap, ok := tmp["attach_text"].(map[string]interface{}); attchMap != nil && ok {
+		for _, tmpData1 := range attchMap {
+			if tmpData2, ok := tmpData1.(map[string]interface{}); tmpData2 != nil && ok {
+				for _, result := range tmpData2 {
+					if resultMap, ok := result.(map[string]interface{}); resultMap != nil && ok {
+						if attach_url := util.ObjToString(resultMap["attach_url"]); attach_url != "" {
+							bs := oss.OssGetObject(attach_url) //oss读数据
+							if utf8.RuneCountInString(filetext+bs) < fileLength {
+								filetext += bs + "\n"
+							} else {
+								if utf8.RuneCountInString(bs) > fileLength {
+									filetext = bs[0:fileLength]
+								} else {
+									filetext = bs
+								}
+								break
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	return
+}
+
+// 预测结果时间
+func YuceEndtime(tmp map[string]interface{}) {
+	flag := true
+	scope := []string{"服务采购_法律咨询", "服务采购_会计", "服务采购_物业", "服务采购_审计", "服务采购_安保", "服务采购_仓储物流",
+		"服务采购_广告宣传印刷"}
+	subscopeclass := util.ObjToString(tmp["s_subscopeclass"])
+	for _, v := range scope {
+		if strings.Contains(subscopeclass, v) {
+			flag = false
+			break
+		}
+	}
+	if flag {
+		return
+	}
+	subtype := util.ObjToString(tmp["subtype"])
+	if subtype == "成交" || subtype == "合同" {
+		// yucestarttime、yuceendtime
+		yucestarttime, yuceendtime := int64(0), int64(0)
+		// 项目周期中
+		if util.ObjToString(tmp["projectperiod"]) != "" {
+			dateStr := date1.FindStringSubmatch(util.ObjToString(tmp["projectperiod"]))
+			if len(dateStr) == 2 {
+				sdate := FormatDateStr(dateStr[0])
+				edate := FormatDateStr(dateStr[1])
+				if sdate < edate && sdate != 0 && edate != 0 {
+					yucestarttime = sdate
+					yuceendtime = edate
+				}
+			}
+		}
+		if yucestarttime > 0 && yuceendtime > yucestarttime {
+			tmp["yuceendtime"] = yuceendtime
+			return
+		}
+		// 预测开始时间 合同签订日期
+		if yucestarttime == 0 {
+			if util.IntAll(tmp["signaturedate"]) <= 0 {
+				if util.IntAll(tmp["publishtime"]) <= 0 {
+					return
+				} else {
+					yucestarttime = util.Int64All(tmp["publishtime"])
+				}
+			} else {
+				yucestarttime = util.Int64All(tmp["signaturedate"])
+			}
+		}
+		// 预测结束时间
+		if yucestarttime > 0 && yuceendtime == 0 {
+			if util.IntAll(tmp["project_duration"]) > 0 && util.ObjToString(tmp["project_timeunit"]) != "" {
+				yuceendtime = YcEndTime(yucestarttime, util.IntAll(tmp["project_duration"]), util.ObjToString(tmp["project_timeunit"]))
+				tmp["yuceendtime"] = yuceendtime
+			}
+		}
+	}
+}
+
+func YcEndTime(starttime int64, num int, unit string) int64 {
+	yuceendtime := int64(0)
+	if unit == "日历天" || unit == "天" || unit == "日" {
+		yuceendtime = starttime + int64(num*86400)
+	} else if unit == "周" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num*7).Unix()
+	} else if unit == "月" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, num, 0).Unix()
+	} else if unit == "年" {
+		yuceendtime = time.Unix(starttime, 0).AddDate(num, 0, 0).Unix()
+	} else if unit == "工作日" {
+		n := num / 7 * 2
+		yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num+n).Unix()
+	}
+	return yuceendtime
+}
+
+func FormatDateStr(ds string) int64 {
+	ds = strings.Replace(ds, "年", "-", -1)
+	ds = strings.Replace(ds, "月", "-", -1)
+	ds = strings.Replace(ds, "日", "", -1)
+	ds = strings.Replace(ds, "/", "-", -1)
+	ds = strings.Replace(ds, ".", "-", -1)
+
+	location, err := time.ParseInLocation(util.Date_Short_Layout, ds, time.Local)
+	if err != nil {
+		util.Debug(err)
+		return 0
+	} else {
+		return location.Unix()
+	}
+}
+
+// @Description entidlist
+// @Author J 2022/6/7 2:36 PM
+func FieldFun(tmp map[string]interface{}) (cid []string) {
+	sWinnerarr := strings.Split(util.ObjToString(tmp["s_winner"]), ",")
+	for _, w := range sWinnerarr {
+		if w != "" {
+			id := redis.GetStr("qyxy_id", w)
+			if id == "" {
+				ents, _ := standardMgo.Find("qyxy_std", map[string]interface{}{"company_name": w}, map[string]interface{}{"updatetime": -1}, nil, false, -1, -1)
+				if len(*ents) > 0 {
+					id = util.ObjToString((*ents)[0]["_id"])
+					redis.PutCKV("qyxy_id", w, id)
+				} else {
+					ent, _ := qyxyMgo.FindOne("company_history_name", map[string]interface{}{"history_name": w})
+					if len(*ent) > 0 {
+						id = util.ObjToString((*ent)["company_id"])
+						redis.PutCKV("qyxy_id", w, id)
+					}
+				}
+			}
+			if id == "" {
+				id = "-"
+			}
+			cid = append(cid, id)
+		}
+	}
+	return cid
+}
+
+var filterSpace = regexp.MustCompile("<[^>]*?>|[\\s\u3000\u2003\u00a0]")
+
+func FilterDetail(text string) (string, bool) {
+	b := false // 清理标记
+	for _, s := range FilterKeyword {
+		reg := regexp.MustCompile(s)
+		if reg.MatchString(text) {
+			text = reg.ReplaceAllString(text, "")
+			if !b {
+				b = true
+			}
+		}
+	}
+	return text, b
+}
+
+// 正则判断是否包含
+func checkContains(s, sub string) bool {
+	reg := regexp.MustCompile(`(?i)(^|([\s\t\n]+))(` + sub + `)($|([\s\t\n]+))`)
+	return reg.MatchString(s)
+}
+
+var TimeV1 = regexp.MustCompile("(\\d{4})[年.]?$")
+var TimeV2 = regexp.MustCompile("(\\d{4}[年.\\-/]?)(\\d{1,2}[月.\\-/]?$)")
+var TimeClear = regexp.MustCompile("[年|月|/|.|-]")
+
+// @Description 采购意向 预计采购时间处理
+// @Author J 2022/6/7 8:04 PM
+func getMethod(str string) int64 {
+	if TimeV1.MatchString(str) {
+		arr := TimeV1.FindStringSubmatch(str)
+		st := arr[1] + "0000"
+		parseInt, err := strconv.ParseInt(st, 10, 64)
+		if err == nil {
+			return parseInt
+		}
+	} else if TimeV2.MatchString(str) {
+		arr := TimeV2.FindStringSubmatch(str)
+		str1 := arr[2]
+		if len(str1) == 1 {
+			str1 = "0" + str1
+		}
+		str2 := TimeClear.ReplaceAllString(arr[1], "") + TimeClear.ReplaceAllString(str1, "") + "00"
+		parseInt, err := strconv.ParseInt(str2, 10, 64)
+		if err == nil {
+			return parseInt
+		}
+	}
+	return 0
+}
+
+// @Description 字段空值处理
+// @Author J 2022/6/7 8:04 PM
+func clearMap(tmp map[string]interface{}) {
+	for k := range tmp {
+		if tmp[k] == nil {
+			continue
+		}
+		if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok && len(purchasinglist) == 0 {
+			delete(tmp, "purchasinglist")
+		} else if reflect.TypeOf(tmp[k]).String() == "string" && util.ObjToString(tmp[k]) == "" {
+			delete(tmp, k)
+		}
+	}
+}
+
+// @Description 附件有效字段（isValidFile）
+// @Author J 2022/7/8 14:41
+func validFile(tmp map[string]interface{}) int {
+	isContinue := false
+	if pinfo, o := tmp["projectinfo"].(map[string]interface{}); o {
+		if atts, o1 := pinfo["attachments"].(map[string]interface{}); o1 {
+			for _, att := range atts {
+				if att == nil {
+					util.Debug(tmp["_id"])
+					continue
+				}
+				if reflect.TypeOf(att).String() == "string" {
+					util.Debug(tmp["_id"])
+					continue
+				}
+				att1 := att.(map[string]interface{})
+				if fid := util.ObjToString(att1["fid"]); fid != "" {
+					isContinue = true
+					break
+				}
+			}
+			if isContinue {
+				if attachTxt, o := tmp["attach_text"].(map[string]interface{}); o {
+					if len(attachTxt) > 0 {
+						for _, at := range attachTxt {
+							at1 := at.(map[string]interface{})
+							if len(at1) > 0 {
+								for k, _ := range at1 {
+									if reflect.TypeOf(at1[k]).String() == "string" {
+										util.Debug(tmp["_id"])
+										continue
+									}
+									at2 := at1[k].(map[string]interface{})
+									s := strings.ToLower(util.ObjToString(at2["file_name"]))
+									if !strings.Contains(s, "jpg") || !strings.Contains(s, "jpeg") != strings.Contains(s, "png") ||
+										strings.Contains(s, "pdf") {
+										if strings.Contains(s, "swf") || strings.Contains(s, "html") {
+											return -1
+										} else if AnalysisFile(oss.OssGetObject(util.ObjToString(at2["attach_url"]))) {
+											return 1
+										}
+									}
+								}
+								break
+							} else {
+								break
+							}
+						}
+					}
+				}
+				flag := false
+				for _, att := range atts {
+					if att == nil {
+						continue
+					}
+					if reflect.TypeOf(att).String() == "string" {
+						continue
+					}
+					att1 := att.(map[string]interface{})
+					if fid := util.ObjToString(att1["fid"]); fid != "" {
+						ftype := strings.ToLower(util.ObjToString(tmp["ftype"]))
+						if ftype != "swf" && ftype != "html" && oss.OssObjExists("jy-datafile", fid) {
+							return 1
+						} else {
+							flag = true
+						}
+					}
+				}
+				if flag {
+					return -1
+				}
+			}
+		}
+	}
+	return 0
+}

udpcreateindex/buyerindex.go

@@ -0,0 +1,256 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	"log"
+	"sync"
+	util "utils"
+	"utils/elastic"
+	"utils/mongodb"
+)
+
+/*
+func buyerTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	session := extractmgo.GetMgoConn()
+	defer extractmgo.DestoryMongoConn(session)
+	c, _ := buyer["collect"].(string)
+	db, _ := buyer["db"].(string)
+	index, _ := buyer["index"].(string)
+	itype, _ := buyer["type"].(string)
+	count, _ := session.DB(db).C(c).Find(&q).Count()
+	savepool := make(chan bool, 10)
+
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	query := session.DB(db).C(c).Find(q).Select(map[string]interface{}{
+		"buyer_name": 1,
+		"province":   1,
+		"city":       1,
+		"district":   1,
+	}).Iter()
+
+	arr := make([]map[string]interface{}, savesizei)
+	var n int
+	i := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+		//go IS.Add("buyer")
+		tmp["name"] = tmp["buyer_name"]
+		delete(tmp, "buyer_name")
+		arr[i] = tmp
+		n++
+		if i == savesizei-1 {
+			savepool <- true
+			tmps := arr
+			go func(tmpn *[]map[string]interface{}) {
+				defer func() {
+					<-savepool
+				}()
+				elastic.BulkSave(index, itype, tmpn, true)
+			}(&tmps)
+			i = 0
+			arr = make([]map[string]interface{}, savesizei)
+		}
+		if n%savesizei == 0 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	if i > 0 {
+		elastic.BulkSave(index, itype, &arr, true)
+	}
+	log.Println(mapInfo, "create buyer index...over", n)
+}
+*/
+
+//buyer_err
+func buyerTask_err(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	//mongo
+	sess := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(sess)
+	c, _ := buyer["collect"].(string)
+	index, _ := buyer["index"].(string)
+	itype, _ := buyer["type"].(string)
+
+	count, _ := sess.DB(standardMgo.DbName).C(c).Find(&q).Count()
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	it := sess.DB(standardMgo.DbName).C(c).Find(&q).Select(map[string]interface{}{
+		"name":              1,
+		"buyer_name":        1,
+		"institute_type":    1,
+		"buyerclass":        1,
+		"fixedphone":        1,
+		"mobilephone":       1,
+		"latestfixedphone":  1,
+		"latestmobilephone": 1,
+		"province":          1,
+		"city":              1,
+	}).Sort("_id").Iter()
+
+	arrEs := []map[string]interface{}{}
+	buyerEsLock := &sync.Mutex{}
+	pool := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+	i := 0
+	for tmp := make(map[string]interface{}); it.Next(tmp); i = i + 1 {
+		if i%1000 == 0 {
+			log.Println("current:", i)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			util.Debug(tmp)
+			savetmp := map[string]interface{}{}
+			tmp_id := util.BsonIdToSId(tmp["_id"])
+			savetmp["_id"] = tmp_id
+			savetmp["name"] = tmp["name"]
+			savetmp["buyer_name"] = tmp["name"]
+			util.Debug(tmp["buyerclass"])
+			if tmp["buyerclass"] != nil {
+				savetmp["buyerclass"] = tmp["buyerclass"]
+			}
+			for _, f := range fieldArr {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					savetmp[f] = val
+				}
+			}
+			util.Debug(savetmp)
+			buyerEsLock.Lock()
+			arrEs = append(arrEs, savetmp)
+			if len(arrEs) >= MgoBulkSize {
+				tmps := arrEs
+				elastic.BulkSave(index, itype, &tmps, true)
+				arrEs = []map[string]interface{}{}
+			}
+			buyerEsLock.Unlock()
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	buyerEsLock.Lock()
+	if len(arrEs) > 0 {
+		tmps := arrEs
+		elastic.BulkSave(index, itype, &tmps, true)
+		arrEs = []map[string]interface{}{}
+	}
+	buyerEsLock.Unlock()
+	log.Println(mapInfo, "create buyer index...over", i)
+}
+
+//buyer_enterprise
+func buyerTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	//mongo
+	sess := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(sess)
+
+	c, _ := buyer["collect"].(string)
+	index, _ := buyer["index"].(string)
+	itype, _ := buyer["type"].(string)
+
+	count, _ := sess.DB(standardMgo.DbName).C(c).Find(&q).Count()
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	it := sess.DB(standardMgo.DbName).C(c).Find(&q).Select(map[string]interface{}{
+		"buyer_name":        1,
+		"institute_type":    1,
+		"buyerclass":        1,
+		"fixedphone":        1,
+		"mobilephone":       1,
+		"latestfixedphone":  1,
+		"latestmobilephone": 1,
+		"province":          1,
+		"city":              1,
+	}).Sort("_id").Iter()
+
+	arrEs := []map[string]interface{}{}
+	buyerEsLock := &sync.Mutex{}
+	pool := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+	i := 0
+	for tmp := make(map[string]interface{}); it.Next(tmp); i = i + 1 {
+		if i%1000 == 0 {
+			log.Println("current:", i)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			savetmp := map[string]interface{}{}
+			_id := util.BsonIdToSId(tmp["_id"])
+			//if buyerclass, ok := tmp["buyerclass"].([]interface{}); ok && len(buyerclass) > 0 {
+			//	for _, v := range util.ObjArrToStringArr(buyerclass) {
+			//		if len(buyerclass) >= 2 && v != "其它" {
+			//			savetmp["buyerclass"] = v
+			//			break
+			//		} else if len(buyerclass) == 1 {
+			//			savetmp["buyerclass"] = v
+			//			break
+			//		}
+			//	}
+			//}
+			if util.ObjToString(tmp["buyerclass"]) != "" {
+				savetmp["buyerclass"] = tmp["buyerclass"]
+			}
+			savetmp["_id"] = _id
+			savetmp["name"] = tmp["buyer_name"]
+			savetmp["buyer_name"] = tmp["buyer_name"]
+			for _, f := range fieldArr {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					savetmp[f] = val
+				}
+			}
+			buyerEsLock.Lock()
+			arrEs = append(arrEs, savetmp)
+			if len(arrEs) >= EsBulkSize {
+				tmps := arrEs
+				elastic.BulkSave(index, itype, &tmps, true)
+				arrEs = []map[string]interface{}{}
+			}
+			buyerEsLock.Unlock()
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	buyerEsLock.Lock()
+	if len(arrEs) > 0 {
+		tmps := arrEs
+		elastic.BulkSave(index, itype, &tmps, true)
+		arrEs = []map[string]interface{}{}
+	}
+	buyerEsLock.Unlock()
+	log.Println(mapInfo, "create buyer index...over", i)
+}

udpcreateindex/buyertask.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"log"
+	"sync"
+	"time"
+	util "utils"
+	"utils/mongodb"
+)
+
+var fieldArr = []string{"institute_type", "fixedphone", "mobilephone", "latestfixedphone", "latestmobilephone", "province", "city"}
+
+func buyerEsTaskOnce() {
+	defer util.Catch()
+	arrEs := []map[string]interface{}{}
+	buyerEsLock := &sync.Mutex{}
+	pool := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+
+	now := time.Now()
+	preTime := time.Date(now.Year(), now.Month(), now.Day()-1, now.Hour(), 0, 0, 0, time.Local)
+	curTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.Local)
+	task_sid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(preTime))
+	task_eid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(curTime))
+	//task_sid = "5e6611b7aec95406dccf714f"
+	//task_eid = "625c79bf799a3acc48890f48"
+	log.Println("buyer 区间id:", task_sid, task_eid)
+	//区间id
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			//"$gte": mongodb.StringTOBsonId(task_sid),
+			"$lt": mongodb.StringTOBsonId(task_eid),
+		},
+	}
+	//参数
+	buyerent, _ := standard["buyerent"].(map[string]interface{})
+	buyer_ent := util.ObjToString(buyerent["collect1"])
+	//buyer_enterr := qu.ObjToString(buyerent["collect2"])
+	index, _ := buyerent["index"].(string)
+	itype, _ := buyerent["type"].(string)
+	//mongo
+	sess := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(sess)
+
+	log.Println("q:", q, "db:", standardMgo.DbName, "coll:", buyer_ent)
+	it_1 := sess.DB(standardMgo.DbName).C(buyer_ent).Find(&q).Select(map[string]interface{}{
+		"buyer_name":        1,
+		"institute_type":    1,
+		"buyerclass":        1,
+		"fixedphone":        1,
+		"mobilephone":       1,
+		"latestfixedphone":  1,
+		"latestmobilephone": 1,
+		"province":          1,
+		"city":              1,
+	}).Sort("_id").Iter()
+	num_1 := 0
+	for tmp := make(map[string]interface{}); it_1.Next(&tmp); num_1++ {
+		if num_1%2000 == 0 && num_1 > 0 {
+			log.Println("当前表:", buyer_ent, "数量:", num_1)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			savetmp := map[string]interface{}{}
+			_id := mongodb.BsonIdToSId(tmp["_id"])
+			//if buyerclass, ok := tmp["buyerclass"].([]interface{}); ok && len(buyerclass) > 0 {
+			//	for _, v := range qu.ObjArrToStringArr(buyerclass) {
+			//		if len(buyerclass) >= 2 && v != "其它" {
+			//			savetmp["buyerclass"] = v
+			//			break
+			//		} else if len(buyerclass) == 1 {
+			//			savetmp["buyerclass"] = v
+			//			break
+			//		}
+			//	}
+			//}
+			if util.ObjToString(tmp["buyerclass"]) != "" {
+				savetmp["buyerclass"] = tmp["buyerclass"]
+			}
+			savetmp["_id"] = _id
+			savetmp["name"] = tmp["buyer_name"]
+			savetmp["buyer_name"] = tmp["buyer_name"]
+			for _, f := range fieldArr {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					savetmp[f] = val
+				}
+			}
+			buyerEsLock.Lock()
+			arrEs = append(arrEs, savetmp)
+			if len(arrEs) >= EsBulkSize {
+				tmps := arrEs
+				//Es1.BulkSave(index, itype, &tmps, true)
+				Es2.BulkSave(index, itype, &tmps, true)
+				arrEs = []map[string]interface{}{}
+			}
+			buyerEsLock.Unlock()
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+
+	// log.Println("q:", q, "db:", mgostandard.DbName, "coll:", buyer_enterr)
+	// it_2 := sess.DB(mgostandard.DbName).C(buyer_enterr).Find(&q).Sort("_id").Iter()
+	// num_2 := 0
+	// for tmp := make(map[string]interface{}); it_2.Next(&tmp); num_2++ {
+	// 	if num_2%100 == 0 && num_2 > 0 {
+	// 		log.Println("当前表:", buyer_enterr, "数量:", num_2)
+	// 	}
+	// 	pool <- true
+	// 	wg.Add(1)
+	// 	go func(tmp map[string]interface{}) {
+	// 		defer func() {
+	// 			<-pool
+	// 			wg.Done()
+	// 		}()
+	// 		savetmp := map[string]interface{}{}
+	// 		tmp_id := mongodb.BsonIdToSId(tmp["_id"])
+	// 		savetmp["_id"] = tmp_id
+	// 		savetmp["name"] = tmp["name"]
+	// 		savetmp["buyer_name"] = tmp["name"]
+	// 		if tmp["buyerclass"] != nil {
+	// 			savetmp["buyerclass"] = tmp["buyerclass"]
+	// 		}
+	// 		for _, f := range fieldArr {
+	// 			if val := qu.ObjToString(tmp[f]); val != "" {
+	// 				savetmp[f] = val
+	// 			}
+	// 		}
+	// 		buyerEsLock.Lock()
+	// 		arrEs = append(arrEs, savetmp)
+	// 		if len(arrEs) >= BulkSize {
+	// 			tmps := arrEs
+	// 			elastic.BulkSave(index, itype, &tmps, true)
+	// 			arrEs = []map[string]interface{}{}
+	// 		}
+	// 		buyerEsLock.Unlock()
+	// 	}(tmp)
+	// 	tmp = make(map[string]interface{})
+	// }
+
+	wg.Wait()
+	buyerEsLock.Lock()
+	if len(arrEs) > 0 {
+		tmps := arrEs
+		//Es1.BulkSave(index, itype, &tmps, true)
+		Es2.BulkSave(index, itype, &tmps, true)
+		arrEs = []map[string]interface{}{}
+	}
+	buyerEsLock.Unlock()
+	log.Println("buyeres  索引完毕！  总计:", num_1)
+}

udpcreateindex/common.txt

@@ -0,0 +1,9103 @@
+标	58819516
+人	39327632
+采	37936120
+的	37706721
+购	37481886
+公	36236358
+项	35631355
+目	33967273
+件	26066781
+中	24248928
+有	23875013
+工	23036670
+投	22866151
+名	21407564
+号	20671264
+市	19269013
+交	19033811
+应	18848999
+系	18504891
+文	18482498
+合	18472714
+时	18067230
+日	17950354
+招	17888352
+限	17325715
+联	17053179
+供	16297165
+方	16220463
+理	16165812
+司	16150407
+商	16003508
+价	15723465
+单	15684031
+称	15405229
+资	15092717
+电	14801139
+行	14290956
+信	14155867
+要	13825731
+程	13582836
+间	13557620
+本	13486562
+地	13426348
+务	13103530
+式	12939750
+金	12911438
+政	12791451
+建	12618565
+业	12450689
+同	12355117
+一	12177349
+设	12025515
+年	12008036
+格	11898630
+区	11850149
+元	11787621
+求	11629922
+成	11521187
+期	11447562
+证	11446141
+在	11348311
+及	10533500
+法	10372269
+报	10283071
+服	9784153
+不	9714957
+国	9649759
+提	9540450
+位	9525848
+告	9451662
+发	9378666
+机	9371225
+代	9351397
+为	9316699
+开	9309262
+规	9292674
+用	9284145
+定	9257293
+内	9196330
+月	9090142
+上	9008766
+网	8787776
+和	8738743
+量	8637377
+或	8430681
+息	8395952
+收	8243861
+备	7938980
+关	7895512
+编	7764352
+额	7686138
+府	7664619
+其	7632937
+保	7491743
+二	7468216
+分	7430224
+以	7359694
+下	7312858
+质	7298552
+址	7204975
+品	7153116
+管	7143789
+路	7054318
+大	6817743
+通	6781003
+三	6640535
+具	6519282
+计	6476683
+产	6321054
+点	6310657
+省	6308498
+数	6253031
+南	6085380
+询	6077527
+明	6067214
+包	6017638
+安	5958643
+作	5909347
+书	5831227
+构	5700053
+民	5652617
+验	5629225
+需	5612571
+体	5571681
+台	5571331
+于	5571030
+请	5524015
+自	5419321
+技	5398602
+北	5362314
+子	5338761
+准	5321892
+县	5312416
+参	5251467
+心	5230099
+源	5217124
+平	5202515
+进	5135833
+话	5108527
+注	5095567
+家	5077132
+取	5062035
+事	4986453
+后	4964046
+州	4954410
+医	4947546
+他	4925427
+第	4914147
+个	4908503
+无	4892453
+能	4855887
+责	4827705
+共	4823283
+易	4746731
+经	4729181
+等	4682049
+审	4644034
+加	4621486
+费	4620905
+可	4596597
+施	4580836
+止	4566158
+生	4562760
+对	4560231
+物	4534178
+评	4523552
+结	4501496
+四	4470421
+部	4437842
+料	4436663
+次	4409259
+东	4407033
+主	4376419
+江	4282625
+型	4275385
+出	4254273
+至	4244973
+新	4242474
+术	4239479
+现	4236283
+选	4232888
+西	4231203
+须	4200297
+院	4175189
+天	4174511
+相	4139154
+按	4135925
+布	4113542
+容	4061766
+与	4060170
+接	4048018
+所	4027172
+统	4011386
+办	4010128
+水	3993851
+山	3968762
+会	3960513
+动	3959106
+货	3957738
+过	3924695
+截	3798874
+监	3792703
+见	3746579
+全	3734245
+原	3692204
+并	3689534
+实	3684910
+企	3679242
+员	3655475
+制	3652198
+条	3620903
+录	3618905
+学	3613588
+查	3552078
+详	3544265
+果	3532413
+前	3514368
+五	3501341
+示	3498546
+如	3491606
+高	3446041
+华	3376518
+算	3374267
+附	3373930
+表	3368417
+小	3352573
+京	3351770
+预	3349917
+道	3343760
+城	3321651
+范	3314534
+情	3304161
+局	3290648
+记	3276988
+面	3268452
+楼	3256312
+门	3253149
+重	3226097
+印	3224345
+受	3215067
+场	3210992
+级	3193092
+总	3183047
+册	3174882
+议	3151135
+性	3138405
+委	3130202
+得	3127922
+器	3113768
+况	3097288
+任	3093234
+承	3085080
+海	3070804
+力	3046506
+划	3043746
+负	3041284
+专	3040204
+万	3036017
+类	3008622
+未	2992826
+递	2977149
+码	2974054
+化	2968585
+广	2937610
+集	2911090
+获	2907817
+执	2854796
+段	2854308
+效	2848833
+复	2835076
+将	2828753
+装	2824141
+六	2807725
+到	2794867
+营	2786367
+室	2761559
+财	2732123
+阳	2731002
+照	2719034
+否	2717402
+造	2706437
+响	2702722
+起	2686014
+被	2669621
+镇	2669036
+序	2668425
+份	2666035
+竞	2659169
+之	2658254
+是	2642337
+科	2618360
+传	2601158
+福	2601147
+入	2590900
+组	2564730
+材	2558388
+登	2556419
+支	2555946
+签	2550655
+问	2547868
+权	2545841
+湖	2526457
+七	2524657
+河	2463953
+线	2397851
+基	2389455
+处	2342891
+户	2316553
+载	2304768
+章	2295472
+知	2286348
+疗	2278376
+度	2265837
+据	2262838
+团	2252328
+清	2245080
+咨	2233629
+活	2219336
+修	2214296
+符	2185225
+站	2183927
+履	2175340
+满	2175306
+款	2167447
+完	2163147
+者	2152709
+向	2148274
+由	2139074
+补	2113953
+意	2085657
+改	2079798
+税	2072666
+售	2053677
+正	2050850
+外	2050049
+订	2016076
+运	2012633
+街	2008563
+必	2002018
+测	2000473
+充	1998204
+案	1986382
+邮	1983327
+最	1977975
+口	1970028
+宜	1967344
+筑	1965412
+达	1962867
+银	1962409
+约	1952071
+检	1947493
+十	1923604
+社	1917230
+足	1904144
+已	1902157
+违	1886809
+利	1860096
+控	1846022
+失	1816579
+密	1799438
+牌	1780836
+维	1743889
+整	1728112
+付	1724149
+展	1723353
+长	1714646
+村	1707377
+环	1698287
+纳	1682969
+围	1682073
+异	1676047
+云	1675338
+历	1671881
+八	1650409
+防	1649703
+光	1644023
+超	1642035
+列	1640257
+核	1639047
+当	1638361
+认	1636001
+盖	1635575
+确	1624651
+申	1620648
+套	1620502
+宁	1609579
+含	1608284
+股	1602058
+币	1600503
+械	1599592
+园	1586133
+因	1582770
+账	1577571
+配	1572687
+疑	1572451
+授	1570086
+林	1569986
+师	1559571
+立	1557217
+指	1552264
+特	1540052
+托	1536315
+治	1516899
+车	1498069
+批	1489197
+缴	1483000
+磋	1482588
+送	1482456
+页	1480137
+更	1477474
+则	1469191
+节	1463848
+图	1462932
+许	1456337
+形	1436368
+房	1428089
+龙	1427671
+来	1401990
+纸	1397182
+字	1391895
+候	1372832
+织	1368523
+各	1362012
+督	1361394
+介	1347730
+判	1342959
+解	1335683
+功	1331088
+属	1329554
+担	1322561
+视	1318483
+手	1314356
+除	1312429
+持	1311455
+存	1302933
+使	1300048
+库	1298811
+午	1296306
+细	1295534
+述	1288728
+身	1288311
+真	1286909
+根	1282324
+买	1276428
+依	1265894
+厂	1257546
+概	1246760
+张	1246529
+诉	1241188
+谈	1240393
+直	1238271
+箱	1232270
+答	1227754
+调	1220945
+农	1218421
+置	1216536
+九	1204848
+石	1198228
+简	1197159
+际	1194497
+比	1194391
+济	1185709
+甲	1180284
+说	1178859
+川	1178662
+描	1165155
+综	1161174
+境	1158081
+护	1152230
+苏	1142862
+操	1135255
+良	1128030
+争	1119630
+庆	1115380
+先	1108248
+策	1101585
+排	1091831
+层	1091273
+流	1083425
+试	1083088
+乙	1082814
+仪	1075257
+钢	1067532
+写	1066151
+兴	1053950
+转	1052534
+函	1049527
+乡	1047734
+色	1038789
+厦	1033927
+每	1031299
+米	1027902
+且	1011499
+好	1001482
+若	1001364
+职	1000882
+德	994605
+模	989962
+律	986850
+铁	977547
+予	972159
+该	971655
+拟	967470
+头	952926
+常	951499
+青	944357
+绩	941121
+低	936126
+均	924547
+双	924053
+凡	920874
+潜	918937
+严	901540
+汇	896222
+独	894595
+多	893667
+消	892593
+落	892344
+变	891450
+卫	887497
+闽	887130
+武	882437
+打	865467
+输	863569
+健	849511
+王	845662
+康	838924
+假	838390
+板	830632
+题	824597
+都	819962
+诺	819783
+率	812115
+校	810883
+免	799996
+票	796946
+教	796243
+障	792729
+我	792345
+险	790976
+带	790377
+李	790357
+声	789469
+桥	787112
+气	787074
+束	784727
+尔	783021
+致	776873
+协	774789
+风	772603
+土	771839
+领	764287
+别	764109
+优	763618
+黄	758961
+版	755180
+始	750320
+黑	749884
+导	743570
+此	742403
+终	732958
+座	729964
+退	728591
+临	721257
+厅	717619
+态	717526
+填	716332
+昌	715791
+媒	713721
+智	711668
+压	705360
+扫	702607
+红	701325
+热	698947
+括	685298
+育	682628
+看	679201
+它	672626
+普	672052
+浙	671796
+徽	671000
+没	665237
+微	663354
+租	663328
+少	662218
+放	661492
+永	659283
+油	656468
+泰	654782
+研	653029
+空	653016
+吉	652196
+近	651187
+白	645898
+深	645705
+族	645342
+连	642519
+察	640540
+周	636695
+诚	636461
+访	634730
+温	629821
+木	628892
+沙	627434
+何	627305
+非	626273
+考	625994
+卖	624840
+汉	624740
+暂	623664
+老	621887
+增	620848
+田	620595
+封	619285
+速	614710
+回	612668
+陈	611766
+续	609367
+从	608428
+状	605306
+卡	603216
+升	602959
+销	601662
+种	599399
+强	597368
+片	596808
+惠	593772
+软	590538
+途	589875
+启	588734
+贸	584442
+疾	583420
+络	578317
+创	576733
+即	575383
+刷	571485
+凭	571175
+顺	567131
+首	564841
+远	562521
+显	561006
+贵	558861
+两	557293
+花	555432
+刘	554439
+值	554414
+勘	553159
+然	550417
+港	549652
+古	548037
+估	547561
+步	545343
+决	544923
+块	543163
+养	540807
+域	538414
+只	537028
+绿	536529
+盘	534696
+适	534231
+推	525541
+积	522808
+丰	522470
+害	522180
+美	520428
+逾	518868
+频	514587
+克	513650
+波	513060
+病	510298
+彩	509235
+再	502802
+志	499488
+陆	499361
+誉	498431
+究	497083
+拒	495951
+架	495749
+档	495416
+津	494678
+延	493707
+益	493165
+春	490742
+马	488359
+澄	488264
+击	484520
+庄	483584
+停	483555
+栋	480547
+蒙	475415
+滨	475251
+影	470374
+了	470366
+例	468021
+博	463843
+星	462271
+圳	461268
+融	461076
+岛	456059
+盒	455555
+邀	455123
+就	454012
+液	452874
+残	451707
+客	450172
+队	450087
+士	449679
+药	449048
+择	446665
+景	444950
+湘	444038
+杨	441363
+换	440345
+宝	439426
+航	436031
+训	434164
+派	432880
+给	432546
+义	429575
+副	428467
+移	427259
+住	427172
+辆	427101
+疆	425981
+姓	423716
+损	423332
+兰	420834
+疫	419548
+泉	416974
+废	416783
+峰	416160
+齐	414265
+杭	411186
+胶	409107
+佰	408538
+拾	407902
+离	407322
+固	406132
+里	405771
+还	405579
+绝	404978
+宣	404414
+店	403264
+盛	402774
+溪	402289
+恒	401451
+助	398303
+晨	397791
+肥	395526
+促	394839
+端	394830
+郑	391935
+瑞	390973
+故	389524
+鑫	389423
+太	385905
+镜	385750
+儿	383975
+仟	380866
+扣	380252
+唐	379886
+便	379224
+女	378710
+幼	378459
+狱	377796
+拍	376537
+干	375794
+角	371351
+随	370540
+笔	370470
+允	367437
+乐	364749
+析	363915
+屋	363683
+携	363514
+辽	363429
+煤	363365
+玉	362396
+洲	362294
+反	362041
+急	359243
+血	358968
+屏	355456
+警	354122
+鲁	353796
+丽	353596
+但	353586
+脑	352475
+夏	352467
+侧	352380
+陵	350955
+柜	350837
+剂	347529
+衡	347243
+润	347039
+矿	345748
+甘	345383
+军	344710
+食	344272
+乌	343950
+切	343853
+池	343624
+观	342031
+竣	341366
+初	338222
+做	338083
+罚	337408
+陕	337243
+样	336863
+余	336281
+馆	334194
+污	333536
+音	333206
+宏	331776
+底	327448
+像	326855
+哈	325920
+壹	324938
+贰	323995
+威	323265
+精	322741
+吴	322740
+球	321429
+础	320076
+富	320058
+塘	318694
+漳	318585
+硬	317850
+培	317583
+徐	315872
+烟	313773
+岳	312843
+针	310659
+岗	309567
+洁	309091
+纪	306780
+佛	305187
+昆	304555
+嘉	303542
+拉	299470
+冷	297788
+叁	297036
+储	296875
+毒	295427
+抽	295271
+汽	293705
+革	292202
+栏	292014
+肃	291995
+佳	291589
+灯	290829
+断	290533
+阅	289715
+较	286451
+荐	286092
+似	285281
+诊	284445
+宇	283239
+渠	282893
+寸	281989
+绍	281806
+罗	281503
+寄	281110
+鼓	280308
+浦	280199
+旗	279901
+游	279710
+宽	278890
+火	277913
+荣	277707
+幢	277502
+伟	276430
+梁	275411
+英	274739
+览	274317
+沈	272754
+百	271595
+粉	270584
+禁	270268
+湾	269859
+快	268656
+另	267900
+床	267759
+零	267239
+慧	267000
+洪	265949
+珠	265518
+伍	263125
+锦	261538
+赁	261482
+涉	261101
+迎	260661
+锁	260381
+虚	260338
+互	259548
+潭	259206
+素	258795
+松	258060
+凤	255820
+链	255485
+耗	254175
+亚	252672
+洗	252370
+蓝	251176
+边	251174
+令	250416
+泵	249481
+季	249045
+留	248883
+世	248349
+呼	247725
+塔	247567
+颁	247545
+铜	247536
+界	247291
+袋	244777
+仅	244275
+轮	243430
+厚	242942
+浩	242699
+象	242590
+救	240884
+吨	240433
+巴	240203
+识	239326
+引	237841
+雨	235922
+隆	233912
+妇	233314
+届	232987
+振	232949
+飞	232669
+桂	232604
+弃	232079
+墙	232074
+晓	232059
+香	232031
+键	232005
+泽	231824
+夹	231485
+祥	231242
+而	231121
+论	230655
+阿	229886
+梅	229834
+赵	229576
+阀	228931
+螺	228855
+秀	226884
+署	226875
+辉	226188
+居	224886
+泥	224771
+桌	224541
+感	223961
+饰	223939
+贿	223293
+众	223027
+邵	222469
+径	220523
+钟	219549
+勒	218685
+旅	216138
+激	215429
+沟	214415
+胜	213757
+插	213111
+射	213089
+圾	212373
+鼎	212324
+玖	212118
+锈	211983
+垃	211956
+岭	211951
+官	210550
+党	210200
+挂	209284
+让	209074
+吸	208692
+皮	208390
+播	208291
+劳	207428
+误	206165
+斯	205871
+井	204974
+梯	204436
+抗	204374
+牛	204255
+洋	204000
+坊	203957
+艺	203665
+扩	203587
+把	203287
+混	203145
+才	201868
+塑	200340
+罪	199516
+尺	198578
+欢	198389
+草	198386
+酸	197182
+静	196475
+阶	195490
+犯	195319
+读	194884
+凯	194464
+极	193701
+叶	193576
+浮	193051
+冠	192907
+缆	191595
+淮	191494
+扶	191317
+株	191021
+善	190395
+苑	189944
+椅	189935
+聚	189808
+郭	189740
+索	189647
+神	188988
+捌	187419
+鸿	184524
+筹	183922
+酒	182089
+岩	181801
+吊	180177
+休	179503
+拆	179153
+灾	179135
+坪	178153
+杰	177776
+怀	176058
+肆	175935
+谷	175755
+征	175193
+胡	174584
+遵	173949
+丹	173835
+千	173487
+赣	173388
+群	173307
+毕	172966
+右	172565
+冻	172411
+卷	172383
+茂	172293
+树	171078
+顶	170996
+辅	170487
+菜	170385
+殊	169923
+杆	169776
+凝	169653
+措	169205
+破	169194
+芯	169096
+茶	168544
+堂	168315
+腾	166938
+差	166312
+守	165700
+柳	165444
+森	165414
+铺	164585
+轴	163913
+框	163747
+想	163678
+尚	163313
+己	163276
+朝	163016
+扬	162517
+偏	162114
+亮	161923
+洛	161788
+莞	161782
+鹏	161723
+仓	161293
+孙	161099
+迁	161010
+敏	160888
+思	160153
+谢	160060
+绘	159322
+瓶	159112
+氧	159035
+锡	158162
+宿	157642
+净	157197
+曲	156730
+朱	156174
+减	155980
+雅	155103
+避	154856
+宾	154706
+左	154417
+缺	154382
+染	153999
+越	153882
+苗	153819
+叉	153163
+燕	152938
+般	152606
+娄	152057
+友	151677
+摄	151340
+尾	151259
+仁	151065
+墨	150671
+滤	150421
+举	149480
+寿	149439
+享	149110
+亿	148510
+碳	148411
+裁	148340
+帐	148333
+透	148263
+浏	148228
+今	148074
+孔	148072
+踏	147564
+燃	146629
+冶	146553
+丝	146188
+也	146029
+筒	145719
+奖	145628
+隔	144845
+坝	143956
+晶	143511
+偿	142705
+触	142564
+柒	142547
+旧	142410
+桶	142247
+待	142001
+爱	141858
+半	141666
+班	141550
+距	140838
+菌	140760
+沂	140759
+骨	140674
+撤	139261
+垫	139098
+磁	138647
+吾	137985
+圆	137867
+靖	137807
+耐	137148
+喷	136761
+翔	136193
+刀	135479
+雷	135340
+赛	135140
+鄂	135135
+郴	135074
+坡	134846
+够	134787
+冲	134205
+晋	133711
+桃	133379
+盐	133265
+宋	132551
+窗	132251
+贫	131439
+毛	131414
+秒	130996
+磨	130773
+勤	129967
+岸	129828
+横	129400
+探	128766
+驻	128669
+腔	128086
+雪	127748
+败	127542
+往	127053
+央	127008
+藏	126959
+莱	126776
+餐	126495
+折	126438
+幅	126228
+缮	125581
+秦	125468
+炉	124986
+暖	124673
+练	124626
+童	124550
+雄	124453
+霞	123350
+弄	122135
+轨	121102
+植	120074
+捷	119952
+芜	119867
+汕	119757
+饮	118840
+仙	118785
+灵	118161
+潍	117968
+聊	117915
+降	117745
+援	117731
+鲜	117621
+散	117489
+拖	116874
+帮	116748
+晟	116725
+唯	116526
+错	116331
+望	116222
+欣	115814
+硒	115573
+粤	115122
+讯	114626
+渝	114047
+冀	113978
+罩	113764
+稳	113576
+芳	113400
+贷	113285
+纯	113283
+母	112877
+阜	112744
+贴	112718
+灭	112554
+麻	112403
+课	112320
+遗	112175
+铝	111887
+占	111816
+君	111784
+兵	111296
+滩	110628
+潮	110393
+紧	109256
+玛	108943
+史	108691
+戴	108558
+语	108472
+寻	108170
+艳	107904
+俊	107349
+奥	107307
+漏	106859
+焦	106433
+釆	106427
+牵	106314
+缩	106046
+壁	105952
+焊	105420
+颜	105375
+卓	105361
+膜	105327
+彭	105052
+漆	104206
+亦	104161
+杯	103568
+蔡	103541
+鹤	103473
+旭	103264
+紫	102862
+戒	102594
+什	102423
+阴	102323
+淀	102305
+沿	101600
+恶	101490
+勇	101473
+闭	101217
+柱	101189
+巷	101086
+竹	100898
+洞	100501
+晰	100405
+旺	100280
+辖	99990
+尽	99666
+拓	99630
+继	99243
+赤	98701
+曾	98521
+欧	98291
+曹	98258
+灌	98089
+累	97992
+豫	97580
+玲	97470
+肺	97441
+拨	97404
+邦	97373
+你	97096
+某	97029
+灰	96807
+旋	96378
+辰	95807
+纤	95676
+杂	95497
+伊	95412
+舍	94990
+黔	94778
+短	94717
+涛	94537
+馈	94396
+弹	94382
+柯	94306
+忠	93836
+玻	93608
+萍	93455
+棉	93256
+遇	92981
+屯	92897
+壮	92750
+危	92653
+湛	92430
+榆	92275
+亩	92035
+背	92034
+莲	91985
+丁	91774
+衣	91733
+崇	91712
+鞍	91696
+循	91627
+铭	91622
+赔	91300
+尼	91099
+辨	91071
+楚	90967
+迪	90950
+眼	90945
+庭	90789
+挥	90192
+栓	90022
+恩	89974
+圈	89927
+槽	89657
+棚	89582
+裕	89328
+沧	89320
+陷	89301
+禹	89187
+涂	89071
+悦	88930
+坤	88857
+滑	88857
+卸	88757
+伦	88728
+斌	88670
+韩	88401
+追	87762
+峡	87214
+阻	87071
+船	86963
+粮	86683
+靠	86576
+鉴	86240
+埠	86054
+兆	85893
+碎	85784
+廊	85764
+姚	85757
+缝	85691
+宗	85674
+骗	85549
+炎	85187
+幕	84929
+亭	84858
+脉	84856
+睿	84793
+襄	84240
+丘	84058
+患	83896
+虑	83895
+兼	83841
+锋	83816
+虹	83581
+伏	83544
+姐	83234
+债	83138
+肖	83093
+露	83078
+鸡	83043
+橡	82949
+纹	82726
+贤	82687
+侯	82346
+驱	82324
+绵	82290
+荆	82210
+豪	82133
+皇	81481
+刻	81450
+镀	81446
+澳	81267
+希	80839
+涵	80791
+钻	80737
+脱	80555
+湿	79898
+弘	79875
+贝	79699
+璃	79383
+奇	79376
+锌	78930
+瑶	78864
+麦	78751
+娟	78689
+顾	78607
+秋	78511
+邓	78485
+杜	78048
+渡	78046
+串	77955
+画	77376
+砂	77256
+命	77110
+钮	77090
+醒	77087
+震	76781
+钉	76474
+撑	76153
+蓉	76045
+习	75511
+韶	75289
+迟	75063
+聘	74398
+唱	74369
+仔	74349
+刚	74010
+讼	73937
+骤	73331
+斗	73295
+谱	73013
+愿	72737
+冬	72643
+荷	72636
+吕	72610
+寨	72588
+凉	72572
+困	72534
+宅	72425
+饶	72194
+锐	72137
+稿	72074
+难	72009
+闸	71562
+桩	71475
+牧	71385
+仲	71293
+畅	71158
+庐	71035
+冰	71028
+喀	70953
+听	70862
+仑	70798
+翠	70750
+脚	70730
+熟	70649
+雁	70518
+孚	70497
+弯	70484
+凌	70259
+寓	70251
+席	70129
+厨	69934
+剩	69768
+冯	69740
+昭	69730
+久	69643
+牙	69630
+释	69558
+董	69527
+盟	69408
+尘	69349
+贺	69252
+邢	69191
+泸	69057
+乳	68994
+典	68436
+抚	68432
+圣	68284
+鱼	68070
+冈	68017
+鼠	67149
+禾	66691
+硕	66455
+魏	66187
+萨	65996
+坑	65864
+锅	65777
+瓦	65498
+潘	65442
+恢	65400
+柏	65374
+邯	65064
+旁	65055
+桐	64903
+婷	64869
+晖	64765
+伤	64625
+舒	64596
+尝	64512
+轻	64377
+厕	64329
+慈	64305
+淄	64209
+丙	64174
+巡	63999
+芦	63875
+斤	63829
+陶	63635
+黎	63112
+揭	62949
+势	62949
+敬	62912
+肇	62780
+耀	62612
+羊	62605
+旦	62568
+枣	62562
+粒	62340
+映	62142
+替	61903
+爆	61757
+铸	61708
+言	61699
+穿	61393
+胞	61249
+袁	61208
+归	61074
+宫	61025
+肉	61023
+隐	60926
+刺	60755
+磊	60694
+琴	60628
+仍	60506
+搬	60021
+蒸	59723
+跨	59405
+去	59230
+堰	59196
+甸	59100
+姜	58721
+沥	58641
+钥	58625
+惩	58623
+缘	58574
+卢	58516
+虎	58212
+烧	58155
+壶	58102
+昊	58057
+腐	57755
+粘	57678
+沃	57656
+演	57578
+浆	57171
+凰	57125
+邹	57101
+迈	57022
+礼	57013
+廉	57012
+蒋	56973
+硅	56831
+砖	56823
+隧	56658
+郸	56630
+奉	56606
+蚌	56510
+咸	56423
+碧	56423
+汪	56356
+荧	56297
+剪	56288
+耳	56051
+战	56031
+馨	56007
+覆	55646
+巾	55368
+蓄	55196
+筋	55186
+盈	55171
+拥	55035
+堤	54948
+怡	54829
+找	54742
+泡	54527
+鸣	54527
+返	54264
+剑	54188
+彦	54133
+莉	53865
+祁	53845
+尿	53668
+舟	53631
+渣	53576
+皖	53418
+庙	53227
+钱	53194
+巨	53161
+挡	52918
+霍	52781
+疏	52723
+跃	52714
+沅	52629
+扎	52571
+孟	52391
+艾	52244
+割	52076
+逐	51870
+纬	51794
+耕	51622
+挖	51228
+辑	51149
+鹿	51060
+凳	50516
+筛	50474
+朗	50398
+炼	50356
+堡	50323
+肠	50315
+纠	50183
+遂	49916
+早	49911
+钦	49854
+腹	49692
+兑	49685
+喜	49664
+瓷	49551
+汤	49524
+翻	49435
+轧	49242
+沁	49210
+轩	49107
+谊	48897
+彝	48815
+珍	48814
+噪	48754
+绥	48733
+邑	48699
+彬	48634
+胎	48632
+枝	48603
+醉	48595
+顿	48478
+孝	48351
+野	48105
+柴	48042
+坚	47991
+绳	47745
+碱	47722
+芙	47698
+佩	47596
+匹	47591
+邱	47535
+纷	47435
+恕	47171
+梦	47169
+驰	47018
+脂	47001
+绑	46942
+蛋	46851
+枪	46342
+芝	46295
+蜀	46058
+谭	45998
+罐	45775
+瘤	45561
+您	45168
+坏	45144
+走	45049
+刑	45020
+励	45005
+贡	45005
+钳	44914
+韵	44821
+暨	44553
+突	44548
+肿	44533
+寺	44529
+删	44352
+琼	44347
+熊	44296
+莆	44283
+埔	44281
+臂	44134
+舜	44086
+仿	44019
+伸	43952
+衬	43888
+添	43866
+枫	43663
+娜	43413
+墩	43387
+炭	43383
+虫	43364
+狮	43213
+蔬	42993
+跟	42933
+琪	42582
+堵	42556
+抵	42443
+琳	42270
+欠	42234
+荔	42208
+毅	42151
+翼	42100
+陇	42085
+滚	41937
+伪	41839
+廖	41747
+乾	41604
+缓	41584
+埋	41551
+梧	41529
+柔	41339
+氏	41324
+堆	41317
+鹰	41304
+募	41250
+押	41241
+默	41186
+亲	41152
+奔	41131
+鼻	41123
+帆	40901
+烯	40722
+番	40693
+胃	40621
+症	40583
+汾	40547
+阵	40526
+纺	40409
+塞	40364
+扇	40276
+觉	40203
+斜	39973
+贾	39833
+盆	39821
+坛	39768
+崔	39668
+硫	39645
+畜	39622
+倒	39577
+嘴	39574
+讲	39568
+弱	39529
+窥	39460
+铅	39319
+摇	39316
+滁	39316
+赫	39198
+遥	39196
+殖	38944
+乔	38943
+葛	38911
+雾	38830
+御	38796
+衢	38728
+纽	38612
+薛	38496
+齿	38485
+榕	38029
+牡	37929
+扰	37917
+闻	37906
+渔	37869
+霄	37867
+孜	37687
+摸	37603
+盾	37454
+尖	37435
+肌	37404
+圩	37222
+侗	37141
+攻	37063
+秘	36979
+槐	36898
+帽	36777
+柄	36741
+壳	36731
+骏	36726
+辛	36724
+倍	36676
+味	36547
+毁	36534
+篮	36466
+曙	36431
+颖	36293
+浪	36291
+纵	36215
+坐	36049
+曼	36040
+略	36011
+垦	35841
+鞋	35728
+夫	35656
+男	35645
+喉	35573
+跳	35531
+借	35254
+闪	35253
+搜	35207
+几	35145
+侨	35142
+踪	35133
+泄	35060
+薄	34824
+逸	34795
+译	34739
+召	34697
+缸	34562
+尹	34460
+氯	34454
+匙	34357
+刊	34318
+抢	34246
+胸	34181
+攀	34144
+澧	34141
+遭	34086
+这	34059
+伙	34007
+籍	33974
+氢	33866
+禅	33863
+桑	33857
+擦	33737
+钩	33647
+芬	33642
+着	33425
+氨	33359
+私	33155
+沪	33101
+郊	32997
+鄞	32820
+醇	32783
+莹	32686
+醴	32664
+幸	32545
+枢	32482
+念	32408
+悉	32390
+叠	32299
+垂	32231
+麟	32202
+禺	32134
+弟	32131
+酌	32107
+驿	32037
+洱	32025
+泾	31925
+那	31913
+侵	31830
+披	31779
+溶	31712
+杀	31689
+渭	31584
+巩	31552
+棒	31489
+裂	31253
+丧	31216
+耒	31017
+辐	30920
+氟	30700
+帘	30686
+颗	30637
+碑	30629
+砌	30515
+淘	30312
+匀	30266
+脏	30262
+遴	30202
+伯	30167
+莫	29966
+麓	29931
+颤	29823
+菏	29765
+拌	29752
+搭	29750
+窑	29731
+蒲	29721
+兄	29639
+肤	29634
+趣	29632
+仕	29590
+泓	29589
+杏	29588
+宸	29564
+羽	29548
+沉	29478
+乘	29344
+菲	29332
+浓	29280
+岚	29247
+渗	29201
+役	29183
+绕	29105
+们	29093
+糖	28989
+砼	28777
+壤	28768
+歌	28743
+夷	28733
+舞	28644
+驶	28620
+韦	28604
+慎	28505
+秩	28496
+淑	28463
+萧	28416
+茅	28416
+驾	28340
+芷	28289
+末	28262
+菊	28151
+悬	28150
+澜	28024
+却	27991
+脸	27942
+摩	27846
+遣	27822
+蓬	27791
+阁	27758
+酯	27699
+诸	27602
+峪	27553
+迹	27535
+泗	27444
+涟	27394
+麒	27350
+楠	27293
+辊	27278
+稀	27077
+椒	27037
+芗	26948
+龚	26925
+巧	26864
+荫	26858
+朔	26837
+罕	26817
+虞	26763
+苍	26639
+猪	26623
+奎	26611
+酶	26587
+倩	26533
+烤	26488
+汝	26485
+泊	26456
+涌	26436
+抄	26412
+聪	26377
+掘	26376
+券	26281
+亳	26258
+豆	26094
+隅	26019
+淳	25994
+勿	25987
+磷	25944
+厢	25794
+梓	25712
+瑕	25701
+溯	25682
+巢	25582
+疵	25525
+熔	25518
+抑	25472
+舶	25465
+囊	25457
+敷	25423
+忻	25394
+呈	25311
+舱	25226
+珊	25180
+乱	25136
+姑	25132
+赖	25041
+倾	24952
+饭	24868
+绣	24858
+蚀	24840
+濮	24825
+溉	24802
+贯	24799
+肢	24791
+尧	24577
+矩	24499
+龄	24482
+揽	24460
+诏	24351
+拼	24313
+浑	24266
+磅	24196
+滇	24137
+伴	24079
+繁	24051
+瀚	24005
+郝	23937
+劣	23906
+蔽	23903
+掌	23891
+邻	23870
+眉	23794
+谋	23782
+碍	23771
+雕	23755
+闫	23754
+媛	23746
+琦	23676
+毫	23565
+亨	23543
+哲	23542
+拔	23516
+浴	23480
+岁	23441
+涪	23409
+弥	23362
+洽	23324
+谨	23301
+慢	23252
+卧	23217
+氮	23203
+粗	23194
+婴	23154
+淖	23125
+暗	23044
+傅	23025
+闵	22905
+淤	22888
+烈	22846
+迅	22772
+帅	22747
+樊	22697
+祝	22689
+楞	22679
+煌	22656
+昇	22653
+捏	22624
+献	22509
+昱	22391
+倪	22322
+祖	22320
+滦	22284
+妙	22269
+熙	22260
+敦	22189
+抓	22037
+煜	21974
+闲	21951
+菱	21942
+樟	21907
+炜	21851
+晚	21847
+握	21785
+霖	21701
+禄	21690
+蝶	21633
+翁	21628
+坎	21623
+抹	21553
+厘	21541
+腺	21540
+瑜	21521
+酷	21403
+著	21375
+滕	21373
+灶	21272
+粪	21225
+簧	21216
+浔	21201
+盗	21190
+垣	21186
+枚	21117
+蜜	21090
+涤	21085
+鸟	20978
+穗	20978
+棠	20948
+葫	20797
+摆	20718
+炬	20711
+钠	20653
+锥	20650
+又	20633
+锚	20633
+昕	20566
+骑	20535
+诗	20532
+皓	20492
+陀	20446
+陂	20414
+牟	20408
+犁	20389
+岔	20359
+冒	20305
+漯	20275
+胆	20174
+刮	20130
+墅	20114
+搏	20102
+攸	20071
+尤	20053
+吐	19986
+苯	19937
+践	19855
+孵	19794
+雯	19737
+诈	19710
+奶	19703
+肝	19648
+拱	19629
+既	19550
+璐	19485
+泌	19472
+醛	19351
+彻	19349
+扳	19324
+殡	19310
+磐	19275
+淞	19205
+莎	19176
+泛	19128
+潼	19103
+嵌	19038
+纱	19029
+酵	19003
+嵩	18946
+妥	18894
+蕾	18882
+鸭	18834
+稻	18834
+帝	18736
+泳	18727
+臭	18709
+玮	18691
+裙	18683
+浇	18631
+颅	18615
+奕	18580
+颍	18539
+死	18494
+伞	18476
+颈	18463
+滋	18441
+璧	18438
+衔	18438
+旱	18348
+祺	18319
+皂	18289
+夜	18234
+锂	18173
+渤	18126
+钰	18053
+些	17907
+庞	17862
+搅	17794
+臻	17697
+铲	17635
+聂	17607
+猜	17594
+吹	17535
+丛	17534
+俞	17467
+殷	17461
+涡	17455
+蔚	17436
+膏	17412
+燥	17392
+胺	17386
+翰	17360
+铃	17356
+瓜	17186
+皋	17154
+腰	17137
+汶	17090
+卉	16967
+涝	16962
+坦	16951
+拦	16945
+覃	16912
+薇	16894
+遮	16894
+佑	16816
+灿	16812
+蕉	16808
+叫	16764
+栅	16749
+晏	16740
+杉	16689
+滴	16688
+汛	16660
+溧	16583
+汀	16547
+剧	16468
+郡	16465
+佐	16445
+梨	16435
+莒	16432
+谐	16355
+岐	16340
+濉	16324
+沭	16276
+潞	16182
+碗	16123
+貌	16111
+掉	16105
+郎	16104
+歧	16096
+颐	16086
+巫	16057
+寰	16056
+箭	16055
+淋	16052
+恺	16033
+袖	16017
+朋	15948
+尊	15932
+斑	15903
+擅	15820
+耿	15799
+桦	15789
+膨	15750
+涧	15718
+跑	15713
+谅	15651
+沫	15590
+娇	15582
+赢	15539
+屈	15537
+寒	15527
+芒	15513
+拜	15510
+浅	15428
+烷	15344
+胀	15333
+沣	15332
+逻	15330
+曝	15315
+荒	15287
+岱	15237
+潇	15193
+溢	15191
+轿	15100
+鹅	15097
+肩	15069
+箕	14994
+腿	14987
+兽	14903
+玩	14902
+炳	14870
+屿	14868
+栽	14858
+眠	14853
+沐	14848
+戏	14823
+蚊	14777
+纲	14770
+栗	14730
+栖	14730
+莘	14691
+摘	14687
+屹	14680
+钛	14668
+骆	14637
+酬	14629
+炮	14595
+牢	14592
+郁	14474
+浚	14424
+猫	14356
+勐	14342
+渍	14331
+丢	14190
+匠	14078
+窝	14075
+侦	14069
+刹	14060
+尉	14056
+枕	14054
+劲	14024
+绒	14002
+昂	13992
+挤	13932
+巍	13931
+缠	13911
+沽	13884
+帧	13826
+椎	13784
+赋	13783
+穆	13779
+洼	13753
+赉	13745
+旌	13744
+荡	13734
+勾	13733
+皆	13723
+妃	13711
+沛	13690
+晴	13671
+瓯	13653
+洒	13649
+兹	13647
+栾	13619
+嘎	13601
+绞	13488
+浸	13477
+咽	13473
+潢	13437
+浠	13422
+扁	13399
+弧	13374
+撞	13248
+崂	13198
+崖	13193
+蜡	13178
+篇	13178
+谦	13118
+讨	13101
+萌	13071
+荟	13070
+冉	13059
+穴	13046
+曦	13039
+锻	13028
+郫	13013
+烨	13010
+笼	12992
+岑	12987
+催	12867
+邺	12854
+峨	12837
+缀	12832
+傣	12792
+禽	12790
+矫	12748
+玺	12716
+赠	12713
+墓	12703
+卜	12697
+垒	12663
+淼	12639
+藤	12635
+棋	12622
+勋	12619
+睡	12511
+坞	12499
+鲍	12465
+棕	12424
+峻	12424
+漫	12400
+痕	12376
+玄	12366
+霸	12354
+喇	12305
+肾	12262
+凸	12240
+乒	12219
+喻	12159
+蜂	12122
+滘	12079
+阮	12077
+芹	12067
+锯	12057
+矛	12056
+乓	12052
+鳌	11997
+坂	11939
+羌	11922
+碚	11888
+脊	11886
+伽	11862
+贞	11858
+泷	11829
+捆	11799
+渌	11794
+珞	11784
+毯	11770
+辩	11744
+彪	11709
+翟	11706
+诱	11682
+畴	11680
+嵊	11674
+兖	11656
+烘	11636
+剖	11599
+雀	11445
+茌	11433
+沱	11392
+妹	11387
+榄	11360
+儒	11337
+丈	11336
+帚	11327
+茹	11323
+削	11309
+妮	11238
+璟	11232
+硝	11184
+隙	11153
+猛	11151
+乃	11128
+爽	11126
+郧	11091
+廷	11061
+锤	11044
+痛	11040
+逆	10999
+叙	10995
+苹	10931
+孕	10896
+抛	10885
+屉	10884
+裤	10875
+詹	10812
+赂	10803
+舰	10686
+弓	10678
+睢	10635
+箍	10630
+彤	10584
+腊	10529
+竖	10519
+敖	10495
+鲤	10471
+亡	10469
+痰	10449
+忽	10439
+汰	10438
+铬	10424
+隶	10415
+瞒	10387
+庵	10384
+耽	10372
+铂	10349
+赞	10337
+迷	10332
+簸	10317
+寅	10309
+帕	10260
+暴	10218
+沾	10192
+卿	10176
+綦	10171
+恰	10166
+洮	10132
+茗	10124
+词	10101
+妍	10070
+碰	10043
+焕	10042
+媳	10029
+笑	9935
+拷	9841
+俗	9829
+靶	9772
+芸	9766
+顷	9743
+哨	9708
+籽	9698
+怒	9672
+溆	9591
+忆	9582
+钙	9579
+邗	9568
+擎	9540
+柘	9538
+戈	9529
+蚁	9527
+婧	9525
+盲	9525
+肯	9523
+娥	9516
+檀	9514
+茄	9502
+茜	9499
+糊	9498
+臣	9475
+掖	9473
+秤	9416
+衷	9335
+霉	9331
+弋	9323
+篷	9291
+塌	9282
+饱	9255
+樱	9220
+窄	9199
+桔	9179
+汗	9157
+琛	9153
+甬	9149
+挺	9136
+畔	9129
+淇	9127
+饲	9096
+氛	9068
+旬	9030
+铣	9024
+萝	8994
+甄	8955
+刨	8946
+涿	8935
+舆	8928
+寮	8925
+胁	8877
+岷	8877
+纶	8871
+翡	8861
+耦	8849
+咳	8827
+焚	8817
+乏	8812
+笛	8760
+圃	8730
+鹭	8720
+菁	8711
+垛	8709
+禧	8680
+柠	8647
+扭	8642
+砚	8619
+忘	8608
+婺	8607
+烽	8604
+硚	8587
+很	8574
+绪	8554
+桓	8545
+裴	8534
+梳	8510
+橙	8508
+汨	8481
+艇	8480
+棵	8477
+钧	8470
+窦	8469
+趋	8442
+剥	8435
+拐	8413
+秸	8370
+湄	8369
+贮	8351
+伐	8341
+阔	8340
+讫	8289
+漂	8274
+甜	8270
+哑	8265
+茵	8253
+蛟	8231
+奋	8228
+淡	8212
+宪	8210
+煲	8209
+咀	8204
+姆	8197
+秆	8191
+杠	8186
+捕	8165
+涞	8149
+盼	8139
+璜	8104
+湟	8101
+萃	8093
+甫	8077
+嫌	8070
+亏	8048
+辣	8028
+匾	8012
+驼	7958
+陪	7956
+樾	7950
+镍	7949
+雍	7915
+盏	7906
+葬	7904
+焱	7901
+吧	7897
+抱	7896
+氩	7860
+叭	7836
+蕴	7823
+熏	7810
+灸	7791
+骅	7788
+喆	7773
+蟠	7750
+宛	7734
+钾	7732
+衍	7725
+阎	7719
+棱	7715
+靴	7714
+烁	7709
+钨	7695
+冗	7686
+斥	7646
+珑	7641
+坳	7632
+楷	7572
+傲	7570
+葡	7565
+勃	7551
+篡	7538
+遍	7478
+厝	7474
+韬	7473
+刁	7457
+阐	7449
+浈	7448
+渚	7441
+珲	7417
+蛇	7411
+岙	7378
+哪	7377
+凹	7321
+偶	7302
+邸	7289
+埃	7283
+韧	7279
+漕	7246
+渊	7238
+驳	7233
+瑾	7188
+烫	7155
+胱	7115
+昶	7086
+瞿	7082
+蔺	7079
+胰	7075
+珂	7066
+慰	7054
+逊	7053
+庚	7049
+癌	7044
+迫	7039
+勺	6996
+咏	6991
+锗	6989
+鄢	6979
+枯	6964
+么	6963
+糕	6932
+氰	6927
+谯	6902
+忧	6825
+酮	6790
+榜	6782
+摊	6764
+奈	6754
+饼	6751
+仰	6742
+琅	6709
+邛	6706
+赶	6677
+酉	6676
+廓	6676
+篦	6659
+陡	6656
+筐	6649
+檬	6639
+萄	6636
+瀛	6636
+膀	6614
+殿	6606
+玫	6593
+啤	6582
+汞	6577
+秭	6574
+荥	6566
+毓	6557
+焰	6555
+彰	6548
+撒	6544
+垢	6543
+钊	6533
+坻	6533
+娱	6526
+蓟	6517
+匿	6497
+秉	6493
+姣	6450
+吞	6438
+劝	6416
+悔	6412
+焉	6400
+爬	6383
+炫	6373
+佘	6371
+蕊	6369
+榨	6357
+渑	6345
+颂	6342
+卞	6333
+姿	6332
+句	6329
+濠	6328
+裸	6309
+沚	6276
+皿	6247
+涨	6246
+鄱	6244
+嫩	6242
+靳	6237
+峒	6219
+鲲	6214
+裹	6212
+芽	6204
+褚	6192
+涯	6185
+耆	6181
+碘	6163
+欺	6153
+茨	6148
+拣	6144
+杞	6142
+锰	6134
+祠	6116
+滞	6107
+骄	6077
+腻	6060
+瑛	6059
+婚	6048
+镁	5989
+霜	5976
+薪	5955
+浜	5951
+芮	5945
+塬	5943
+畲	5937
+钼	5924
+盱	5887
+贻	5881
+罘	5876
+戚	5874
+晒	5858
+弗	5851
+玥	5777
+蕲	5772
+缪	5756
+匡	5751
+葵	5747
+颌	5714
+菇	5694
+酰	5680
+淅	5671
+慕	5655
+眙	5635
+璇	5620
+栈	5619
+拿	5605
+娅	5604
+撕	5594
+铵	5588
+儋	5570
+懿	5557
+抬	5495
+缔	5474
+棣	5464
+衰	5463
+朴	5459
+虽	5457
+畈	5455
+镊	5427
+剔	5427
+酱	5417
+婉	5417
+酚	5406
+崃	5398
+努	5381
+晔	5367
+刃	5357
+凿	5348
+斐	5337
+膝	5316
+烃	5313
+藁	5309
+碟	5299
+狗	5297
+抖	5295
+腈	5289
+骁	5272
+漠	5269
+矮	5267
+瞳	5266
+魁	5258
+铰	5254
+腕	5223
+镶	5223
+檐	5212
+炔	5208
+姬	5205
+垌	5198
+髓	5193
+笺	5182
+鞘	5157
+卤	5141
+晗	5132
+偃	5132
+哥	5125
+歙	5118
+帜	5117
+炸	5100
+樵	5095
+挑	5094
+瓣	5091
+篓	5090
+簿	5088
+曳	5087
+郯	5084
+邳	5079
+爵	5076
+琨	5075
+晃	5068
+槎	5063
+垄	5056
+霁	5054
+垭	5040
+臧	5006
+缙	4982
+匝	4981
+灞	4980
+卵	4974
+鸠	4961
+灏	4947
+撰	4932
+昀	4921
+朵	4913
+珺	4912
+厉	4901
+枞	4899
+瑚	4895
+碾	4890
+撮	4886
+娃	4882
+邕	4881
+榭	4873
+筷	4863
+肪	4862
+炒	4853
+咖	4841
+屑	4834
+钜	4831
+俱	4828
+铠	4821
+崩	4819
+荻	4812
+矸	4800
+樘	4798
+嵘	4797
+屠	4792
+矶	4786
+稽	4777
+漾	4770
+垚	4767
+蓓	4752
+兔	4744
+魔	4729
+埭	4719
+橱	4710
+姗	4709
+拘	4708
+钬	4686
+弦	4656
+汁	4656
+竟	4647
+蒜	4640
+醚	4608
+轶	4595
+瑰	4582
+婕	4568
+煎	4564
+葱	4548
+崆	4543
+铆	4531
+侠	4520
+迭	4516
+朐	4510
+钞	4509
+豹	4508
+浊	4487
+衫	4484
+濂	4482
+汴	4472
+雇	4458
+扑	4447
+跌	4431
+戳	4422
+坯	4404
+伺	4400
+歇	4394
+熠	4394
+胥	4390
+肋	4357
+舌	4356
+疃	4351
+箔	4351
+狼	4350
+蠡	4348
+碣	4328
+蒂	4320
+仆	4306
+薯	4299
+寇	4288
+谌	4286
+鹃	4282
+邬	4279
+啡	4270
+眩	4269
+酿	4266
+恐	4262
+盂	4260
+猴	4255
+掇	4254
+悟	4250
+锹	4248
+裱	4233
+虾	4230
+隽	4228
+丸	4219
+醋	4216
+峄	4206
+泺	4204
+犬	4203
+飘	4202
+爪	4202
+嗽	4200
+畸	4190
+宙	4172
+萱	4168
+邡	4159
+啸	4158
+钓	4127
+俐	4115
+歪	4092
+昔	4091
+棍	4085
+膳	4082
+惯	4068
+碶	4049
+疼	4047
+瓮	4036
+逢	4021
+祜	4001
+溜	4000
+拭	3990
+椿	3988
+幻	3971
+褥	3969
+堃	3961
+郏	3960
+毡	3948
+峥	3945
+孤	3938
+黏	3922
+贩	3918
+镐	3915
+捞	3915
+肛	3871
+钒	3868
+尕	3858
+妆	3846
+瞬	3846
+钣	3845
+狂	3842
+倡	3827
+吻	3813
+罡	3808
+笋	3795
+盔	3779
+笤	3772
+撬	3768
+勉	3743
+榴	3716
+氦	3699
+仇	3679
+鸽	3659
+娣	3659
+佟	3656
+咪	3626
+筠	3623
+棘	3607
+铄	3598
+蝇	3597
+坨	3590
+塍	3587
+戎	3584
+绛	3579
+咬	3562
+煮	3560
+垅	3535
+埕	3530
+桨	3526
+佬	3524
+杖	3523
+椰	3510
+瘦	3510
+溅	3509
+隋	3508
+磴	3498
+霆	3485
+稍	3484
+俭	3481
+姻	3478
+铖	3469
+梵	3465
+砾	3456
+曜	3440
+燊	3434
+佤	3432
+柑	3429
+阙	3428
+赐	3425
+俄	3417
+陉	3408
+馏	3407
+宴	3404
+婵	3403
+睦	3400
+徕	3398
+僳	3397
+娴	3394
+欲	3392
+傈	3390
+疸	3386
+砀	3383
+闯	3382
+毂	3379
+祎	3376
+偷	3360
+蚕	3359
+娘	3348
+苇	3342
+赟	3341
+汐	3338
+椭	3335
+铎	3316
+佣	3313
+铀	3301
+夯	3289
+璞	3279
+兜	3273
+宰	3269
+淦	3268
+琰	3266
+皱	3263
+蹈	3260
+绸	3258
+晾	3258
+猗	3246
+翊	3240
+浒	3229
+镉	3226
+龟	3226
+阈	3218
+疲	3216
+靓	3212
+祯	3209
+嘱	3201
+鲸	3184
+蹄	3182
+苟	3175
+磺	3173
+髋	3156
+胤	3143
+溴	3133
+粟	3132
+徒	3130
+姝	3120
+藻	3113
+烦	3113
+吃	3109
+梭	3109
+筱	3094
+飚	3091
+挠	3087
+耶	3080
+橘	3076
+箐	3073
+恋	3072
+柿	3070
+於	3069
+吗	3064
+崎	3063
+骥	3061
+硼	3054
+奚	3052
+胚	3050
+珏	3036
+钴	3029
+铮	3028
+凑	3026
+蜗	3018
+斡	3017
+矢	3005
+郓	2998
+埂	2994
+盎	2973
+戊	2972
+搁	2961
+蔓	2955
+鞠	2955
+霏	2949
+坭	2948
+拳	2947
+坠	2925
+踞	2920
+藕	2917
+珀	2915
+桁	2912
+滔	2911
+鸦	2906
+疮	2893
+碁	2887
+掺	2884
+曌	2878
+蹲	2870
+扒	2858
+蝴	2849
+趟	2848
+篱	2846
+苦	2844
+锉	2836
+墟	2825
+烹	2824
+茉	2819
+澎	2818
+窖	2814
+宕	2813
+卯	2813
+赴	2800
+蒗	2797
+旨	2793
+翘	2790
+辙	2790
+稷	2785
+沸	2782
+娩	2780
+竭	2773
+岘	2771
+锣	2766
+埇	2759
+踢	2755
+缤	2745
+喊	2738
+褪	2722
+垸	2716
+甚	2710
+踝	2708
+锟	2701
+岫	2697
+粱	2690
+旷	2679
+渐	2675
+囗	2674
+谁	2672
+唇	2671
+汊	2662
+琊	2661
+灼	2659
+壕	2652
+忱	2652
+耘	2651
+陌	2646
+迳	2638
+楔	2636
+蟑	2633
+柚	2625
+旻	2624
+鳄	2618
+泮	2614
+翌	2614
+钤	2612
+契	2601
+恭	2598
+冕	2587
+烂	2582
+翎	2578
+犹	2577
+琥	2568
+坟	2567
+絮	2563
+捐	2538
+坍	2532
+驹	2522
+狄	2516
+伶	2511
+丞	2508
+捣	2498
+愈	2494
+骼	2489
+砷	2488
+鄠	2487
+骞	2486
+珙	2484
+蔗	2478
+噶	2476
+帖	2476
+狭	2471
+锴	2471
+橄	2463
+竿	2461
+羡	2451
+竺	2449
+吋	2445
+郢	2445
+瞻	2433
+砸	2432
+燎	2431
+赏	2429
+侃	2424
+蓥	2419
+槟	2418
+耙	2418
+啄	2414
+翀	2413
+仫	2409
+父	2404
+瓴	2400
+厌	2396
+媚	2375
+亘	2373
+莓	2371
+妈	2367
+粥	2361
+暑	2359
+莺	2357
+艘	2354
+淆	2352
+桢	2350
+砍	2344
+幽	2343
+骋	2341
+斋	2338
+釜	2328
+俯	2325
+漱	2323
+熄	2323
+锑	2309
+枋	2307
+濞	2307
+犍	2305
+熹	2304
+峁	2302
+甩	2302
+浯	2296
+犀	2288
+踊	2285
+薰	2285
+鲅	2283
+颇	2279
+搪	2268
+锷	2262
+氙	2260
+婆	2259
+卒	2259
+缅	2258
+唤	2255
+逃	2254
+焙	2250
+敕	2248
+鸥	2245
+栩	2245
+瘟	2240
+澡	2239
+砟	2222
+翅	2218
+惟	2214
+啦	2214
+瘫	2212
+摔	2189
+袜	2184
+梗	2183
+唑	2182
+峙	2179
+讷	2170
+铨	2167
+阆	2165
+炊	2163
+沤	2161
+碌	2159
+钎	2157
+燚	2152
+糯	2146
+锭	2143
+钝	2138
+驷	2134
+邝	2134
+拧	2130
+陟	2127
+铧	2122
+脆	2121
+稠	2119
+恤	2118
+炯	2118
+肘	2102
+妻	2102
+濛	2090
+镭	2089
+倚	2084
+惕	2080
+淝	2079
+捉	2076
+辕	2068
+诬	2060
+贪	2057
+狐	2053
+寝	2050
+浉	2046
+蛛	2043
+崧	2042
+囱	2037
+螂	2037
+庸	2035
+煊	2035
+肽	2033
+祉	2031
+迦	2027
+懋	2026
+晕	2026
+洺	2025
+赈	2021
+垟	2015
+骐	2011
+箬	2010
+饪	2007
+蓼	2005
+挛	2001
+夕	1993
+孩	1990
+珩	1980
+裘	1980
+衙	1978
+悠	1966
+闹	1965
+袍	1960
+滢	1955
+礁	1953
+渎	1949
+镂	1947
+琉	1945
+鸾	1941
+脐	1940
+鄯	1933
+痉	1929
+弈	1929
+澍	1928
+鸯	1927
+秣	1926
+黟	1926
+潺	1919
+兢	1918
+忙	1915
+喂	1914
+弊	1912
+韭	1901
+沼	1891
+绷	1887
+劵	1883
+浣	1878
+巽	1877
+蠕	1874
+骎	1871
+淠	1870
+掏	1869
+掩	1867
+锆	1865
+猇	1864
+芋	1858
+俏	1848
+吡	1844
+牦	1838
+垱	1836
+萤	1836
+侬	1831
+虔	1827
+煦	1823
+滏	1822
+郜	1821
+阡	1813
+孢	1811
+敲	1803
+逄	1801
+銮	1796
+芃	1796
+聋	1795
+岢	1791
+烜	1789
+魅	1789
+淹	1784
+梢	1776
+祈	1772
+逗	1772
+臆	1769
+拽	1765
+吟	1759
+猎	1758
+遏	1756
+漓	1755
+鄄	1755
+挪	1750
+琶	1744
+鳞	1739
+胭	1737
+驭	1736
+丫	1734
+茫	1733
+砧	1731
+扈	1731
+鹊	1728
+搓	1728
+辜	1728
+浐	1727
+蚂	1720
+榔	1719
+圪	1707
+芭	1703
+犇	1702
+歆	1694
+蝉	1689
+窨	1687
+奏	1684
+泪	1676
+稔	1676
+谟	1675
+愉	1673
+舵	1670
+硐	1667
+蒿	1666
+郦	1666
+糙	1662
+踩	1646
+嶷	1636
+逼	1631
+羿	1628
+蜘	1626
+胖	1623
+胪	1614
+崴	1611
+壬	1603
+彼	1603
+牲	1600
+肚	1596
+珈	1596
+叔	1596
+珉	1588
+癿	1586
+郾	1585
+茎	1583
+枧	1580
+琚	1580
+疤	1572
+洹	1569
+绮	1562
+弛	1559
+纫	1557
+懒	1536
+圭	1533
+呆	1525
+苴	1520
+埗	1518
+骜	1515
+秧	1507
+敞	1506
+隘	1506
+匣	1505
+羟	1497
+漩	1496
+榫	1496
+绶	1492
+柬	1492
+纂	1488
+乎	1488
+疹	1487
+仝	1484
+侣	1481
+宠	1476
+辞	1474
+吓	1473
+恬	1468
+忌	1466
+旸	1464
+尸	1462
+膛	1458
+芊	1458
+毽	1451
+柞	1447
+冼	1444
+蟹	1444
+槛	1440
+喧	1434
+渲	1427
+翱	1421
+泖	1414
+斧	1412
+赭	1411
+妨	1407
+胫	1407
+苔	1407
+塅	1406
+璋	1398
+谤	1394
+诽	1394
+釉	1392
+睛	1387
+耸	1378
+漪	1376
+钡	1375
+榈	1374
+穹	1367
+羚	1360
+哗	1359
+瞩	1354
+骊	1354
+寥	1349
+邾	1349
+沔	1347
+逍	1346
+祚	1342
+龈	1342
+笃	1338
+佃	1336
+垠	1336
+昵	1332
+豚	1327
+盯	1324
+査	1315
+敌	1315
+瑙	1313
+鬃	1312
+窒	1311
+沌	1303
+桉	1302
+埚	1298
+垡	1295
+町	1294
+桷	1292
+颢	1290
+臀	1288
+崮	1285
+佗	1285
+毗	1284
+泼	1278
+篆	1278
+舫	1278
+蕙	1274
+绅	1271
+宵	1270
+馥	1267
+懂	1263
+惜	1259
+搞	1259
+坩	1251
+阚	1246
+洣	1243
+妊	1241
+璨	1241
+谛	1239
+钲	1236
+畚	1235
+锄	1235
+丕	1234
+腋	1232
+葭	1232
+昙	1230
+晁	1228
+苷	1227
+鹄	1224
+妤	1221
+黛	1217
+瀍	1216
+砻	1214
+饵	1214
+捡	1211
+堑	1206
+笙	1205
+躺	1205
+缎	1203
+扉	1203
+蚝	1201
+轲	1199
+茸	1199
+窟	1198
+笠	1198
+趸	1196
+浍	1189
+烙	1186
+蹬	1185
+氡	1182
+钵	1181
+丨	1177
+峤	1177
+敢	1176
+焜	1175
+鬼	1174
+砦	1171
+嵛	1169
+涓	1166
+垓	1164
+洵	1163
+洸	1162
+湫	1159
+闰	1158
+酃	1157
+篁	1157
+臼	1156
+瀑	1152
+帛	1151
+埝	1150
+嗜	1148
+挚	1144
+呢	1143
+芍	1143
+滂	1142
+戬	1139
+膈	1133
+苓	1130
+鸳	1127
+哇	1126
+荃	1125
+雒	1123
+拯	1120
+霓	1119
+隰	1118
+缨	1109
+蘑	1108
+汲	1101
+葆	1101
+夺	1098
+奢	1098
+芥	1098
+亓	1096
+愚	1090
+蛀	1089
+缉	1088
+褐	1088
+亢	1087
+脲	1084
+菩	1083
+俬	1083
+榉	1083
+褂	1082
+蛾	1081
+斛	1080
+啶	1079
+筏	1077
+嗣	1075
+瘘	1070
+筜	1066
+隍	1065
+筼	1061
+暮	1057
+曰	1050
+飒	1048
+瞭	1048
+淬	1043
+茭	1042
+豉	1042
+怎	1041
+涅	1039
+摹	1039
+掸	1037
+冢	1036
+湃	1034
+苫	1032
+镧	1030
+劼	1030
+嵇	1029
+栎	1028
+叩	1027
+俤	1025
+躯	1023
+疝	1020
+崛	1020
+枸	1017
+羲	1017
+蛙	1011
+囚	1009
+菠	1009
+拇	1007
+琵	1006
+舸	1004
+聿	1004
+镔	1003
+擂	1003
+橇	1000
+塱	999
+觅	997
+孪	996
+泻	995
+惰	993
+诋	993
+宦	992
+枹	992
+艮	990
+粑	989
+檩	989
+靛	988
+闷	988
+猕	985
+脓	980
+颉	980
+稼	980
+荀	973
+腱	972
+庾	971
+戛	970
+鋆	969
+丶	968
+苞	967
+茆	963
+珅	961
+颛	959
+闾	957
+帷	957
+瓢	956
+娠	953
+阑	950
+嗨	947
+莴	943
+亥	942
+楹	938
+嫦	937
+螨	936
+牯	934
+苕	932
+筇	931
+铿	930
+蛮	930
+弼	929
+镗	929
+樽	928
+啉	928
+涣	928
+脖	924
+歉	924
+酐	917
+呷	916
+钗	916
+缇	915
+廿	913
+砝	912
+爷	910
+诵	908
+窃	908
+钽	906
+桡	904
+泐	904
+玑	903
+濑	902
+驮	899
+脾	898
+雉	898
+堪	897
+瞰	894
+莽	893
+莼	893
+熨	890
+涔	888
+斓	887
+慷	885
+嫣	884
+喝	883
+鹗	880
+仡	880
+跆	880
+琢	879
+炀	876
+氐	876
+垴	875
+菡	873
+瑟	873
+剁	871
+炽	871
+沨	866
+雲	862
+赘	861
+琐	860
+荤	860
+肱	859
+葳	859
+奠	858
+凇	848
+沩	848
+虢	848
+烛	846
+濒	846
+祭	845
+逯	844
+淌	843
+馒	842
+傍	842
+卦	836
+佶	835
+蔷	833
+澈	833
+鲨	833
+洑	832
+暄	829
+叮	827
+祛	825
+槿	825
+铱	825
+桠	820
+籼	819
+朕	819
+趵	816
+咱	814
+俪	813
+呋	813
+鲢	812
+跷	811
+骖	810
+迤	808
+樑	808
+唾	807
+姥	806
+藩	803
+滹	802
+邰	801
+笆	801
+漷	800
+偲	800
+砺	800
+聯	799
+泞	796
+矾	794
+筝	793
+溃	793
+濯	790
+咕	790
+鞭	789
+阱	789
+內	789
+畋	786
+巅	784
+啟	784
+乍	783
+憩	783
+鮀	783
+巿	783
+僵	779
+畏	778
+湍	777
+唛	777
+泠	777
+饺	775
+潦	774
+惊	772
+徂	771
+癀	769
+磬	766
+劈	766
+铍	762
+呵	762
+朦	762
+褶	759
+瑄	759
+瞄	757
+奂	757
+噻	756
+绰	756
+愁	755
+畹	751
+胨	750
+淏	749
+睑	747
+贇	747
+捧	745
+鲇	741
+嶂	741
+陬	738
+笪	738
+挽	736
+矗	734
+嫔	732
+腥	731
+孰	727
+抠	726
+霾	722
+挟	722
+羧	720
+枇	718
+崤	714
+侍	714
+蟾	713
+潆	711
+膦	709
+酪	708
+谣	707
+庶	705
+骚	705
+俸	702
+扯	701
+扼	700
+荪	700
+簇	699
+呐	693
+啊	692
+嗪	692
+恪	691
+剡	691
+倘	687
+稚	686
+溺	685
+杷	683
+铛	683
+骇	682
+澹	680
+琏	679
+镰	678
+喃	672
+昝	671
+聆	670
+雌	669
+嫁	668
+滆	667
+揉	667
+痪	666
+琤	665
+誓	663
+锝	662
+垮	661
+墒	661
+扔	660
+糜	659
+菖	659
+宬	657
+洙	655
+跤	653
+鸬	652
+雹	652
+锨	651
+绚	649
+憬	648
+骉	645
+斩	639
+薮	635
+僧	635
+湧	633
+枭	632
+棺	629
+粹	629
+颚	628
+衅	626
+箅	625
+吲	624
+俩	623
+佼	622
+堌	621
+谏	619
+谎	619
+炕	619
+峃	618
+號	617
+苛	617
+髁	617
+圻	617
+怠	614
+萘	614
+墘	611
+潋	610
+痫	608
+崀	606
+茈	605
+骶	604
+沦	604
+淯	601
+皎	600
+澥	600
+鹦	599
+徳	598
+哚	598
+刍	597
+琮	597
+稞	595
+孺	595
+沮	595
+狸	593
+凼	593
+焓	592
+隗	589
+穷	587
+兮	587
+髂	585
+馕	583
+钺	582
+阊	582
+溇	581
+亟	581
+溱	578
+澴	577
+癣	577
+茁	576
+痒	575
+谓	575
+嵬	574
+拂	574
+裔	572
+嘛	571
+栢	570
+譬	570
+夼	567
+腌	566
+菉	564
+舷	564
+荞	564
+喹	563
+炖	563
+敛	562
+溥	561
+頔	560
+甪	560
+菅	556
+爸	555
+鳙	554
+苜	553
+铼	553
+酥	553
+碛	551
+痘	551
+擀	551
+砜	551
+邪	550
+夔	549
+眷	549
+楣	547
+芨	547
+箫	546
+憨	545
+岜	545
+咯	544
+嗅	542
+诿	542
+癫	541
+桅	539
+渫	538
+眸	537
+嬴	537
+沄	536
+彧	536
+蹉	535
+诞	535
+钿	534
+恳	532
+疱	532
+忍	531
+旖	527
+捻	526
+拴	526
+璠	525
+嵋	525
+壆	524
+厍	523
+涠	523
+凫	523
+锶	522
+镱	522
+趾	521
+镒	520
+螳	519
+镖	519
+勰	518
+颀	516
+笏	515
+碴	515
+滠	513
+乞	513
+鸢	512
+芩	512
+腩	511
+馀	511
+绎	511
+徘	510
+钕	509
+钏	506
+怕	506
+朽	504
+娉	504
+拢	503
+奴	500
+奓	500
+蓿	499
+镓	499
+藉	498
+徊	498
+崟	497
+酩	497
+蘭	497
+闳	497
+垵	494
+晞	493
+札	493
+檫	492
+哄	492
+茬	491
+盥	491
+漉	491
+贲	491
+豸	490
+硖	489
+雏	489
+糠	489
+窜	488
+峦	488
+熵	487
+哺	487
+嗓	487
+瑀	487
+苒	487
+鼾	486
+帼	485
+堽	484
+萎	484
+焘	484
+襟	482
+磕	482
+羔	481
+豁	481
+呕	481
+窠	480
+虬	479
+纾	478
+顽	478
+薏	478
+榛	478
+袭	476
+氖	476
+苄	475
+疙	474
+鹉	474
+邴	473
+怖	473
+贠	472
+旎	471
+戌	471
+翥	470
+鳅	468
+拎	468
+蚬	468
+囤	468
+陋	467
+镕	466
+粵	466
+葑	464
+藜	463
+羁	463
+蹇	462
+箩	461
+鳝	460
+劭	460
+挝	459
+崭	458
+掷	457
+喘	456
+瘩	454
+拙	453
+煅	453
+肼	452
+昼	451
+辋	451
+颊	450
+砲	449
+粕	449
+颞	448
+驴	446
+睾	446
+璀	446
+髙	446
+滥	445
+迩	445
+倬	445
+堎	444
+涑	443
+镛	440
+褒	440
+笨	439
+邨	438
+砬	438
+诣	435
+铯	433
+訾	431
+滟	431
+岀	429
+暾	429
+鲈	429
+蛳	429
+撇	429
+珮	425
+蔻	425
+笕	425
+礐	424
+幺	424
+铟	422
+鹚	422
+砥	422
+蜻	421
+菘	421
+昉	419
+橼	418
+荚	418
+蓖	417
+堇	415
+粽	415
+郗	415
+蛤	412
+棪	411
+庹	410
+丑	410
+铉	410
+茧	410
+麋	409
+硇	408
+毋	408
+颠	407
+赌	407
+雩	406
+晧	405
+鹍	405
+蟆	404
+菀	404
+殴	403
+汭	402
+剃	401
+驯	401
+峯	401
+铌	401
+嫱	399
+哔	398
+绊	398
+逵	397
+铐	397
+搐	397
+罍	396
+盅	395
+熬	395
+炘	395
+袂	393
+蹦	393
+莪	393
+仵	392
+龛	392
+楂	391
+扮	390
+蟒	390
+氘	390
+鎏	389
+稜	388
+韫	387
+槌	386
+篢	386
+郇	386
+瑢	382
+啮	382
+瀼	382
+辟	381
+噁	381
+徇	380
+孛	380
+掀	379
+瑭	379
+糟	378
+濡	377
+舂	377
+棟	377
+镫	376
+痔	375
+叽	373
+怿	373
+矣	373
+漖	373
+惺	372
+痤	372
+绽	371
+裝	371
+氿	370
+坌	370
+甽	370
+悍	369
+摒	369
+豌	368
+绢	368
+聃	367
+學	367
+鄣	366
+猷	366
+粳	365
+砣	363
+夸	363
+藿	361
+祐	361
+嫂	360
+戍	360
+蜱	360
+侏	360
+碓	359
+卑	358
+犊	358
+岽	356
+茚	356
+癸	355
+硷	354
+镟	353
+埌	353
+栀	352
+電	351
+嫚	350
+庠	350
+烺	350
+飙	349
+珣	349
+悲	348
+傢	348
+蒽	347
+樨	347
+陲	346
+嘧	346
+疽	346
+桧	344
+郅	342
+吿	341
+魂	340
+钇	340
+渱	340
+憾	339
+鳍	339
+黍	338
+仗	338
+肴	337
+镲	336
+忞	335
+诰	334
+锲	333
+涩	333
+喱	333
+袄	332
+邈	329
+椹	328
+眺	328
+橦	328
+怪	327
+磙	326
+泃	325
+銶	324
+洧	323
+蕨	322
+價	321
+郄	321
+鲫	321
+厥	320
+遐	320
+跋	320
+舀	320
+嬉	319
+挨	318
+轸	318
+酞	317
+琯	317
+宥	317
+昏	315
+浄	315
+洄	314
+汫	314
+堠	314
+腮	313
+艰	312
+窊	312
+诘	312
+钫	311
+涢	309
+禀	309
+戟	309
+挎	309
+捍	307
+疡	307
+淙	307
+蚣	307
+哮	307
+铥	307
+苋	306
+麗	305
+遒	305
+蓁	304
+紊	303
+呱	302
+焯	299
+鹞	299
+钯	299
+囡	298
+骡	298
+铋	297
+卅	297
+埒	296
+徵	296
+嵝	295
+嘟	295
+睫	295
+爻	294
+捺	294
+骝	294
+骈	293
+螯	292
+赓	292
+锃	292
+恼	292
+瘢	291
+嫘	291
+裳	291
+岣	291
+缚	291
+轰	290
+嫒	289
+蜓	289
+磜	289
+籁	288
+哌	288
+芘	288
+蕰	288
+寐	287
+葩	287
+扦	285
+財	285
+鲘	283
+蝗	283
+玎	283
+钶	282
+蔼	282
+坷	282
+洎	281
+啫	281
+茯	280
+粼	280
+贱	279
+楮	277
+煨	277
+僚	277
+渼	277
+荄	276
+窿	276
+馅	274
+東	273
+栟	273
+飓	273
+痧	273
+虒	271
+渥	271
+遨	271
+抒	270
+劢	269
+椤	269
+炅	269
+汈	268
+畛	266
+隼	266
+娆	266
+羹	265
+涕	265
+姊	265
+蒌	265
+槭	264
+璘	264
+機	264
+腓	264
+脯	264
+驩	263
+匮	263
+妞	263
+岈	262
+榻	262
+灈	261
+栌	260
+惑	260
+虓	259
+妖	258
+螟	258
+浛	258
+簕	258
+牤	257
+鹫	257
+鱿	257
+砭	257
+凋	256
+珐	255
+锘	255
+诒	255
+洈	254
+牺	254
+谍	254
+飏	254
+楸	254
+龋	253
+浬	252
+疣	251
+磻	251
+祟	251
+柅	251
+偌	250
+堯	249
+桀	249
+夭	248
+玠	248
+噢	247
+柽	246
+馗	246
+宓	245
+嵖	245
+浃	245
+請	244
+規	244
+钹	243
+幔	243
+増	242
+驸	242
+郚	241
+莅	241
+谆	241
+萬	241
+謇	241
+蔄	240
+茴	240
+翮	239
+渴	238
+崾	238
+铑	237
+剜	237
+阖	236
+焗	236
+罔	235
+塄	235
+疟	235
+叹	234
+铤	234
+報	233
+億	232
+蚤	232
+竑	231
+琎	231
+揿	230
+孱	230
+龍	230
+轭	230
+瑗	230
+琬	230
+廠	230
+谕	229
+酊	228
+佚	227
+梆	227
+逛	227
+暧	227
+塝	226
+耻	226
+湔	226
+昫	226
+镝	225
+顼	225
+劫	225
+鹂	224
+獐	224
+嫄	224
+陨	223
+蒯	223
+绗	223
+揣	223
+甦	223
+蕃	222
+蘸	222
+挫	222
+俎	221
+蹊	220
+螭	220
+牍	220
+葺	219
+虏	219
+廻	219
+榧	219
+廍	218
+瓒	218
+荏	218
+翕	217
+醪	217
+硌	217
+骢	217
+霈	217
+骧	217
+弶	216
+坔	216
+俶	216
+場	216
+哒	215
+扪	214
+鲷	213
+墁	213
+碉	213
+炙	213
+桕	213
+黉	213
+堀	212
+沺	212
+珥	211
+簪	211
+爨	211
+噜	211
+巳	211
+芪	211
+湉	210
+萼	210
+蜊	210
+塆	210
+獭	209
+澔	207
+锵	207
+榀	207
+梽	207
+弢	206
+圬	206
+痱	206
+迄	205
+厄	205
+坵	205
+髻	204
+蚜	204
+绀	203
+叄	202
+峭	202
+錾	202
+椑	202
+妩	201
+萁	201
+腭	200
+椽	200
+咚	200
+啬	200
+瑁	199
+垲	199
+胍	199
+胛	199
+袢	198
+璎	197
+孳	197
+蜈	197
+泔	196
+筎	196
+坼	196
+痼	195
+譞	195
+寡	194
+佲	194
+悖	194
+撷	193
+滍	193
+肟	193
+髌	191
+剅	191
+诠	191
+痣	191
+惹	191
+瞎	190
+湓	189
+岕	189
+聖	189
+杵	189
+珪	189
+輋	188
+崞	188
+苌	188
+祀	188
+芫	188
+泱	188
+義	188
+枡	187
+瘴	187
+迴	187
+铕	187
+乖	186
+湴	186
+捶	186
+篙	186
+忪	185
+骂	185
+躲	185
+倞	185
+屺	185
+垤	184
+惧	184
+邙	184
+冥	184
+俚	184
+氚	183
+鄚	183
+赊	183
+杓	183
+瓿	183
+囯	182
+麂	182
+躁	181
+镨	181
+搂	180
+躬	180
+仉	180
+質	180
+铊	179
+埫	178
+祸	177
+芾	177
+玢	177
+娑	177
+泯	176
+辫	176
+秃	176
+汌	175
+辔	175
+痹	175
+镦	175
+崯	175
+昨	175
+氾	174
+晩	174
+悼	174
+玟	174
+琍	174
+鲟	174
+旴	173
+吖	173
+烝	173
+箴	172
+烔	172
+岿	172
+笊	172
+掠	171
+垈	171
+竤	171
+炑	171
+讴	171
+蛔	170
+酋	170
+備	170
+佺	170
+恂	169
+呜	169
+琙	169
+邃	169
+僻	168
+漋	168
+殳	168
+虱	168
+禤	167
+項	167
+庇	167
+蓤	167
+棹	167
+測	167
+泫	167
+彗	166
+曈	166
+鄌	166
+珹	165
+舲	165
+昪	165
+鹑	165
+蜚	165
+裆	164
+矽	164
+咔	163
+屲	163
+嚼	163
+眭	163
+標	162
+瑷	162
+嵯	162
+焌	162
+箦	162
+佧	162
+鹧	162
+秽	161
+鳊	161
+潴	161
+汩	161
+粲	160
+灜	160
+龑	160
+桲	160
+铒	159
+碲	159
+歺	158
+墉	158
+呗	157
+現	157
+哩	157
+懈	157
+埜	157
+澉	156
+鹌	156
+眶	156
+臺	156
+铳	155
+垧	155
+尓	155
+廛	155
+鼐	155
+筻	155
+呙	154
+溦	154
+罢	154
+浭	154
+阪	154
+單	153
+倭	153
+貂	153
+疯	152
+辘	152
+祊	152
+她	151
+枳	151
+舾	151
+蜥	151
+竜	151
+時	150
+鸪	150
+玳	150
+筲	149
+歹	149
+碇	149
+撼	149
+嘡	149
+胧	148
+莳	148
+雳	148
+蒹	148
+恽	148
+峣	148
+珽	147
+凛	147
+杲	147
+壑	147
+吩	147
+凃	146
+動	146
+萸	146
+鲊	146
+娓	146
+绯	146
+傻	146
+汜	146
+萩	146
+麝	145
+镪	145
+憧	145
+迂	145
+塭	145
+岂	145
+戢	145
+趁	144
+莊	144
+浧	144
+缷	143
+壓	143
+赑	143
+進	143
+噬	143
+勍	143
+吼	142
+鳃	142
+蜍	142
+哦	142
+椴	142
+為	142
+鹮	141
+璴	141
+燮	141
+鳗	141
+呤	140
+苡	140
+徙	140
+荭	140
+缑	140
+涮	139
+發	138
+谧	138
+僰	138
+毳	138
+郪	138
+憎	138
+潓	137
+塾	137
+玹	137
+屛	137
+锬	137
+铡	136
+舣	136
+镣	136
+吁	136
+辈	136
+崚	136
+缗	136
+羞	136
+荘	136
+瘾	135
+堍	135
+囿	135
+脒	135
+呀	135
+嬿	135
+漈	135
+焖	135
+褔	134
+甾	134
+嶝	134
+苎	134
+楝	134
+皤	134
+哭	133
+隈	133
+娈	133
+镏	133
+貳	133
+俣	133
+麸	133
+琇	133
+崄	132
+仃	132
+粦	132
+湲	132
+吵	132
+垾	132
+馍	131
+吝	131
+藓	131
+冚	131
+杈	131
+雎	131
+镙	131
+镌	131
+喵	131
+幂	130
+頻	130
+爹	130
+娶	130
+镯	130
+蛉	130
+趴	130
+鲶	130
+倮	129
+蚯	129
+蝎	129
+颡	129
+榷	129
+缦	129
+撂	129
+痴	128
+斫	128
+谜	128
+诃	127
+酗	127
+捅	127
+贬	127
+統	127
+錱	126
+缥	126
+诫	126
+蚓	126
+啰	126
+楦	126
+強	125
+沓	125
+鲵	125
+辗	125
+铈	125
+潔	125
+岖	125
+铷	125
+稂	125
+鳖	124
+惢	124
+哆	124
+蒺	123
+吏	123
+澛	123
+鮓	123
+腘	123
+瞪	122
+辱	122
+傩	122
+擒	122
+绫	122
+倴	122
+硃	122
+轱	122
+嘶	121
+蛎	121
+赳	121
+孖	121
+毎	121
+椐	121
+鹁	121
+兀	120
+擘	120
+扛	120
+數	120
+維	120
+橫	119
+桄	119
+鋼	119
+姮	119
+復	119
+鑑	119
+櫈	118
+琟	118
+翩	118
+鲆	118
+豇	118
+凎	117
+膊	117
+髯	117
+軸	117
+悯	117
+歩	116
+仨	116
+煋	116
+甯	116
+溁	116
+哉	116
+坮	116
+嘈	115
+鲖	115
+稱	115
+坣	115
+沘	115
+甑	114
+痂	113
+艉	113
+娲	113
+赜	113
+頭	113
+癜	113
+偕	112
+弁	112
+胳	111
+谞	111
+翚	111
+恵	111
+鲹	111
+冮	111
+圐	110
+資	110
+嵴	110
+谄	110
+屡	110
+帏	109
+赪	109
+垩	109
+桫	109
+弩	109
+奘	109
+倜	108
+猝	108
+巉	108
+谬	108
+圙	108
+彐	108
+鸰	107
+眨	107
+墚	107
+俨	107
+褀	107
+跪	107
+袤	107
+楒	107
+蜒	107
+浞	106
+匯	106
+睐	106
+酂	106
+噗	106
+茱	106
+嶺	106
+蚶	106
+繆	106
+郐	106
+腑	106
+剌	105
+骦	105
+帶	105
+浤	105
+屎	105
+钭	105
+黃	105
+坜	105
+幄	104
+弸	104
+匈	104
+僮	104
+環	103
+逹	103
+垯	103
+鈺	103
+拈	102
+虛	102
+蜿	102
+膘	101
+漹	101
+饿	101
+痢	101
+蜇	101
+晙	101
+乜	101
+郞	101
+鹳	101
+浡	101
+幛	101
+螃	100
+屾	100
+涎	100
+匆	100
+無	99
+峧	99
+咛	99
+別	99
+蛹	99
+荼	99
+蚨	99
+洇	99
+湜	99
+務	99
+礤	98
+瑆	98
+挞	98
+線	98
+瑅	98
+劦	98
+贶	97
+爿	97
+淸	97
+渺	97
+晅	97
+紙	97
+囵	97
+國	97
+饯	97
+芎	96
+翃	96
+姁	96
+囫	96
+畦	96
+溫	96
+婳	96
+猩	96
+阗	95
+崽	95
+誌	95
+礴	95
+蚴	94
+氣	94
+燏	94
+弇	94
+砵	93
+蕤	93
+崑	93
+斟	93
+犋	93
+镑	92
+芡	92
+闩	92
+區	92
+溟	92
+杧	92
+檑	91
+贛	91
+唁	91
+悄	91
+墕	91
+赃	91
+嶙	91
+迢	90
+蛸	90
+胙	90
+篪	90
+潏	90
+桤	89
+霰	89
+艏	89
+坋	89
+巯	89
+燈	89
+慨	88
+黒	88
+瑝	88
+筵	88
+阄	88
+稅	88
+際	88
+姨	88
+採	88
+葶	88
+嘌	88
+胄	88
+涒	88
+夙	88
+啭	87
+觐	87
+堕	87
+倢	87
+傑	87
+砰	87
+寂	87
+蘋	87
+幵	87
+窘	86
+儲	86
+穰	86
+蘅	86
+經	86
+俘	86
+給	86
+翦	86
+塚	86
+晤	85
+聍	85
+矯	85
+膠	85
+皕	85
+罾	85
+藴	85
+運	85
+耵	84
+鲛	84
+翾	84
+氪	84
+跻	84
+醌	84
+赎	84
+磡	84
+钚	84
+狩	84
+珵	84
+稗	84
+鸶	83
+妺	83
+熺	83
+铪	83
+蝠	83
+個	83
+奡	83
+皙	82
+迓	82
+妲	82
+蠊	82
+頌	82
+車	82
+載	82
+氹	82
+啥	82
+蹭	82
+跸	82
+畬	81
+輸	81
+跖	81
+佀	81
+蓢	80
+枥	80
+皞	80
+馄	80
+滃	80
+瑧	80
+烊	80
+桴	80
+篾	80
+晌	80
+鱻	80
+彥	80
+伲	80
+灡	80
+墊	79
+耍	79
+舢	79
+帯	79
+饨	79
+榃	79
+贽	79
+颏	79
+锜	79
+蟥	78
+瘀	78
+朳	78
+蛭	78
+祗	78
+華	78
+蝙	78
+辂	78
+搽	78
+浥	78
+觀	77
+辇	77
+罹	77
+镬	77
+亊	77
+魄	76
+镠	76
+炤	76
+羰	76
+贼	76
+噸	76
+琻	76
+涴	76
+诤	76
+嗄	76
+铚	75
+诩	75
+缕	75
+橛	75
+閥	75
+巜	75
+韻	75
+妫	74
+檗	74
+鳜	74
+奀	74
+玙	74
+屁	74
+畎	74
+淫	74
+弭	73
+昮	73
+嚣	73
+滧	73
+雙	73
+荇	73
+書	73
+轵	73
+順	73
+蓋	72
+鞲	72
+彷	72
+腧	72
+浰	72
+恙	72
+豊	72
+鞅	72
+嘿	72
+鞴	71
+糍	71
+涸	71
+挣	71
+掰	71
+焮	71
+鎰	71
+吒	71
+妪	71
+長	71
+瘪	71
+橹	71
+钍	71
+揆	71
+袆	70
+莜	70
+岞	70
+勖	70
+園	70
+滓	70
+詠	70
+脿	69
+璁	69
+扆	69
+熳	69
+咋	69
+仞	69
+侮	68
+蕗	68
+奧	68
+璩	68
+昳	68
+佽	68
+須	68
+霭	68
+湑	68
+缜	67
+暇	67
+鋐	67
+潽	67
+飨	67
+侥	67
+栲	67
+肓	67
+啃	66
+俠	66
+鞣	66
+烆	66
+菟	66
+設	66
+翙	66
+淓	66
+蘖	66
+惇	66
+袱	66
+剿	66
+埼	65
+窈	65
+赚	65
+欹	65
+恣	65
+呦	65
+绦	65
+暘	65
+戥	65
+杼	64
+間	64
+咇	64
+圹	64
+缫	64
+閤	64
+眀	64
+哙	64
+捎	64
+蛆	64
+殓	63
+蜃	63
+關	63
+柁	63
+锢	63
+饥	63
+廒	63
+枰	63
+迥	63
+怜	63
+碼	63
+搋	63
+锞	62
+逮	62
+滈	62
+鸵	62
+岧	62
+茼	62
+茏	62
+湕	62
+換	62
+級	62
+霅	61
+衠	61
+厩	61
+條	61
+筆	61
+嗮	61
+轼	61
+證	61
+墈	61
+缄	61
+诀	61
+撩	61
+囟	61
+俵	61
+舔	61
+圏	61
+岌	60
+凶	60
+瀂	60
+椟	60
+堉	60
+対	60
+佴	60
+挹	60
+體	60
+睟	60
+哃	59
+岵	59
+臬	59
+盤	59
+熀	59
+掐	59
+購	59
+佾	59
+業	59
+廪	59
+製	58
+錧	58
+祾	58
+刈	58
+葚	58
+酝	58
+钋	58
+廨	58
+犸	58
+胴	58
+肜	57
+抟	57
+蔫	57
+悌	57
+膑	57
+儀	57
+記	57
+緑	57
+骸	57
+旃	56
+堼	56
+棆	56
+玏	56
+缶	56
+溋	56
+俑	56
+蚪	56
+茔	56
+栝	56
+鳕	56
+熱	56
+蓆	56
+銘	55
+淩	55
+闱	55
+衩	55
+苘	55
+溏	55
+煕	55
+畿	55
+檢	55
+朸	55
+氓	55
+愣	54
+亐	54
+兩	54
+镡	54
+硊	54
+匪	54
+荊	54
+漭	54
+後	54
+簰	54
+蔸	54
+掛	54
+暐	54
+喋	54
+鸷	54
+苖	54
+舁	54
+睁	54
+枨	53
+傳	53
+猬	53
+柸	53
+眯	53
+埤	53
+蛏	53
+鮋	53
+燿	53
+錶	53
+倉	53
+阇	52
+缵	52
+險	52
+喽	52
+侑	52
+翬	52
+窍	52
+颋	52
+罂	52
+纻	52
+酣	52
+礅	52
+攒	52
+苣	52
+烶	51
+猿	51
+繫	51
+冋	51
+圌	51
+炟	51
+鎮	51
+钆	51
+墻	51
+嘻	51
+蒡	51
+陞	51
+锺	50
+葸	50
+遁	50
+寀	50
+鸫	50
+剐	50
+滿	50
+荍	50
+衿	50
+狙	50
+鲩	50
+啼	50
+飛	49
+蝌	49
+槲	49
+開	49
+監	49
+鲂	49
+珰	49
+暹	49
+荑	49
+磑	49
+汆	49
+風	49
+煞	49
+烩	49
+韡	49
+梱	49
+缬	49
+岠	49
+纰	48
+坬	48
+檔	48
+峘	48
+帙	48
+缌	48
+胗	48
+睹	48
+掣	48
+摞	48
+衖	48
+俾	48
+瑯	48
+鲳	48
+乂	48
+噴	47
+組	47
+镎	47
+锔	47
+郵	47
+饴	47
+試	47
+旮	47
+耩	47
+罄	46
+襻	46
+嘹	46
+蓑	46
+淜	46
+莛	46
+戗	46
+逑	46
+鄃	46
+逞	46
+锎	46
+鉻	46
+蜕	46
+鳟	46
+洨	46
+樂	46
+酢	46
+姹	46
+垇	45
+蔴	45
+實	45
+繇	45
+鹛	45
+瑨	45
+笈	45
+寳	45
+谴	45
+曩	45
+敝	45
+锏	45
+怨	45
+編	45
+摁	44
+叡	44
+秫	44
+肄	44
+狠	44
+壅	44
+牂	44
+濾	44
+郤	44
+伉	44
+骰	44
+闺	44
+惬	44
+霹	44
+萦	44
+蜞	44
+鲴	44
+牁	44
+婀	44
+鲌	44
+靜	44
+圓	44
+則	43
+殪	43
+澂	43
+莠	43
+薹	43
+愫	43
+甡	43
+锕	43
+萋	43
+荦	43
+筘	43
+鼋	43
+計	43
+侕	43
+潩	43
+忐	43
+哎	43
+俅	43
+珎	43
+堨	43
+蚧	43
+铦	42
+虅	42
+蛰	42
+桎	42
+敔	42
+檵	42
+細	42
+喾	42
+臵	42
+樯	42
+膺	42
+將	42
+門	42
+哓	42
+槠	42
+洢	42
+惨	42
+渉	42
+貿	42
+珖	42
+糞	41
+枷	41
+荠	41
+嗒	41
+庥	41
+铙	41
+芶	41
+孽	41
+腙	41
+莨	41
+肈	41
+啜	41
+劑	41
+鲽	41
+朏	41
+鲃	41
+棧	41
+镥	41
+菽	41
+陝	40
+蠹	40
+鋁	40
+铔	40
+變	40
+慌	40
+録	40
+僖	40
+逝	40
+踵	40
+寬	40
+粜	40
+媲	40
+揪	40
+唢	40
+逅	40
+澞	39
+醍	39
+愛	39
+側	39
+冽	39
+槚	39
+葉	39
+伾	39
+慜	39
+袅	39
+忤	39
+悫	39
+嶍	39
+抺	39
+傧	39
+祤	39
+澌	39
+圴	39
+昴	39
+灋	39
+楻	39
+袪	39
+岡	39
+苊	39
+慑	39
+纛	39
+儆	39
+瓘	39
+喙	39
+梪	39
+瑱	39
+亁	38
+圖	38
+導	38
+昡	38
+淂	38
+侉	38
+拮	38
+蓠	38
+韮	38
+潲	38
+許	38
+晷	38
+護	38
+坽	38
+荨	38
+単	38
+甏	38
+苾	38
+丅	38
+俢	38
+悅	38
+赅	38
+貨	38
+匏	38
+蹼	38
+曁	37
+飬	37
+赡	37
+筌	37
+鏽	37
+剽	37
+钪	37
+秾	37
+迸	37
+嘀	37
+樓	37
+仄	37
+敉	37
+恻	37
+镢	37
+叱	37
+婓	37
+玗	37
+砒	37
+辍	37
+昄	36
+恨	36
+谙	36
+嘣	36
+煽	36
+犏	36
+榬	36
+嘏	36
+薤	36
+竦	36
+獾	36
+徑	36
+疌	36
+嗡	36
+話	36
+跺	36
+杳	36
+陛	35
+艭	35
+稣	35
+鴻	35
+胂	35
+岺	35
+皈	35
+對	35
+饸	35
+栊	35
+垆	35
+芴	35
+埙	35
+枉	35
+菪	35
+戽	35
+剀	35
+叟	35
+塊	35
+凱	35
+煖	35
+瞧	35
+鲀	34
+熤	34
+納	34
+過	34
+畝	34
+簖	34
+锠	34
+沆	34
+穂	34
+竽	34
+塨	34
+俫	34
+磏	34
+癞	34
+麾	34
+棤	34
+嗦	34
+诂	34
+荛	34
+應	34
+赧	34
+猁	34
+猞	34
+秝	34
+姺	34
+垞	33
+诶	33
+騰	33
+骛	33
+匍	33
+傥	33
+禇	33
+芈	33
+窎	33
+迮	33
+愷	33
+仂	33
+紅	33
+猄	33
+愽	33
+丼	33
+頫	33
+蕖	32
+霡	32
+鳯	32
+鐵	32
+鬲	32
+劬	32
+貮	32
+確	32
+诧	32
+點	32
+瞌	32
+屬	32
+昦	32
+羑	32
+滙	32
+厷	32
+冑	32
+卮	32
+尙	32
+辦	31
+爰	31
+呛	31
+嶕	31
+庫	31
+竝	31
+钌	31
+潥	31
+鋒	31
+逦	31
+軌	31
+離	31
+虻	31
+艿	31
+遄	31
+視	31
+處	31
+興	31
+轳	31
+調	31
+蚺	31
+俍	31
+貝	30
+堔	30
+鵾	30
+詳	30
+笥	30
+囍	30
+靡	30
+泣	30
+鸹	30
+谵	30
+鬓	30
+顔	30
+瘙	30
+張	30
+僅	30
+湙	30
+嗯	30
+昧	30
+埯	30
+簌	30
+惫	30
+苝	30
+菔	29
+髭	29
+曽	29
+灃	29
+馋	29
+弾	29
+類	29
+愤	29
+嚏	29
+銲	29
+売	29
+鞑	29
+窕	29
+楗	29
+倦	29
+峋	29
+暻	29
+谚	29
+燧	29
+榮	29
+轾	29
+骟	29
+捭	29
+遊	29
+蕻	29
+鯵	29
+奤	29
+耷	29
+媞	29
+農	29
+攝	29
+萂	29
+議	29
+骺	29
+甁	29
+绺	28
+軟	28
+喏	28
+搀	28
+咐	28
+腴	28
+栳	28
+嬛	28
+虐	28
+凊	28
+釡	28
+偻	28
+桖	28
+拗	28
+茑	28
+矜	28
+枊	28
+費	28
+吮	28
+苁	28
+呻	28
+畖	28
+唧	28
+總	28
+哀	28
+逖	27
+噤	27
+祼	27
+吱	27
+佈	27
+幹	27
+菓	27
+扥	27
+爾	27
+庒	27
+阝	27
+芰	27
+摧	27
+玕	27
+誊	27
+嚎	27
+掬	27
+禛	27
+玘	27
+拄	27
+槊	27
+連	27
+侓	27
+淨	27
+锖	27
+汘	27
+闼	27
+禚	27
+煺	27
+參	27
+苲	27
+蟛	27
+釘	27
+邽	26
+鲑	26
+負	26
+徼	26
+羯	26
+咫	26
+碡	26
+驗	26
+抡	26
+優	26
+伢	26
+搻	26
+耱	26
+彈	26
+撖	26
+蝰	26
+尭	26
+鲒	26
+洚	26
+翯	26
+憶	26
+術	26
+壇	26
+痨	26
+罟	26
+鲎	26
+濎	26
+哼	26
+菴	26
+塥	26
+滗	26
+祢	26
+笫	25
+繪	25
+藐	25
+絲	25
+蓦	25
+幌	25
+磹	25
+椋	25
+瓠	25
+詢	25
+嫡	25
+擢	25
+佞	25
+缰	25
+玷	25
+酇	25
+猊	25
+跛	25
+蚩	25
+卟	25
+奸	25
+耈	25
+笸	25
+懃	25
+孑	25
+镈	25
+厮	24
+庑	24
+藍	24
+丠	24
+艽	24
+幡	24
+廣	24
+菻	24
+椪	24
+柵	24
+媖	24
+鲮	24
+綠	24
+棂	24
+愧	24
+縮	24
+伋	24
+槃	24
+埉	24
+牒	24
+荽	24
+邠	24
+纭	24
+骠	24
+桿	24
+晢	24
+忏	24
+湮	24
+約	24
+遆	24
+甍	24
+柰	24
+碈	24
+奄	23
+終	23
+庖	23
+憋	23
+遼	23
+悸	23
+谒	23
+邊	23
+稹	23
+氵	23
+鏖	23
+糸	23
+笮	23
+畯	23
+觃	23
+洌	23
+襞	23
+宮	23
+赝	23
+潖	23
+炆	23
+額	23
+崐	23
+裢	23
+栆	23
+結	23
+圧	23
+沬	23
+玚	23
+泇	23
+嗥	22
+荸	22
+狒	22
+奭	22
+團	22
+跬	22
+鹣	22
+匐	22
+貉	22
+紹	22
+昃	22
+皝	22
+濫	22
+棻	22
+鍍	22
+楫	22
+诜	22
+專	22
+見	22
+茘	22
+诡	22
+樹	22
+哂	22
+裡	22
+蝮	22
+殇	22
+硗	22
+爃	22
+垻	22
+噉	22
+妄	22
+陣	22
+蔑	22
+叛	22
+齡	22
+嗖	22
+啵	22
+尬	22
+驫	22
+鲡	21
+讳	21
+柃	21
+鮠	21
+卲	21
+紋	21
+薜	21
+薷	21
+荖	21
+臾	21
+飲	21
+颙	21
+瓤	21
+銅	21
+圍	21
+崙	21
+镅	21
+員	21
+衎	21
+昰	21
+掲	21
+繞	21
+邂	21
+溎	21
+洳	21
+毙	21
+匕	21
+廈	21
+尴	21
+準	21
+収	21
+荩	21
+锪	21
+構	21
+賨	21
+纏	20
+鏡	20
+鲉	20
+乪	20
+裥	20
+顯	20
+達	20
+痿	20
+颓	20
+娼	20
+滾	20
+镳	20
+吳	20
+娙	20
+钐	20
+摺	20
+変	20
+僡	20
+咩	20
+塗	20
+菰	20
+藳	20
+舥	20
+嶲	20
+琲	20
+騄	20
+瓏	20
+酽	20
+俺	20
+枘	20
+戤	20
+茺	20
+銹	20
+氺	20
+滮	20
+嗞	20
+蚱	20
+褡	20
+竔	20
+祇	20
+唆	20
+偱	20
+艚	20
+俆	20
+棬	20
+叻	20
+屴	20
+芐	19
+蝽	19
+鳎	19
+膻	19
+颧	19
+佢	19
+艹	19
+咘	19
+鹓	19
+琀	19
+奿	19
+锽	19
+遺	19
+廢	19
+婼	19
+哟	19
+気	19
+埮	19
+銜	19
+倧	19
+頠	19
+帱	19
+琠	19
+鴒	19
+轫	19
+蟀	19
+與	19
+辄	19
+蛄	19
+歼	19
+儇	19
+烎	19
+磉	19
+愍	19
+陽	19
+蟋	19
+壸	18
+斷	18
+鍪	18
+幷	18
+跗	18
+嗑	18
+蹑	18
+埸	18
+谗	18
+樗	18
+種	18
+徨	18
+釰	18
+鯤	18
+盺	18
+杮	18
+廋	18
+崁	18
+拋	18
+錂	18
+铽	18
+兒	18
+瘸	18
+鼔	18
+棽	18
+姒	18
+決	18
+箸	18
+迀	18
+渰	18
+婶	18
+鏠	18
+岦	18
+丿	18
+訇	18
+迺	18
+蛊	18
+疚	18
+溝	18
+撸	18
+侈	18
+诲	18
+垕	18
+籣	18
+棰	18
+笳	18
+啓	18
+靬	18
+陔	17
+洴	17
+頂	17
+枼	17
+绾	17
+塢	17
+鹢	17
+蛐	17
+茳	17
+旄	17
+钘	17
+揺	17
+貟	17
+趄	17
+畓	17
+鎖	17
+沒	17
+矬	17
+瑸	17
+卾	17
+趙	17
+澫	17
+饹	17
+讶	17
+澐	17
+蛑	17
+锱	17
+怛	17
+伈	17
+糅	17
+鉄	17
+衕	17
+勝	17
+責	17
+夽	17
+蓸	17
+満	17
+耔	17
+蠖	17
+皛	17
+產	17
+溞	17
+祙	16
+暫	16
+諾	16
+苠	16
+鍵	16
+倔	16
+喔	16
+晥	16
+糁	16
+踌	16
+呇	16
+疁	16
+巣	16
+聩	16
+铢	16
+洰	16
+臃	16
+鎓	16
+歐	16
+瞾	16
+遛	16
+鉓	16
+酡	16
+橈	16
+钖	16
+呂	16
+悙	16
+骕	16
+樣	16
+鈜	16
+溵	16
+棁	16
+繕	16
+玓	16
+搔	16
+姌	16
+絡	16
+丐	16
+绉	16
+皑	16
+鉏	16
+獒	16
+說	16
+玭	16
+卣	15
+燔	15
+衮	15
+砩	15
+繘	15
+璛	15
+逶	15
+哝	15
+痊	15
+鄩	15
+噌	15
+靑	15
+鈉	15
+圜	15
+輪	15
+秕	15
+鉤	15
+婍	15
+疖	15
+鈦	15
+窩	15
+飖	15
+範	15
+堿	15
+泅	15
+饷	15
+冄	15
+問	15
+墡	15
+刂	15
+塵	15
+啷	15
+绨	15
+抉	15
+錦	15
+墺	15
+硍	15
+祧	15
+嗲	15
+豳	15
+溚	15
+朊	15
+寛	15
+玧	15
+蘇	15
+柢	15
+鮑	15
+傎	15
+蒨	15
+牮	15
+碹	15
+況	15
+欄	15
+锛	15
+刖	15
+偉	15
+谠	15
+揚	15
+荗	15
+攜	14
+轉	14
+镘	14
+澪	14
+蝈	14
+鄀	14
+緡	14
+稆	14
+氽	14
+牋	14
+篏	14
+詩	14
+硂	14
+铗	14
+枱	14
+叨	14
+嘤	14
+侖	14
+礽	14
+舖	14
+罴	14
+閉	14
+埊	14
+喈	14
+烬	14
+舅	14
+梃	14
+寶	14
+廬	14
+胯	14
+讬	14
+箜	14
+筸	14
+麿	14
+節	14
+蓊	14
+蕈	14
+浵	14
+蹴	14
+骔	14
+嵥	14
+講	14
+搧	14
+呃	14
+畫	14
+烑	14
+勀	14
+養	14
+餘	14
+審	14
+乸	14
+婿	14
+犟	14
+抿	13
+滉	13
+襁	13
+課	13
+佝	13
+篼	13
+帀	13
+貊	13
+狨	13
+枓	13
+啾	13
+罝	13
+栐	13
+妟	13
+曉	13
+渃	13
+禮	13
+淵	13
+鏈	13
+倌	13
+蕞	13
+恍	13
+袒	13
+咤	13
+圉	13
+軍	13
+砕	13
+狡	13
+跞	13
+砗	13
+堙	13
+萏	13
+钅	13
+夾	13
+裾	13
+硿	13
+瀹	13
+暠	13
+隩	13
+閣	13
+蟲	13
+埴	13
+缐	13
+嵄	13
+潾	13
+雜	13
+埏	13
+汹	13
+甙	13
+曵	13
+坉	13
+掮	13
+痈	13
+沵	13
+璿	13
+寔	13
+槜	13
+捩	13
+鲚	13
+嶶	13
+妘	13
+徴	13
+巻	13
+鉵	13
+铩	13
+薸	12
+夠	12
+弨	12
+瑔	12
+臊	12
+呖	12
+剎	12
+縫	12
+锒	12
+紐	12
+朩	12
+泜	12
+漴	12
+毐	12
+謝	12
+沋	12
+咭	12
+嚗	12
+枵	12
+籲	12
+枒	12
+叼	12
+栯	12
+躜	12
+櫻	12
+坧	12
+肫	12
+鶴	12
+玆	12
+澶	12
+仠	12
+蹻	12
+筢	12
+柝	12
+嘜	12
+簡	12
+碜	12
+硏	12
+剋	12
+俛	12
+酔	12
+飔	12
+乢	12
+渟	12
+伣	12
+頁	12
+讪	12
+惶	12
+偞	12
+稥	12
+坶	12
+歅	12
+芛	12
+係	12
+針	12
+沖	12
+厡	12
+嵕	12
+鉗	12
+倏	12
+霑	12
+瀞	12
+熖	12
+嗝	12
+鋅	12
+缯	12
+恃	12
+襦	12
+忒	12
+狍	12
+怦	12
+註	12
+腳	11
+妾	11
+銀	11
+顆	11
+淺	11
+佂	11
+菥	11
+唏	11
+簋	11
+壵	11
+澺	11
+楪	11
+臨	11
+圯	11
+讀	11
+緊	11
+滐	11
+骱	11
+炷	11
+偰	11
+扡	11
+寫	11
+陸	11
+澤	11
+觸	11
+燫	11
+廯	11
+郉	11
+碥	11
+轺	11
+鹼	11
+綜	11
+涳	11
+橢	11
+粿	11
+筀	11
+殉	11
+狀	11
+嚞	11
+減	11
+氶	11
+埲	11
+掊	11
+襯	11
+黠	11
+啁	11
+絢	11
+藠	11
+耨	11
+極	11
+筫	11
+嚓	11
+躇	11
+葙	11
+葏	11
+殘	11
+麇	11
+嶪	11
+嗵	11
+銷	11
+馚	11
+瀮	11
+蛲	11
+篝	11
+湭	11
+捂	10
+橧	10
+鄙	10
+栜	10
+蔣	10
+渂	10
+來	10
+顏	10
+葜	10
+汣	10
+箥	10
+嘢	10
+峛	10
+漤	10
+槺	10
+蓣	10
+題	10
+纮	10
+扞	10
+胼	10
+玶	10
+叵	10
+讽	10
+耄	10
+褓	10
+猖	10
+瀔	10
+氤	10
+暒	10
+崦	10
+捋	10
+诛	10
+菾	10
+譜	10
+盍	10
+铇	10
+菝	10
+吭	10
+瓌	10
+觞	10
+跶	10
+郃	10
+缒	10
+螈	10
+鲻	10
+屮	10
+佯	10
+泂	10
+怯	10
+鈡	10
+啪	10
+選	10
+耧	10
+鳘	10
+榑	10
+闇	10
+圫	10
+侒	10
+岄	10
+腆	10
+篌	10
+椆	10
+旒	10
+螅	10
+勗	10
+勻	10
+緣	10
+檺	10
+绱	10
+莀	10
+咡	10
+叒	10
+怵	10
+炻	10
+鸺	10
+杻	10
+镤	10
+鹜	10
+贋	10
+汚	10
+硋	9
+玿	9
+骓	9
+娌	9
+漢	9
+協	9
+峽	9
+彡	9
+蕺	9
+觜	9
+簏	9
+梣	9
+煸	9
+旯	9
+觎	9
+獎	9
+厎	9
+孓	9
+捲	9
+岬	9
+廸	9
+締	9
+掁	9
+玒	9
+菹	9
+訚	9
+芼	9
+膄	9
+鳢	9
+脘	9
+哞	9
+頜	9
+揖	9
+岎	9
+冇	9
+栒	9
+旼	9
+伵	9
+咎	9
+迨	9
+錡	9
+蘆	9
+辏	9
+轄	9
+寞	9
+杺	9
+鐡	9
+亍	9
+璆	9
+仸	9
+焬	9
+袷	9
+旳	9
+權	9
+叆	9
+抻	9
+珆	9
+窼	9
+饬	9
+鈎	9
+盉	9
+輌	9
+彎	9
+嘚	9
+晛	9
+哧	9
+孀	9
+酫	9
+尢	9
+昺	9
+夥	9
+氫	9
+網	9
+粬	9
+咆	9
+袓	9
+癔	9
+鋈	9
+鞥	9
+賀	9
+悃	9
+慊	9
+惚	9
+羮	9
+閱	9
+舭	9
+燄	9
+獗	9
+髦	9
+瘠	9
+貅	9
+燠	9
+箒	8
+瀨	8
+磦	8
+稊	8
+珝	8
+絕	8
+疥	8
+佥	8
+逬	8
+疔	8
+認	8
+侔	8
+頸	8
+翳	8
+虸	8
+囧	8
+恁	8
+晣	8
+夿	8
+琸	8
+迆	8
+頲	8
+鎔	8
+坫	8
+鍏	8
+鐘	8
+牽	8
+鲧	8
+齑	8
+膩	8
+蛴	8
+攪	8
+嫆	8
+沗	8
+潛	8
+層	8
+牳	8
+弍	8
+縠	8
+續	8
+薔	8
+谩	8
+爖	8
+肮	8
+灝	8
+撵	8
+懵	8
+斳	8
+枟	8
+闿	8
+冤	8
+鳛	8
+揩	8
+較	8
+苧	8
+裉	8
+薅	8
+貼	8
+酆	8
+殆	8
+椁	8
+咙	8
+伥	8
+鑚	8
+呰	8
+螬	8
+鼹	8
+荜	8
+忖	8
+礵	8
+盹	8
+喳	8
+鮰	8
+砉	8
+紗	8
+箧	8
+輛	8
+撅	8
+闐	8
+撝	8
+迖	8
+纨	8
+镆	8
+赦	8
+锫	8
+姫	8
+顣	8
+馫	8
+蝕	8
+錐	8
+悭	8
+寖	8
+飾	8
+饕	8
+窋	8
+蒞	8
+蚄	8
+沏	8
+閃	8
+苪	8
+臌	7
+語	7
+杄	7
+谼	7
+骍	7
+骙	7
+冐	7
+殃	7
+鱀	7
+蒴	7
+懸	7
+靯	7
+餮	7
+檞	7
+旵	7
+舻	7
+糨	7
+纡	7
+亞	7
+怔	7
+宖	7
+隨	7
+馬	7
+墰	7
+祷	7
+鄉	7
+暋	7
+婢	7
+淉	7
+隹	7
+绐	7
+擴	7
+乚	7
+繡	7
+呲	7
+豺	7
+冿	7
+咢	7
+礓	7
+侪	7
+妒	7
+肭	7
+埽	7
+堞	7
+柌	7
+鹇	7
+绡	7
+築	7
+滲	7
+纜	7
+鬣	7
+慆	7
+錫	7
+幾	7
+畑	7
+預	7
+帎	7
+邘	7
+馓	7
+嗉	7
+瘗	7
+創	7
+櫶	7
+燨	7
+饋	7
+戆	7
+仹	7
+晈	7
+撘	7
+禕	7
+愔	7
+純	7
+樁	7
+泙	7
+曐	7
+潟	7
+萜	7
+韾	7
+鵬	7
+熇	7
+瑩	7
+蔟	7
+阽	7
+遙	7
+醣	7
+踹	7
+厔	7
+囹	7
+滪	7
+螵	7
+鬟	7
+眞	7
+楿	7
+龆	7
+哽	7
+泚	7
+靈	7
+忑	7
+蘊	7
+愐	7
+豐	7
+嘲	6
+礳	6
+泩	6
+鞡	6
+眾	6
+硙	6
+駿	6
+碃	6
+瀜	6
+浕	6
+乗	6
+鏊	6
+醡	6
+玦	6
+睬	6
+邁	6
+髈	6
+鄜	6
+睺	6
+綁	6
+蔘	6
+昜	6
+卻	6
+箖	6
+沲	6
+菫	6
+矞	6
+耇	6
+湦	6
+堧	6
+戶	6
+嘘	6
+矴	6
+濩	6
+艨	6
+煩	6
+袈	6
+鈕	6
+尻	6
+櫞	6
+敍	6
+萆	6
+蝥	6
+蔹	6
+蕟	6
+唰	6
+濟	6
+斎	6
+岃	6
+釥	6
+唉	6
+骘	6
+夲	6
+奍	6
+鈤	6
+皴	6
+訊	6
+鹔	6
+奵	6
+魚	6
+槡	6
+輥	6
+鎯	6
+欐	6
+敀	6
+穑	6
+靰	6
+鉀	6
+睆	6
+據	6
+鋪	6
+嫜	6
+夆	6
+棨	6
+賠	6
+鍊	6
+攥	6
+籐	6
+傷	6
+奣	6
+茀	6
+黴	6
+虉	6
+鵝	6
+冊	6
+遠	6
+醫	6
+識	6
+椄	6
+嚢	6
+怅	6
+唔	6
+罅	6
+滶	6
+蹋	6
+笄	6
+匚	6
+耖	6
+顎	6
+蝾	6
+呸	6
+砘	6
+泘	6
+焟	6
+適	6
+鏵	6
+垉	6
+犮	6
+唻	6
+晫	6
+蕌	6
+畇	6
+澱	6
+呔	6
+尅	6
+抨	6
+厠	6
+嵉	6
+氷	6
+箇	6
+皊	6
+漰	6
+晻	6
+並	6
+丟	6
+糄	6
+矷	6
+荙	6
+胬	6
+旆	6
+菂	6
+竸	6
+佸	6
+霪	6
+萣	6
+棲	5
+歀	5
+鹡	5
+逡	5
+釋	5
+潤	5
+靥	5
+羴	5
+纖	5
+茝	5
+衛	5
+礌	5
+糥	5
+蓂	5
+褛	5
+洐	5
+氲	5
+訂	5
+獴	5
+鄅	5
+蘧	5
+侹	5
+媺	5
+琭	5
+缃	5
+帥	5
+帑	5
+伫	5
+缭	5
+咄	5
+鐩	5
+餈	5
+黾	5
+熘	5
+齒	5
+杩	5
+箢	5
+応	5
+蔵	5
+偖	5
+嫉	5
+侄	5
+箘	5
+裣	5
+蛘	5
+咧	5
+戾	5
+搠	5
+勞	5
+涘	5
+殼	5
+噹	5
+揹	5
+噐	5
+挍	5
+秄	5
+玊	5
+勠	5
+黯	5
+荺	5
+卬	5
+嚷	5
+苈	5
+鱔	5
+偈	5
+鲞	5
+讥	5
+瀅	5
+槔	5
+阼	5
+畄	5
+徃	5
+盞	5
+鬐	5
+炝	5
+蹰	5
+嘯	5
+谔	5
+蝲	5
+爐	5
+鹽	5
+徧	5
+鹩	5
+琞	5
+箋	5
+蒉	5
+黇	5
+憭	5
+殒	5
+嗌	5
+朿	5
+觊	5
+盡	5
+損	5
+飯	5
+鮮	5
+蒾	5
+懦	5
+瘳	5
+甭	5
+檾	5
+昤	5
+詺	5
+棑	5
+鉷	5
+爔	5
+扢	5
+庁	5
+襕	5
+叕	5
+壘	5
+甓	5
+態	5
+柷	5
+蝼	5
+翆	5
+榠	5
+湶	5
+榘	5
+訫	5
+殚	5
+戸	5
+揠	5
+婋	5
+觖	5
+髹	5
+跡	5
+珧	5
+侴	5
+濺	5
+苳	5
+軎	5
+徭	5
+昋	5
+鈣	5
+藟	5
+莙	5
+瀋	5
+芑	5
+敇	5
+楯	5
+骷	5
+戮	5
+咦	5
+吣	5
+薬	5
+杕	5
+韌	5
+虺	5
+蠧	5
+蒇	5
+婞	5
+雱	5
+摖	5
+垏	5
+畾	5
+蒎	5
+蓮	5
+阋	5
+瑊	5
+弎	5
+缢	5
+筯	5
+绔	5
+榞	5
+冭	5
+籃	5
+蟮	5
+姸	5
+効	5
+蟇	5
+鳈	5
+汔	5
+咼	5
+錩	5
+萫	5
+羅	5
+顕	5
+埖	5
+粻	5
+峱	5
+胝	5
+鸮	5
+葠	5
+燀	5
+倶	5
+鍛	5
+澋	5
+莰	4
+桜	4
+贏	4
+鮃	4
+鄰	4
+漍	4
+踱	4
+舯	4
+筞	4
+皲	4
+幁	4
+夢	4
+薢	4
+囝	4
+缋	4
+懑	4
+沝	4
+莯	4
+甠	4
+淢	4
+艁	4
+鰡	4
+燐	4
+旡	4
+岒	4
+頣	4
+牝	4
+雋	4
+箝	4
+锍	4
+綷	4
+盜	4
+汙	4
+苼	4
+烏	4
+吔	4
+蚰	4
+鱲	4
+寷	4
+蜴	4
+侀	4
+黙	4
+謦	4
+摭	4
+壺	4
+毖	4
+诹	4
+賽	4
+銩	4
+罱	4
+戡	4
+肸	4
+從	4
+娬	4
+糌	4
+葎	4
+铫	4
+籴	4
+蛞	4
+衝	4
+轟	4
+剤	4
+領	4
+驽	4
+頒	4
+舊	4
+刣	4
+疬	4
+阉	4
+阂	4
+谝	4
+鈊	4
+笩	4
+鞕	4
+楁	4
+厲	4
+簱	4
+詰	4
+鹘	4
+黝	4
+苽	4
+埦	4
+禔	4
+墎	4
+顫	4
+県	4
+搴	4
+積	4
+炐	4
+杪	4
+秞	4
+佔	4
+撐	4
+淲	4
+姉	4
+蟽	4
+陰	4
+挲	4
+仚	4
+茤	4
+喼	4
+苻	4
+銫	4
+滌	4
+廘	4
+鉑	4
+墐	4
+柲	4
+徍	4
+韋	4
+谖	4
+簣	4
+溡	4
+廚	4
+錠	4
+睎	4
+髅	4
+儂	4
+蛱	4
+簽	4
+晦	4
+洘	4
+澽	4
+橞	4
+鰤	4
+奮	4
+磘	4
+執	4
+鹋	4
+骀	4
+汋	4
+蝓	4
+秴	4
+瑪	4
+簾	4
+陎	4
+訁	4
+荮	4
+馐	4
+萊	4
+砱	4
+墣	4
+災	4
+芺	4
+楊	4
+噱	4
+愎	4
+倂	4
+擤	4
+衾	4
+诙	4
+玨	4
+侎	4
+燒	4
+纣	4
+吙	4
+渶	4
+竫	4
+袴	4
+錳	4
+旪	4
+鲭	4
+昑	4
+屷	4
+嫫	4
+瀟	4
+獠	4
+撈	4
+觇	4
+挻	4
+掃	4
+甶	4
+朮	4
+綿	4
+隊	4
+鈃	4
+舐	4
+唬	4
+锾	4
+鷹	4
+璊	4
+焛	4
+鑄	4
+諧	4
+踔	4
+栛	4
+鷽	4
+竼	4
+俜	4
+悝	4
+秠	4
+騵	4
+赬	4
+鱸	4
+衽	4
+噎	4
+缂	4
+裇	4
+粞	4
+帪	4
+硎	4
+撳	4
+萹	4
+暎	4
+硧	4
+響	4
+戙	4
+埪	4
+鉦	4
+獅	4
+闶	4
+嗾	4
+蕥	4
+蜢	4
+疍	4
+摈	4
+顢	4
+珒	4
+飕	4
+醼	4
+霎	4
+屐	4
+璕	4
+瘰	4
+曷	4
+婊	4
+瀢	4
+桝	4
+欯	4
+咗	4
+蕫	4
+衞	4
+蟳	4
+栱	4
+嗳	4
+斅	4
+顸	4
+攫	4
+绌	4
+療	3
+玍	3
+彀	3
+嵫	3
+牆	3
+妏	3
+嗔	3
+慠	3
+怍	3
+盬	3
+鎂	3
+谰	3
+淶	3
+柤	3
+螓	3
+爎	3
+輔	3
+楽	3
+屸	3
+墥	3
+湯	3
+嗷	3
+鋵	3
+貔	3
+禊	3
+寧	3
+銾	3
+啧	3
+夈	3
+凨	3
+韪	3
+脫	3
+穬	3
+堘	3
+莶	3
+湐	3
+併	3
+穩	3
+驛	3
+咝	3
+礠	3
+棿	3
+鲋	3
+柺	3
+鷃	3
+營	3
+齨	3
+虷	3
+瘐	3
+伬	3
+蠓	3
+肷	3
+泤	3
+補	3
+暌	3
+覩	3
+揷	3
+鍙	3
+袼	3
+塰	3
+昽	3
+茖	3
+箓	3
+谿	3
+甴	3
+孬	3
+絜	3
+娭	3
+踫	3
+杙	3
+沢	3
+尠	3
+颔	3
+咂	3
+拃	3
+奁	3
+簒	3
+缈	3
+顥	3
+珘	3
+噔	3
+飼	3
+咒	3
+匉	3
+杋	3
+缡	3
+倫	3
+藥	3
+筃	3
+圤	3
+扠	3
+鎧	3
+鉥	3
+掂	3
+铞	3
+尜	3
+桟	3
+檁	3
+琁	3
+饽	3
+哿	3
+擞	3
+锇	3
+湻	3
+髄	3
+坄	3
+螽	3
+酎	3
+苤	3
+剰	3
+茛	3
+郕	3
+劍	3
+伎	3
+厶	3
+刬	3
+橝	3
+杸	3
+輗	3
+垿	3
+蚍	3
+毬	3
+玡	3
+夌	3
+誤	3
+叅	3
+砫	3
+鴜	3
+崋	3
+軏	3
+湽	3
+勣	3
+圼	3
+樰	3
+値	3
+師	3
+宼	3
+仼	3
+貸	3
+姩	3
+瓅	3
+惦	3
+藝	3
+爲	3
+棷	3
+梀	3
+踇	3
+忭	3
+絔	3
+嗟	3
+耑	3
+欸	3
+僜	3
+帨	3
+亻	3
+篜	3
+閘	3
+曺	3
+谂	3
+鉚	3
+栃	3
+钔	3
+堅	3
+傘	3
+衲	3
+栺	3
+俟	3
+筈	3
+詀	3
+釦	3
+湊	3
+豨	3
+塹	3
+誾	3
+暝	3
+戩	3
+粧	3
+尛	3
+繤	3
+扱	3
+聽	3
+囮	3
+會	3
+诮	3
+悚	3
+咿	3
+黨	3
+荬	3
+貴	3
+讵	3
+鈥	3
+銆	3
+旿	3
+魃	3
+捃	3
+锊	3
+兗	3
+緖	3
+鋭	3
+徜	3
+幀	3
+藨	3
+墪	3
+搌	3
+碕	3
+坰	3
+灣	3
+姵	3
+圊	3
+夐	3
+墀	3
+鳠	3
+橐	3
+涇	3
+鸴	3
+毑	3
+暕	3
+掾	3
+夂	3
+妗	3
+冃	3
+徛	3
+鼯	3
+籀	3
+哏	3
+檠	3
+楢	3
+譽	3
+跚	3
+逴	3
+擬	3
+渖	3
+陳	3
+楡	3
+軋	3
+绁	3
+嘞	3
+蹺	3
+昖	3
+腚	3
+惭	3
+钸	3
+頦	3
+挌	3
+煚	3
+拚	3
+妚	3
+鮡	3
+彽	3
+袚	3
+蠢	3
+臤	3
+嫖	3
+旑	3
+鲯	3
+腦	3
+吆	3
+鈾	3
+曚	3
+镹	3
+嶽	3
+榅	3
+脹	3
+咻	3
+洩	3
+楲	3
+锿	3
+坲	3
+莸	3
+戓	3
+爣	3
+勲	3
+愠	3
+挈	3
+籓	3
+棓	3
+艼	3
+嵹	3
+鋸	3
+緾	3
+礡	3
+蝻	3
+伩	3
+鹬	3
+尨	3
+潪	3
+隠	3
+縱	3
+澣	3
+裬	3
+姳	3
+夅	3
+垍	3
+佷	2
+嶚	2
+鋌	2
+蘩	2
+蓍	2
+脈	2
+嵀	2
+鹴	2
+倷	2
+轷	2
+缊	2
+螠	2
+栁	2
+柖	2
+棡	2
+佫	2
+寤	2
+潄	2
+糗	2
+魰	2
+矅	2
+焺	2
+唷	2
+岍	2
+璪	2
+簊	2
+丌	2
+耠	2
+僬	2
+泒	2
+嗗	2
+詟	2
+禓	2
+禳	2
+寑	2
+趺	2
+嶐	2
+擊	2
+邶	2
+毨	2
+楛	2
+瞵	2
+鳡	2
+磵	2
+驺	2
+岝	2
+潰	2
+榶	2
+葐	2
+禥	2
+峖	2
+秱	2
+赍	2
+绋	2
+楾	2
+砢	2
+麽	2
+挢	2
+繳	2
+耜	2
+陴	2
+缱	2
+搳	2
+腠	2
+鑲	2
+禿	2
+蜺	2
+蘗	2
+躅	2
+癱	2
+憡	2
+獏	2
+蝟	2
+轚	2
+観	2
+鵰	2
+鉬	2
+衆	2
+咹	2
+蚋	2
+忝	2
+媄	2
+曞	2
+欽	2
+嶅	2
+蟎	2
+鏍	2
+兿	2
+亷	2
+醐	2
+漻	2
+簮	2
+磾	2
+霐	2
+掤	2
+庴	2
+躍	2
+砳	2
+莦	2
+漵	2
+栂	2
+濣	2
+鼬	2
+栨	2
+摻	2
+縛	2
+鍔	2
+堺	2
+綌	2
+焴	2
+麐	2
+鍥	2
+鳴	2
+甌	2
+館	2
+咅	2
+湎	2
+鎷	2
+釐	2
+踺	2
+鰼	2
+霦	2
+篑	2
+皁	2
+唎	2
+怄	2
+杦	2
+髀	2
+姖	2
+祘	2
+洟	2
+嚴	2
+翛	2
+渮	2
+睳	2
+髫	2
+怏	2
+罰	2
+漟	2
+綅	2
+礞	2
+犒	2
+煟	2
+蟬	2
+異	2
+剝	2
+鏪	2
+宄	2
+癭	2
+荳	2
+煇	2
+郷	2
+鈭	2
+凈	2
+伃	2
+恿	2
+擺	2
+旽	2
+杴	2
+斒	2
+賦	2
+當	2
+惮	2
+禸	2
+犄	2
+暁	2
+豢	2
+蓱	2
+窺	2
+練	2
+闆	2
+饣	2
+圮	2
+迕	2
+関	2
+瘿	2
+濱	2
+椀	2
+囦	2
+粣	2
+篐	2
+鉆	2
+篩	2
+眈	2
+媕	2
+悻	2
+聲	2
+傛	2
+杅	2
+酲	2
+诟	2
+矦	2
+櫆	2
+宒	2
+嫽	2
+饫	2
+発	2
+誠	2
+夬	2
+踯	2
+埈	2
+繭	2
+讹	2
+槕	2
+徉	2
+飊	2
+鍀	2
+徝	2
+翺	2
+揵	2
+櫡	2
+蹠	2
+鶄	2
+攉	2
+梌	2
+惆	2
+鬻	2
+訓	2
+帄	2
+硾	2
+勢	2
+斻	2
+燹	2
+脷	2
+鈖	2
+簟	2
+坈	2
+衄	2
+缲	2
+笞	2
+鸻	2
+醬	2
+晱	2
+襲	2
+栉	2
+腎	2
+鋇	2
+彳	2
+侘	2
+汦	2
+呠	2
+牚	2
+珋	2
+痍	2
+窸	2
+斶	2
+薶	2
+芣	2
+朾	2
+柩	2
+抷	2
+鳳	2
+峵	2
+朓	2
+馳	2
+冝	2
+鋹	2
+摂	2
+悕	2
+屣	2
+枬	2
+隻	2
+龃	2
+豋	2
+澆	2
+趯	2
+豕	2
+梍	2
+蹒	2
+筚	2
+鉛	2
+煉	2
+丄	2
+鹵	2
+鄫	2
+蹁	2
+冂	2
+唍	2
+訕	2
+靿	2
+妧	2
+覅	2
+抅	2
+袏	2
+棄	2
+蜷	2
+徤	2
+衹	2
+焀	2
+埑	2
+噩	2
+鼢	2
+絼	2
+忼	2
+阌	2
+虼	2
+実	2
+鰍	2
+屻	2
+涥	2
+涙	2
+耢	2
+牖	2
+拊	2
+馔	2
+冡	2
+噫	2
+曇	2
+鮊	2
+罣	2
+壌	2
+麩	2
+牣	2
+驵	2
+峇	2
+踉	2
+掳	2
+簃	2
+苿	2
+芄	2
+貎	2
+勬	2
+鶋	2
+滅	2
+棝	2
+硪	2
+賡	2
+渞	2
+膂	2
+哊	2
+瞐	2
+銳	2
+鋰	2
+脽	2
+嵱	2
+鐤	2
+鐳	2
+衦	2
+様	2
+竂	2
+襖	2
+挿	2
+醮	2
+蘘	2
+嗙	2
+紪	2
+捯	2
+瑋	2
+壽	2
+淿	2
+島	2
+鈞	2
+猢	2
+隺	2
+涼	2
+湨	2
+叏	2
+濆	2
+蕹	2
+羣	2
+燻	2
+鐗	2
+褃	2
+啛	2
+鐼	2
+麃	2
+郛	2
+砠	2
+玞	2
+癖	2
+齋	2
+鑛	2
+铻	2
+顱	2
+挄	2
+祔	2
+籿	2
+跕	2
+跏	2
+繼	2
+窅	2
+楺	2
+嫲	2
+盪	2
+煵	2
+挰	2
+莤	2
+刿	2
+魇	2
+犷	2
+還	2
+梼	2
+靚	2
+樞	2
+胊	2
+鸩	2
+棫	2
+媵	2
+彊	2
+牑	2
+韯	2
+垊	2
+掼	2
+遞	2
+觥	2
+赀	2
+競	2
+暅	2
+諨	2
+踅	2
+漶	2
+濊	2
+谥	2
+鮜	2
+嚇	2
+緩	2
+縋	1
+複	1
+涫	1
+喚	1
+愯	1
+栘	1
+兌	1
+绲	1
+陜	1
+幞	1
+貹	1
+仦	1
+蔲	1
+啇	1
+牞	1
+盝	1
+忺	1
+嬡	1
+葯	1
+洭	1
+顳	1
+紀	1
+澒	1
+騤	1
+逫	1
+楉	1
+櫒	1
+耋	1
+櫃	1
+忿	1
+汖	1
+偎	1
+塂	1
+駕	1
+幫	1
+醑	1
+塣	1
+錣	1
+侅	1
+溓	1
+垶	1
+孮	1
+馺	1
+嬾	1
+烋	1
+歴	1
+疎	1
+鼑	1
+萞	1
+祌	1
+荌	1
+訄	1
+梾	1
+柕	1
+祓	1
+槅	1
+兟	1
+隒	1
+洔	1
+莩	1
+笓	1
+甿	1
+禎	1
+鲥	1
+忂	1
+堟	1
+曬	1
+妳	1
+潙	1
+璥	1
+梴	1
+瘁	1
+眦	1
+猾	1
+脰	1
+錚	1
+湆	1
+臯	1
+稲	1
+梩	1
+釹	1
+绠	1
+敠	1
+袿	1
+偑	1
+墼	1
+謀	1
+匤	1
+揎	1
+霂	1
+遫	1
+惪	1
+敗	1
+瀲	1
+軽	1
+呮	1
+伧	1
+蔈	1
+鋠	1
+瘛	1
+扙	1
+呓	1
+緢	1
+隱	1
+鳂	1
+姼	1
+渁	1
+銦	1
+碏	1
+迚	1
+揸	1
+堈	1
+翉	1
+傾	1
+凵	1
+幪	1
+恔	1
+俳	1
+胩	1
+頑	1
+漁	1
+彍	1
+抦	1
+虮	1
+睚	1
+瘔	1
+缁	1
+樸	1
+尪	1
+顒	1
+軒	1
+伻	1
+睇	1
+濨	1
+屝	1
+墝	1
+矂	1
+検	1
+銃	1
+璍	1
+拸	1
+鮨	1
+嬰	1
+鉺	1
+艅	1
+伓	1
+阕	1
+卩	1
+鯷	1
+窬	1
+墹	1
+梛	1
+舉	1
+峢	1
+跣	1
+龠	1
+娢	1
+杌	1
+鴛	1
+輻	1
+眏	1
+鑁	1
+厫	1
+鐀	1
+醱	1
+猃	1
+枃	1
+鍫	1
+鑰	1
+扟	1
+憜	1
+庺	1
+倅	1
+亙	1
+晲	1
+敩	1
+崍	1
+詈	1
+喑	1
+貵	1
+汏	1
+闄	1
+竢	1
+瀴	1
+塺	1
+稃	1
+頋	1
+氭	1
+揮	1
+靆	1
+慬	1
+讠	1
+榿	1
+睞	1
+騠	1
+棌	1
+孫	1
+龅	1
+鍋	1
+釨	1
+階	1
+磚	1
+鄗	1
+薀	1
+飺	1
+抆	1
+粨	1
+漿	1
+欵	1
+砑	1
+刎	1
+庝	1
+崌	1
+寗	1
+觑	1
+蔃	1
+痞	1
+齻	1
+羭	1
+墄	1
+濓	1
+笵	1
+垀	1
+峸	1
+诓	1
+駉	1
+駸	1
+觳	1
+陖	1
+鱄	1
+汿	1
+飑	1
+踣	1
+楖	1
+窧	1
+傒	1
+剛	1
+汸	1
+魍	1
+鈷	1
+敫	1
+勆	1
+榖	1
+彔	1
+礪	1
+凄	1
+釭	1
+談	1
+荵	1
+擾	1
+珬	1
+暯	1
+礒	1
+榇	1
+潶	1
+礬	1
+湰	1
+豎	1
+穈	1
+鱇	1
+褙	1
+縯	1
+贔	1
+褆	1
+酄	1
+覺	1
+鍂	1
+滛	1
+痐	1
+迏	1
+腼	1
+灬	1
+熂	1
+綸	1
+縻	1
+羸	1
+鼰	1
+踴	1
+燓	1
+揑	1
+觚	1
+揍	1
+該	1
+週	1
+晸	1
+圁	1
+戯	1
+賄	1
+鏑	1
+掕	1
+錄	1
+梏	1
+紑	1
+揳	1
+違	1
+睶	1
+椥	1
+纥	1
+訟	1
+筶	1
+桾	1
+瓨	1
+歃	1
+枙	1
+暤	1
+獨	1
+駝	1
+吶	1
+鹪	1
+箤	1
+託	1
+訸	1
+嵠	1
+柆	1
+梲	1
+铓	1
+癃	1
+籨	1
+腫	1
+宻	1
+鐨	1
+澷	1
+凖	1
+琓	1
+穏	1
+筅	1
+爝	1
+裏	1
+浟	1
+畊	1
+籵	1
+凘	1
+锳	1
+裟	1
+閒	1
+焧	1
+毘	1
+穎	1
+煳	1
+劇	1
+裨	1
+汎	1
+鄪	1
+貢	1
+帰	1
+蠋	1
+鎵	1
+魯	1
+怙	1
+炲	1
+涽	1
+拡	1
+瑒	1
+漮	1
+曱	1
+粔	1
+塁	1
+謬	1
+菶	1
+媌	1
+輞	1
+畨	1
+眝	1
+拝	1
+鵷	1
+戋	1
+铴	1
+撚	1
+帳	1
+鐓	1
+岾	1
+刋	1
+蔀	1
+伅	1
+貯	1
+亜	1
+笯	1
+哳	1
+祫	1
+槸	1
+搙	1
+坿	1
+谪	1
+螣	1
+薿	1
+蛃	1
+鲱	1
+媂	1
+嫰	1
+橎	1
+鎣	1
+圄	1
+簺	1
+覌	1
+咣	1
+瓔	1
+枔	1
+枺	1
+縩	1
+唠	1
+掟	1
+吽	1
+埧	1
+蚽	1
+嫕	1
+眕	1
+懊	1
+俒	1
+兘	1
+廝	1
+揬	1
+稈	1
+痷	1
+斾	1
+艦	1
+屽	1
+鶼	1
+崡	1
+瞠	1
+敎	1
+擔	1
+踽	1
+崊	1
+晳	1
+攵	1
+鍖	1
+瑫	1
+朌	1
+坺	1
+惘	1
+聞	1
+凍	1
+饔	1
+鍦	1
+疳	1
+膽	1
+纄	1
+淰	1
+椇	1
+攬	1
+倆	1
+茷	1
+悊	1
+硁	1
+硤	1
+埆	1
+鮟	1
+舤	1
+雰	1
+茇	1
+遹	1
+湠	1
+啳	1
+堾	1
+厐	1
+虿	1
+懔	1
+烳	1
+秊	1
+織	1
+鎻	1
+攃	1
+豬	1
+垨	1
+踮	1
+畞	1
+阠	1
+琣	1
+熆	1
+筍	1
+蔥	1
+鴿	1
+彫	1
+憙	1
+珗	1
+疊	1
+槁	1
+駆	1
+嘠	1
+瞑	1
+袺	1
+簀	1
+哜	1
+鳐	1
+暈	1
+埾	1
+冦	1
+蹚	1
+矼	1
+冏	1
+鍺	1
+鄭	1
+鮪	1
+臥	1
+齊	1
+玔	1
+柊	1
+蹿	1
+媸	1
+鎗	1
+峈	1
+畤	1
+喣	1
+瑈	1
+囷	1
+籠	1
+瀬	1
+嬲	1
+杬	1
+頰	1
+涀	1
+鳥	1
+殑	1
+錘	1
+鱗	1
+愰	1
+嬫	1
+鍘	1
+灥	1
+圽	1
+醅	1
+嘨	1
+澦	1
+冨	1
+栭	1
+柈	1
+経	1
+癍	1
+嶆	1
+壩	1
+剷	1
+綂	1
+隑	1
+焐	1
+秂	1
+璈	1
+灴	1
+刄	1
+忓	1
+櫵	1
+枆	1
+幜	1
+枴	1
+恝	1
+匜	1
+溲	1
+窛	1
+呒	1
+檍	1
+慶	1
+迿	1
+飩	1
+乇	1
+嶸	1
+勁	1
+忚	1
+捜	1
+隃	1
+瘡	1
+廼	1
+泑	1
+韛	1
+窣	1
+穜	1
+缧	1
+銨	1
+詁	1
+靣	1
+侭	1
+洓	1
+揄	1
+祿	1
+嗬	1
+韓	1
+吠	1
+瓞	1
+婈	1
+鄮	1
+賃	1
+矚	1
+櫧	1
+檄	1
+摬	1
+檇	1
+缣	1
+鞒	1
+湇	1
+忉	1
+鏉	1
+慫	1
+滳	1
+亣	1
+狈	1
+醝	1
+薙	1
+湁	1
+昘	1
+幗	1
+仺	1
+煬	1
+暙	1
+誼	1
+醭	1
+昻	1
+崶	1
+勮	1
+習	1
+譯	1
+鍒	1
+巸	1
+蝪	1
+掎	1
+祅	1
+镚	1
+恆	1
+鈴	1
+騛	1
+碫	1
+唨	1
+鏐	1
+牻	1
+蹸	1
+睒	1
+巟	1
+蠔	1
+秌	1
+夎	1
+晹	1
+鐭	1
+謩	1
+拫	1
+遲	1
+悾	1
+垷	1
+綫	1
+鍨	1
+癯	1
+讣	1
+錋	1
+巺	1
+寵	1
+昞	1
+恊	1
+曧	1
+愭	1
+龉	1
+慣	1
+媱	1
+拰	1
+廇	1
+厈	1
+谘	1
+搢	1
+朣	1
+彘	1
+鏻	1
+垺	1
+眛	1

udpcreateindex/config.json

@@ -0,0 +1,114 @@
+{
+  "udpport": ":1483",
+  "msg_server": "10.171.112.160:7070",
+  "mgo_bidding": {
+    "addr": "192.168.3.207:27092",
+    "size": 10,
+    "db": "wjh",
+    "uname": "",
+    "upwd": "",
+    "collect": "bidding"
+  },
+  "mgo_extract": {
+    "addr": "192.168.3.207:27092",
+    "size": 10,
+    "db": "wjh",
+    "collect": "extract",
+    "collect1": ""
+  },
+  "mgo_qyxy": {
+    "addr": "192.168.3.207",
+    "pool": 10,
+    "db": "mixdata"
+  },
+  "mgo_project": {
+    "addr": "192.168.3.207",
+    "db": "qfw",
+    "collect": "projectset",
+    "index": "projectset",
+    "type": "projectset"
+  },
+  "standard": {
+    "addr": "192.168.3.207:27092",
+    "pool": 10,
+    "db": "mixdata",
+    "coll_area": "address_jy_2021",
+    "winner": {
+      "collect1": "winner_enterprise",
+      "collect2": "winner_err",
+      "index": "winner_v1",
+      "type": "winner"
+    },
+    "buyer": {
+      "collect1": "buyer_enterprise",
+      "collect2": "buyer_err",
+      "index": "buyer_v6",
+      "type": "buyer"
+    },
+    "agency": {
+      "collect": "agency_enterprise",
+      "index": "agencyent_v1",
+      "type": "agencyent"
+    }
+  },
+  "elastic_1": {
+    "addr": "http://192.168.3.206:9800",
+    "pool": 12,
+    "label": "旧es集群库"
+  },
+  "elastic_2": {
+    "addr": "http://192.168.3.206:9800",
+    "pool": 12,
+    "label": "新es集群库"
+  },
+  "nsq_id": {
+    "addr": "192.168.3.166:4150",
+    "topic": "project-id",
+    "channel": "EsIndex",
+    "concurrent": 1
+  },
+  "bidding_index": {
+    "index": "bidding_v2",
+    "type": "bidding",
+    "multiIndex": "",
+    "esfieldsmap": {
+      "buyerzipcode": "string", "winnertel": "string", "winnerperson": "string", "contractcode": "string", "winneraddr": "string", "agencyaddr": "string", "buyeraddr": "string", "signaturedate": "int64",
+      "projectperiod": "string", "projectaddr": "string", "agencytel": "string", "agencyperson": "string", "buyerperson": "string", "agency": "string", "projectscope": "string", "projectcode": "string",
+      "bidopentime": "int64", "supervisorrate": "float64", "buyertel": "string", "bidamount": "float64", "winner": "string", "buyer": "string", "budget": "float64", "projectname": "string", "bidstatus": "string",
+      "buyerclass": "string", "topscopeclass": "", "s_topscopeclass": "string", "s_subscopeclass": "string", "area": "string", "city": "string", "district": "string", "s_winner": "string", "_id": "", "title": "string",
+      "detail": "string", "site": "string", "comeintime": "int64", "href": "string", "infoformat": "int32", "publishtime": "int64", "s_sha": "string", "spidercode": "string", "subtype": "string", "toptype": "string",
+      "projectinfo": "", "purchasing": "string", "purchasinglist": "", "channel": "string", "winnerorder": "", "project_scale": "string", "project_duration": "int32", "project_timeunit": "string",
+      "project_startdate": "int64", "project_completedate": "int64", "payway": "string", "contract_guarantee": "bool", "bid_guarantee": "bool", "qualifies": "", "entidlist": "", "funds": "string",
+      "review_experts": "", "bidmethod": "string", "bidendtime": "int64", "bidopenaddress": "string", "docamount": "float64", "agencyrate": "float64", "agencyfee": "float64", "bidway": "string",
+      "getdocmethod": "string", "china_bidding": "string", "purchasing_tag": "string", "multipackage": "int32", "isValidFile": "bool"
+    },
+    "mgofields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,s_topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass,dataging,winnerorder,project_scale,project_duration,project_timeunit,project_startdate,project_completedate, payway,contract_guarantee,bid_guarantee,qualifies,funds,review_experts,bidmethod,bidendtime,bidopenaddress,docamount,bidway,agencyrate,agencyfee,getdocmethod,purchasing_tag",
+    "projectinfomap": {
+      "approvecode": "string", "approvecontent": "string", "approvestatus": "string", "approvetime": "string", "approvedept": "string", "approvenumber": "string", "projecttype": "string", "approvecity": "string"
+    },
+    "purchasinglistmap": {
+      "itemname": "string", "item": "string", "brandname": "string", "model": "string", "unitname": "string", "number": "float64", "unitprice": "float64", "totalprice": "float64"
+    },
+    "procurementlistmap": {
+      "projectname": "string", "buyer": "string", "item": "string", "projectscope": "string", "expurasingtime": "string", "totalprice": "float64"
+    },
+    "winnerordermap": {
+      "sort": "int", "sortstr": "string", "entname": "string"
+    }
+  },
+  "jyfb_udp": {
+    "addr": "127.0.0.1",
+    "port": 11118
+  },
+  "jkmail": {
+    "to": "zhangjinkun@topnet.net.cn",
+    "api": "http://172.17.145.179:19281/_send/_mail"
+  },
+  "filter-keyword": ["(招标网|千里马|采招网|招标采购导航网|招标与采购网|中国招投标网|中国采购与招标网|中国采购与招标|优质采)[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}(qianlima|zhaobiao|okcis|zbytb|infobidding|bidcenter|youzhicai|chinabidding|Chinabidding｜CHINABIDDING)[a-z0-9.\\/\\/]{0,40}",
+    "招标网[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}zhaobiao[a-z0-9.\\/\\/]{0,40}",
+    "千里马[\\w\\W]{0,15}[a-z0-9:\\/\\/.]{0,20}qianlima[a-z0-9.\\/\\/]{0,10}",
+    "[\\(（]?(网址)?[:：；;]?(http|https|htpps)*[:：]?(\\/\\/)?(www|jinan|WWW)?.(zhaobiao|chinabidding|Chinabidding|CHINABIDDING|infobidding|zbytb|okcis|qianlima|youzhicai).(com|cn|COM|CN)?(.cn|.CN)?\\/?[\\)）]?",
+    "[\\(（]?(网址)?(:：)?(http|https|htpps)*(:|：)?\\/\\/www.bidcenter.com.cn\\/",
+    "千里马(平台|网站)+", "[“\"]?优质采(平台|电子交易平台|云采购平台|交易平台)?[”\"]?", "《?(中国采购与|中国)?招(投)?标(与采购|采购导航)?网》?",
+    "《?元博网(采购与招标网)?》?", "《?(中国)?招标采购导航网》?", "中\\W{0,3}国采\\W{0,3}招\\W{0,3}网\\W*[(（]?(bidcenter.com.cn)?[)）]?", "已方宝", "中国招标与采购"]
+}

udpcreateindex/datamonitor.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"sync"
+	"time"
+)
+
+/**
+增加数据监控服务，每15分钟保存一次抽取情况
+**/
+var IS = &InfoStatus{
+	time.Now().Unix(), 0, map[string]int{}, &sync.Mutex{}}
+
+func init() {
+	go IS.Save()
+}
+
+type InfoStatus struct {
+	Starttime int64
+	Endtime   int64
+	Val       map[string]int
+	Lock      *sync.Mutex
+}
+
+func (is *InfoStatus) Add(t string) {
+	is.Lock.Lock()
+	is.Val[t]++
+	is.Lock.Unlock()
+}
+
+func (is *InfoStatus) Save() {
+	is.Lock.Lock()
+	is.Endtime = time.Now().Unix()
+	save := map[string]interface{}{}
+	all := 0
+	for k, v := range is.Val {
+		all += v
+		save[k] = v
+	}
+	if all > 0 {
+		save["starttime"] = is.Starttime
+		save["endtime"] = is.Endtime
+		save["flag"] = "indexser"
+		is.Val = map[string]int{}
+		go extractMgo.Save("datamonitor", save)
+	}
+	is.Starttime = is.Endtime
+	is.Lock.Unlock()
+	time.AfterFunc(15*time.Minute, is.Save)
+}

udpcreateindex/file.go

@@ -0,0 +1,199 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	util "utils"
+)
+
+var (
+	CmmonDFA        *DFA                                             //常用字
+	NotCommonDFA    *DFA                                             //不常用字
+	TimesLimit      int                                              //常用字界限
+	CmmLmt, NcmmLmt float64                                          //更新界限
+	HanReg          = regexp.MustCompile("[\u4e00-\u9fa5]+")         //中文正则
+	SpaceReg        = regexp.MustCompile("[\\s\u3000\u2003\u00a0]+") //空格正则
+	SpecialReg      = regexp.MustCompile("图片(\\d)+")                 //
+)
+
+func InitFileInfo() {
+	TimesLimit = 500
+	CmmLmt = 0.5
+	NcmmLmt = 0.1
+	CmmonDFA = &DFA{}
+	NotCommonDFA = &DFA{}
+	LoadDict("common.txt") //初始化常用字典
+}
+
+//DFA
+type DFA struct {
+	Link map[string]interface{}
+}
+
+func (d *DFA) AddWord(keys ...string) {
+	d.AddWordAll(true, keys...)
+}
+func (d *DFA) AddWordAll(haskey bool, keys ...string) {
+	if d.Link == nil {
+		d.Link = make(map[string]interface{})
+	}
+	for _, key := range keys {
+		nowMap := &d.Link
+		for i := 0; i < len(key); i++ {
+			kc := key[i : i+1]
+			if v, ok := (*nowMap)[kc]; ok {
+				nowMap, _ = v.(*map[string]interface{})
+			} else {
+				newMap := map[string]interface{}{}
+				newMap["YN"] = "0"
+				(*nowMap)[kc] = &newMap
+				nowMap = &newMap
+			}
+			if i == len(key)-1 {
+				(*nowMap)["YN"] = "1"
+				if haskey {
+					(*nowMap)["K"] = key
+				}
+			}
+		}
+	}
+}
+
+func (d *DFA) CheckSensitiveWord(src string) []string {
+	res := make([]string, 0)
+	for j := 0; j < len(src); j++ {
+		nowMap := &d.Link
+		for i := j; i < len(src); i++ {
+			word := src[i : i+1]
+			nowMap, _ = (*nowMap)[word].(*map[string]interface{})
+			if nowMap != nil { // 存在，则判断是否为最后一个
+				if "1" == util.ObjToString((*nowMap)["YN"]) {
+					s := util.ObjToString((*nowMap)["K"])
+					res = append(res, s)
+				}
+			} else {
+				break
+			}
+		}
+	}
+	return res
+}
+
+//加载统计的常用词
+func LoadDict(path string) {
+	dictFile, err := os.Open(path)
+	if err != nil {
+		util.Debug("Load Common.txt Error")
+		os.Exit(-1)
+	}
+	defer dictFile.Close()
+	reader := bufio.NewReader(dictFile)
+	var (
+		text      string
+		frequency int
+	)
+
+	// 逐行读入分词
+	line := 0
+	for {
+		line++
+		size, fsErr := fmt.Fscanln(reader, &text, &frequency) //读每行赋值
+		if fsErr == io.EOF {                                  //读取到结尾
+			break
+		}
+		if size == 2 { //正确数据
+			if frequency >= TimesLimit { //常用字
+				CmmonDFA.AddWord(text)
+			} else { //非常用字
+				NotCommonDFA.AddWord(text)
+			}
+		} else {
+			util.Debug("Read Line Error:", line)
+		}
+	}
+}
+
+//解析附件
+func AnalysisFile(filetext string) bool {
+	defer util.Catch()
+	//过滤空格
+	filetextTmp := SpaceReg.ReplaceAllString(filetext, "")
+	if filetextTmp == "" { //附件为空
+		return false
+	}
+	//特殊情况：图片0 图片1
+	filetextTmp = SpecialReg.ReplaceAllString(filetextTmp, "")
+	if filetextTmp == "" { //附件为空
+		return false
+	}
+	//中文匹配
+	HanArr := HanReg.FindAllString(filetextTmp, -1)
+	hanText := strings.Join(HanArr, "")
+	hanTextLen := len([]rune(hanText))
+	//长度过滤
+	if hanTextLen <= 100 {
+		return false
+	}
+	//qu.Debug(hanTextLen, hanText)
+	commonArr := CmmonDFA.CheckSensitiveWord(hanText)
+	commonLen := len(commonArr)
+	//qu.Debug(commonLen, commonArr)
+	//commonText := strings.Join(commonArr, "")
+	notCommonArr := NotCommonDFA.CheckSensitiveWord(hanText)
+	notCommonLen := len(notCommonArr)
+	//qu.Debug(notCommonLen, notCommonArr)
+	//解析常用字和非常用字占比(由于常用字或非常用字集不全，会导致比例相加不为100%)
+	commonRatio := float64(commonLen) / float64(hanTextLen)
+	commonRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", commonRatio), 64)
+	//qu.Debug(commonRatio)
+	notCommonRatio := float64(notCommonLen) / float64(hanTextLen)
+	notCommonRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", notCommonRatio), 64)
+	if commonRatio >= CmmLmt && notCommonRatio < NcmmLmt {
+		return true
+	}
+	return false
+}
+
+//测试方法
+func AnalysisFileTest(detail string) (bool, string, int, float64, float64) {
+	//qu.Debug(detail)
+	defer util.Catch()
+	//过滤空格
+	filetextTmp := SpaceReg.ReplaceAllString(detail, "")
+	if filetextTmp == "" { //附件为空
+		return false, "", 0, 0, 0
+	}
+	//特殊情况：图片0 图片1
+	filetextTmp = SpecialReg.ReplaceAllString(filetextTmp, "")
+	if filetextTmp == "" { //附件为空
+		return false, "", 1, 0, 0
+	}
+	//中文匹配
+	HanArr := HanReg.FindAllString(filetextTmp, -1)
+	hanText := strings.Join(HanArr, "")
+	hanTextLen := len([]rune(hanText))
+	//长度过滤
+	if hanTextLen <= 100 {
+		return false, "", 2, 0, 0
+	}
+	//qu.Debug(textLen, text)
+	commonArr := CmmonDFA.CheckSensitiveWord(hanText)
+	commonLen := len(commonArr)
+	util.Debug(commonLen, commonArr)
+	//commonText := strings.Join(commonArr, "")
+	notCommonArr := NotCommonDFA.CheckSensitiveWord(hanText)
+	notCommonLen := len(notCommonArr)
+	util.Debug(notCommonLen, notCommonArr)
+	//notCommonText := strings.Join(notCommonArr, "")
+	//解析常用字和非常用字占比(由于常用字或非常用字集不全，会导致比例相加不为100%)
+	commonRatio := float64(commonLen) / float64(hanTextLen)
+	commonRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", commonRatio), 64)
+	notCommonRatio := float64(notCommonLen) / float64(hanTextLen)
+	notCommonRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", notCommonRatio), 64)
+	return true, filetextTmp, 10, commonRatio, notCommonRatio
+}

udpcreateindex/go.mod

@@ -0,0 +1,14 @@
+module createindex
+
+go 1.16
+
+require (
+	github.com/aliyun/aliyun-oss-go-sdk v2.2.4+incompatible
+	github.com/robfig/cron v1.2.0
+	go.mongodb.org/mongo-driver v1.4.1
+	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
+	utils v0.0.0
+)
+
+replace utils => ../common_utils

udpcreateindex/go.sum

@@ -0,0 +1,177 @@
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/aliyun/aliyun-oss-go-sdk v2.2.4+incompatible h1:cD1bK/FmYTpL+r5i9lQ9EU6ScAjA173EVsii7gAc6SQ=
+github.com/aliyun/aliyun-oss-go-sdk v2.2.4+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.29.15 h1:0ms/213murpsujhsnxnNKNeVouW60aJqSd992Ks3mxs=
+github.com/aws/aws-sdk-go v1.29.15/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
+github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
+github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
+github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
+github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
+github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
+github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
+github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
+github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
+github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
+github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
+github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
+github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
+github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
+github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
+github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
+github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
+github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
+github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
+github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
+github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M=
+github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
+github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
+github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo=
+github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.4.1 h1:38NSAyDPagwnFpUA/D5SFgbugUYR3NzYRNa4Qk9UxKs=
+go.mongodb.org/mongo-driver v1.4.1/go.mod h1:llVBH2pkj9HywK0Dtdt6lDikOjFLbceHVu/Rc0iMKLs=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ=
+golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/olivere/elastic.v2 v2.0.61 h1:7cpl3MW8ysa4GYFBXklpo5mspe4NK0rpZTdyZ+QcD4U=
+gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

udpcreateindex/init.go

@@ -0,0 +1,261 @@
+package main
+
+import (
+	"createindex/oss"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+	util "utils"
+	"utils/elastic"
+	"utils/mongodb"
+	"utils/udp"
+)
+
+var (
+	Sysconfig map[string]interface{}
+
+	biddingMgo  *mongodb.MongodbSim
+	extractMgo  *mongodb.MongodbSim
+	qyxyMgo     *mongodb.MongodbSim
+	projectMgo  *mongodb.MongodbSim
+	standardMgo *mongodb.MongodbSim
+
+	Es1 *elastic.Elastic
+	Es2 *elastic.Elastic
+
+	currentColl string
+	udpclient   udp.UdpClient //udp对象
+	updport     string
+	multiIndex  []string
+
+	biddingMgoFields      []string
+	biddingEsFields       map[string]interface{}
+	projectinfoFields     map[string]interface{}
+	purchasinglistFields  map[string]interface{}
+	procurementlisFields  map[string]interface{}
+	winnerorderlistFields map[string]interface{}
+
+	updateBiddingPool chan []map[string]interface{}
+	updateExtractPool chan []map[string]interface{}
+	saveEsPool        chan map[string]interface{}
+	saveEsAllPool     chan map[string]interface{}
+	saveEsElsePool    chan map[string]interface{}
+	saveProjectEsPool chan map[string]interface{}
+	updateBiddingSp   chan bool
+	updateExtractSp   chan bool
+	saveEsSp          chan bool
+	saveEsAllSp       chan bool
+	saveEsElseSp      chan bool
+	saveProjectSp     chan bool
+
+	JyUdpAddr *net.UDPAddr
+
+	esAddr, esNode string
+
+	FilterKeyword []string              //正文竟品关键词过滤
+	ProvinceDict  map[string][]Province //省份-map
+	CityDict      map[string][]City     //城市-map
+	DistrictDict  map[string][]District //区县-map
+
+	bidding, extract, qyxy, project, standard, buyer, winner, biddingIndex map[string]interface{}
+
+	MgoBulkSize  = 200   // mgo批量保存大小
+	EsBulkSize   = 200   // es批量保存大小
+	detailLength = 50000 // es保存detail长度
+	fileLength   = 50000 // es保存附件文本长度
+	pscopeLength = 32766 // projectscope长度
+)
+
+var StopFlag = false // 程序生索引停止标志
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	updport, _ = Sysconfig["updport"].(string)
+
+	bidding, _ = Sysconfig["mgo_bidding"].(map[string]interface{})
+	extract, _ = Sysconfig["mgo_extract"].(map[string]interface{})
+	qyxy, _ = Sysconfig["mgo_qyxy"].(map[string]interface{})
+	project, _ = Sysconfig["mgo_project"].(map[string]interface{})
+	standard, _ = Sysconfig["standard"].(map[string]interface{})
+	buyer, _ = Sysconfig["buyer"].(map[string]interface{})
+	winner, _ = Sysconfig["winner"].(map[string]interface{})
+
+	currentColl = util.ObjToString(bidding["collect"])
+	biddingIndex, _ = Sysconfig["bidding_index"].(map[string]interface{})
+
+	biddingMgo = &mongodb.MongodbSim{
+		MongodbAddr: bidding["addr"].(string),
+		Size:        util.IntAllDef(bidding["pool"], 5),
+		DbName:      bidding["db"].(string),
+		UserName:    bidding["uname"].(string),
+		Password:    bidding["upwd"].(string),
+	}
+	biddingMgo.InitPool()
+	extractMgo = &mongodb.MongodbSim{
+		MongodbAddr: extract["addr"].(string),
+		Size:        util.IntAllDef(extract["pool"], 5),
+		DbName:      extract["db"].(string),
+	}
+	extractMgo.InitPool()
+	qyxyMgo = &mongodb.MongodbSim{
+		MongodbAddr: qyxy["addr"].(string),
+		Size:        util.IntAllDef(qyxy["pool"], 5),
+		DbName:      qyxy["db"].(string),
+	}
+	qyxyMgo.InitPool()
+	projectMgo = &mongodb.MongodbSim{
+		MongodbAddr: project["addr"].(string),
+		Size:        util.IntAllDef(project["pool"], 5),
+		DbName:      project["db"].(string),
+	}
+	projectMgo.InitPool()
+	standardMgo = &mongodb.MongodbSim{
+		MongodbAddr: standard["addr"].(string),
+		Size:        util.IntAllDef(standard["pool"], 5),
+		DbName:      standard["db"].(string),
+		UserName:    bidding["uname"].(string),
+		Password:    bidding["upwd"].(string),
+	}
+	standardMgo.InitPool()
+
+	econf1 := Sysconfig["elastic_1"].(map[string]interface{})
+	Es1 = &elastic.Elastic{
+		S_esurl: econf1["addr"].(string),
+		I_size:  util.IntAllDef(econf1["pool"], 5),
+	}
+	Es1.InitElasticSize()
+
+	econf2 := Sysconfig["elastic_2"].(map[string]interface{})
+	Es2 = &elastic.Elastic{
+		S_esurl: econf2["addr"].(string),
+		I_size:  util.IntAllDef(econf2["pool"], 5),
+	}
+	Es2.InitElasticSize()
+
+	if mi := util.ObjToString(biddingIndex["multiIndex"]); mi != "" {
+		multiIndex = strings.Split(mi, ",")
+	}
+	util.Debug(multiIndex)
+	biddingMgoFields = strings.Split(biddingIndex["mgofields"].(string), ",")
+	biddingEsFields = biddingIndex["esfieldsmap"].(map[string]interface{})             // bidding es字段
+	projectinfoFields = biddingIndex["projectinfomap"].(map[string]interface{})        // projectinfo
+	purchasinglistFields = biddingIndex["purchasinglistmap"].(map[string]interface{})  //采购清单
+	procurementlisFields = biddingIndex["procurementlistmap"].(map[string]interface{}) //采购意向
+	winnerorderlistFields = biddingIndex["winnerordermap"].(map[string]interface{})    //中标候选
+
+	FilterKeyword = util.ObjArrToStringArr(Sysconfig["filter-keyword"].([]interface{}))
+	initCheckCity()
+	//初始化oss
+	oss.InitOss()
+	InitFileInfo()
+
+	m := Sysconfig["jyfb_udp"].(map[string]interface{})
+	JyUdpAddr = &net.UDPAddr{
+		IP:   net.ParseIP(m["addr"].(string)),
+		Port: util.IntAll(m["port"]),
+	}
+
+	updateBiddingPool = make(chan []map[string]interface{}, 5000)
+	updateBiddingSp = make(chan bool, 5)
+	updateExtractPool = make(chan []map[string]interface{}, 5000)
+	updateExtractSp = make(chan bool, 5)
+	saveEsPool = make(chan map[string]interface{}, 5000)
+	saveEsSp = make(chan bool, 5)
+	saveEsAllPool = make(chan map[string]interface{}, 5000)
+	saveEsAllSp = make(chan bool, 5)
+	saveEsElsePool = make(chan map[string]interface{}, 5000)
+	saveEsElseSp = make(chan bool, 5)
+	saveProjectEsPool = make(chan map[string]interface{}, 5000)
+	saveProjectSp = make(chan bool, 5)
+}
+
+func NewTk(m map[string]interface{}) *TaskInfo {
+	p := &TaskInfo{
+		stype:       m["stype"].(string),
+		currentTime: time.Now().Unix(),
+	}
+	return p
+}
+
+type TaskInfo struct {
+	stype         string
+	thread        int
+	currentTime   int64
+	bidding_count int
+	extract_count int
+	es_count      int
+}
+
+type Province struct {
+	P_Name string
+}
+type City struct {
+	P_Name string
+	C_Name string
+}
+type District struct {
+	P_Name string
+	C_Name string
+	D_Name string
+}
+
+//初始化城市
+func initCheckCity() {
+	//初始化-城市配置
+	ProvinceDict = make(map[string][]Province, 0)
+	CityDict = make(map[string][]City, 0)
+	DistrictDict = make(map[string][]District, 0)
+
+	q := map[string]interface{}{
+		"town_code": map[string]interface{}{
+			"$exists": 0,
+		},
+	}
+	sess := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(sess)
+	it := sess.DB(standardMgo.DbName).C(util.ObjToString(standard["coll_area"])).Find(&q).Iter()
+	total := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		district_code := util.IntAll(tmp["district_code"])
+		city_code := util.IntAll(tmp["city_code"])
+		if district_code > 0 {
+			province := util.ObjToString(tmp["province"])
+			city := util.ObjToString(tmp["city"])
+			district := util.ObjToString(tmp["district"])
+			data := District{province, city, district}
+			if DistrictDict[district] == nil {
+				DistrictDict[district] = []District{data}
+			} else {
+				arr := DistrictDict[district]
+				arr = append(arr, data)
+				DistrictDict[district] = arr
+			}
+		} else {
+			if city_code > 0 {
+				province := util.ObjToString(tmp["province"])
+				city := util.ObjToString(tmp["city"])
+				data := City{province, city}
+				if CityDict[city] == nil {
+					CityDict[city] = []City{data}
+				} else {
+					arr := CityDict[city]
+					arr = append(arr, data)
+					CityDict[city] = arr
+				}
+			} else {
+				province := util.ObjToString(tmp["province"])
+				data := Province{province}
+				if ProvinceDict[province] == nil {
+					ProvinceDict[province] = []Province{data}
+				} else {
+					arr := ProvinceDict[province]
+					arr = append(arr, data)
+					ProvinceDict[province] = arr
+				}
+			}
+		}
+		tmp = make(map[string]interface{})
+	}
+	util.Debug(fmt.Sprintf("城市配置加载完毕...省~%d 市~%d 区~%d", len(ProvinceDict), len(CityDict), len(DistrictDict)))
+}

udpcreateindex/main.go

@@ -0,0 +1,484 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"gopkg.in/mgo.v2/bson"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+	util "utils"
+	"utils/nsq"
+	"utils/udp"
+)
+
+var (
+	Mcmer *gonsq.Consumer
+)
+
+func main() {
+	// company_id
+	//redis.InitRedis1("qyxy_id=172.17.4.189:8379", 4)
+	//inits()
+
+	//go inspectQuery()
+	//go checkMapJob()
+	//go task_index()
+	//go nsqMethod()
+
+	go UpdateBidding()
+	go UpdateExtract()
+	go SaveEsMethod()
+	go SaveAllEsMethod()
+	go SaveElseEsMethod()
+	go SaveProjectEs()
+	updport := Sysconfig["udpport"].(string)
+	udpclient = udp.UdpClient{Local: updport, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	util.Debug("Udp服务监听", updport)
+
+	ch := make(chan bool, 1)
+	<-ch
+}
+
+/**
+检查es查询队列  10s查询一次
+*/
+func inspectQuery() {
+	ticker := time.NewTicker(time.Second * 10)
+	url := esAddr + "/_nodes/stats/thread_pool"
+	for range ticker.C {
+		resp, _ := http.Get(url)
+		if resp != nil && resp.Body != nil {
+			defer resp.Body.Close()
+		}
+		body, _ := ioutil.ReadAll(resp.Body)
+		respMap := make(map[string]interface{})
+		err := json.Unmarshal(body, &respMap)
+		if err == nil {
+			if data, o1 := respMap["nodes"].(map[string]interface{}); o1 {
+				if nodes, o2 := data[esNode].(map[string]interface{}); o2 {
+					if pool, o3 := nodes["thread_pool"].(map[string]interface{}); o3 {
+						index, _ := pool["index"].(map[string]interface{})
+						search, _ := pool["search"].(map[string]interface{})
+						bulk, _ := pool["bulk"].(map[string]interface{})
+						if util.IntAll(index["queue"]) > 0 || util.IntAll(search["queue"]) > 0 || util.IntAll(bulk["queue"]) > 0 {
+							util.Debug("es thread_pool index queue---", index["queue"])
+							util.Debug("es thread_pool search queue---", search["queue"])
+							util.Debug("es thread_pool bulk queue---", bulk["queue"])
+							StopFlag = true
+						} else {
+							StopFlag = false
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+var pool = make(chan bool, 20)
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case udp.OP_TYPE_DATA: //上个节点的数据
+		//从表中开始处理生成企业数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Println("err:", err, "mapInfo:", mapInfo, string(data))
+		if err != nil {
+			udpclient.WriteUdp([]byte("err:"+err.Error()), udp.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			key, _ := mapInfo["key"].(string)
+			if key == "" {
+				key = "udpok"
+			}
+			go udpclient.WriteUdp([]byte(key), udp.OP_NOOP, ra)
+			tasktype, _ := mapInfo["stype"].(string)
+			t := NewTk(mapInfo)
+			switch tasktype {
+			case "winner":
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					winnerTask(data, mapInfo)
+				}()
+			case "bidding": //实时+udp调用，可选择是否生成关键词， 一次性最大20万
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					t.thread = 1
+					t.biddingTask(data, mapInfo)
+				}()
+			case "bidding_history": //增量id段历史数据
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					t.thread = 1
+					t.biddingTask(data, mapInfo)
+				}()
+			case "project":
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					projectTask(data, project, mapInfo)
+				}()
+			case "biddingback": //不联表，使用bidding表直接调用mongo库生成索引
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					t.thread = 30
+					t.biddingBackTask(data, mapInfo)
+				}()
+			case "biddingall": //合并并重新生成索引，不生成关键词
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					t.thread = 30
+					t.biddingAllTask(data, mapInfo)
+				}()
+			case "biddingdata": //bidding全量数据
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					t.thread = 30
+					t.biddingDataTask(data, mapInfo)
+				}()
+			case "biddingmerge": //重新合并但不生成索引，不生成关键词
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					biddingMergeTask(data, mapInfo)
+				}()
+			case "buyer":
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					buyerTask(data, mapInfo)
+				}()
+			case "winnerent": //标准库
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					standardTask("winnerent", mapInfo)
+				}()
+			case "buyerent": //标准库
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					standardTask("buyerent", mapInfo)
+				}()
+			case "agencyent": //标准库
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					standardTask("agencyent", mapInfo)
+				}()
+			case "biddingdelbyextract": //根据repeat删除es
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					biddingDelByExtract(data, mapInfo)
+				}()
+			case "biddingdelbyextracttype": //根据extracttype删除es
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					biddingDelByExtracttype(data, mapInfo)
+				}()
+			default:
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					util.Debug("err ---", mapInfo)
+				}()
+			}
+		}
+	case udp.OP_NOOP: //下个节点回应
+		log.Println("发送成功", string(data))
+	}
+}
+
+func SaveEsMethod() {
+	arru := make([]map[string]interface{}, EsBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveEsPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == EsBulkSize {
+				saveEsSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsSp
+					}()
+					//Es1.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+					Es2.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+					//if len(multiIndex) == 2 {
+					//	Es1.BulkSave(multiIndex[0], multiIndex[1], &arru, true)
+					//}
+				}(arru)
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveEsSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsSp
+					}()
+					//Es1.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+					Es2.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+					//if len(multiIndex) == 2 {
+					//	Es1.BulkSave(multiIndex[0], multiIndex[1], &arru, true)
+					//}
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveElseEsMethod() {
+	arru := make([]map[string]interface{}, EsBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveEsElsePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == EsBulkSize {
+				saveEsElseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsElseSp
+					}()
+					Es2.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+				}(arru)
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveEsElseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsElseSp
+					}()
+					Es2.BulkSave(util.ObjToString(biddingIndex["index"]), util.ObjToString(biddingIndex["type"]), &arru, true)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveAllEsMethod() {
+	arru := make([]map[string]interface{}, EsBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveEsAllPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == EsBulkSize {
+				saveEsAllSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsAllSp
+					}()
+					Es1.BulkSave("biddingall", "bidding", &arru, true)
+				}(arru)
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveEsAllSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsAllSp
+					}()
+					Es1.BulkSave("biddingall", "bidding", &arru, true)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveProjectEs() {
+	arru := make([]map[string]interface{}, EsBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveProjectEsPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == EsBulkSize {
+				saveProjectSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveProjectSp
+					}()
+					//Es1.BulkSave(util.ObjToString(project["index"]), util.ObjToString(project["type"]), &arru, true)
+					Es2.BulkSave(util.ObjToString(project["index"]), util.ObjToString(project["type"]), &arru, true)
+				}(arru)
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveProjectSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveProjectSp
+					}()
+					//Es1.BulkSave(util.ObjToString(project["index"]), util.ObjToString(project["type"]), &arru, true)
+					Es2.BulkSave(util.ObjToString(project["index"]), util.ObjToString(project["type"]), &arru, true)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func UpdateBidding() {
+	arru := make([][]map[string]interface{}, MgoBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updateBiddingPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == MgoBulkSize {
+				updateBiddingSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateBiddingSp
+					}()
+					biddingMgo.UpdateBulk(currentColl, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, MgoBulkSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateBiddingSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateBiddingSp
+					}()
+					biddingMgo.UpdateBulk(currentColl, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, MgoBulkSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func UpdateExtract() {
+	extract := util.ObjToString(extract["collect"])
+	arru := make([][]map[string]interface{}, MgoBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updateExtractPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == MgoBulkSize {
+				updateExtractSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateExtractSp
+					}()
+					extractMgo.UpdateBulk(extract, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, MgoBulkSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateExtractSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateExtractSp
+					}()
+					extractMgo.UpdateBulk(extract, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, MgoBulkSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+// @Description nsq处理id不变，内容替换的竞品数据
+// @Author J 2022/8/10 11:40
+func nsqMethod() {
+	cof := Sysconfig["nsq_id"].(map[string]interface{})
+	var err error
+	Mcmer, err = gonsq.NewConsumer(&gonsq.Cconfig{
+		IsJsonEncode: true, //与生产者配置对应，设为true会取第1个字节进行类型判断
+		Addr:         util.ObjToString(cof["addr"]),
+		ConnectType:  0, //默认连接nsqd
+		Topic:        util.ObjToString(cof["topic"]),
+		Channel:      util.ObjToString(cof["channel"]),
+		Concurrent:   util.IntAllDef(cof["concurrent"], 1), //并发数
+	})
+	if err != nil {
+		util.Debug("nsqMethod err: ", err.Error())
+	}
+	for {
+		select {
+		case obj := <-Mcmer.Ch: //从通道读取即可
+			util.Debug("index nsq: " + fmt.Sprint(obj))
+			id := strings.Split(util.ObjToString(obj), "=")
+			if bson.IsObjectIdHex(id[1]) {
+				taskinfo(id[1])
+			} else {
+				util.Debug("jy nsq id err: ", id[1])
+			}
+		}
+	}
+}

udpcreateindex/oss/ossclient.go

@@ -0,0 +1,66 @@
+// ossclient
+package oss
+
+import (
+	"fmt"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"io/ioutil"
+	"os"
+	util "utils"
+)
+
+var (
+	ossEndpoint        = "oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
+	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
+	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
+	ossBucketName      = "topjy"
+	ossclient          *oss.Client
+)
+
+func InitOss() {
+	client, err := oss.New(ossEndpoint, ossAccessKeyId, ossAccessKeySecret)
+	if err != nil {
+		fmt.Println("Error:", err)
+		os.Exit(-1)
+	}
+	ossclient = client
+}
+
+func OssGetObject(objectName string) string {
+	util.Catch()
+	// 获取存储空间。
+	bucket, err := ossclient.Bucket(ossBucketName)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+
+	// 下载文件到流。
+	body, err := bucket.GetObject(objectName)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+	defer body.Close()
+	data, err := ioutil.ReadAll(body)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+	return string(data)
+}
+
+func OssObjExists(bname, fid string) bool {
+	util.Catch()
+	// 获取存储空间。
+	bucket, err := ossclient.Bucket(bname)
+	if err != nil {
+		fmt.Println("Error:", err)
+	}
+	// 判断文件是否存在。
+	isExist, err := bucket.IsObjectExist(fid)
+	if err != nil {
+		fmt.Println("Error:", err)
+	}
+	return isExist
+}

udpcreateindex/projectindex.go

@@ -0,0 +1,186 @@
+package main
+
+import (
+	"math"
+	"strconv"
+	util "utils"
+	"utils/mongodb"
+)
+
+func projectTask(data []byte, project, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": map[string]interface{}{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	} else {
+		if q["pici"] == nil {
+			idMap, _ := q["_id"].(map[string]interface{})
+			if idMap != nil {
+				tmpQ := map[string]interface{}{}
+				for c, id := range idMap {
+					if idStr, ok := id.(string); ok && id != "" {
+						tmpQ[c] = mongodb.StringTOBsonId(idStr)
+					}
+				}
+				q["_id"] = tmpQ
+			}
+		}
+	}
+
+	conn := projectMgo.GetMgoConn()
+	defer projectMgo.DestoryMongoConn(conn)
+	c, _ := project["collect"].(string)
+	count, _ := conn.DB(projectMgo.DbName).C(c).Find(&q).Count()
+	util.Debug(c, "查询语句:", q, "同步总数:", count, "elastic库:", q["pici"])
+	query := conn.DB(projectMgo.DbName).C(c).Find(q).Iter()
+	n := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+		if n%20000 == 0 {
+			util.Debug("current---------", n)
+		}
+		pp := map[string]map[string]interface{}{}
+		if packages, ok := tmp["package"].(map[string]interface{}); ok {
+			for _, pks := range packages {
+				if pk, ok := pks.([]interface{}); ok {
+					for _, v := range pk {
+						if p, ok := v.(map[string]interface{}); ok {
+							winner := util.ObjToString(p["winner"])
+							bidamount := util.Float64All((p["bidamount"]))
+							if len(winner) > 4 && bidamount > 0 {
+								p := map[string]interface{}{
+									"winner":    winner,
+									"bidamount": bidamount,
+								}
+								pp[winner] = p
+							}
+						}
+					}
+				}
+			}
+		} else {
+			winner := util.ObjToString(tmp["winner"])
+			bidamount := util.Float64All(tmp["bidamount"])
+			if len(winner) > 4 && bidamount > 0 {
+				p := map[string]interface{}{
+					"winner":    winner,
+					"bidamount": bidamount,
+				}
+				pp[winner] = p
+			}
+		}
+
+		pk1 := []map[string]interface{}{}
+		for _, v := range pp {
+			pk1 = append(pk1, v)
+		}
+		if len(pk1) > 0 {
+			tmp["package1"] = pk1
+		}
+		budget := util.Float64All(tmp["budget"])
+		bidamount := util.Float64All(tmp["bidamount"])
+		if float64(budget) > 0 && float64(bidamount) > 0 {
+			rate := float64(1) - float64(bidamount)/float64(budget)
+			f, _ := strconv.ParseFloat(strconv.FormatFloat(rate, 'f', 4, 64), 64)
+			//不在0~0.6之间，不生成费率；只生成预算，中标金额舍弃，索引增加折扣率异常标识
+			if f < 0 || f > 0.6 {
+				delete(tmp, "bidamount")
+				tmp["prate_flag"] = 1
+			} else {
+				tmp["project_rate"] = f
+			}
+		}
+		if topscopeclass, ok := tmp["topscopeclass"].([]interface{}); ok {
+			tc := []string{}
+			m2 := map[string]bool{}
+			for _, v := range topscopeclass {
+				str := util.ObjToString(v)
+				str = reg_letter.ReplaceAllString(str, "") // 去除字母
+				if !m2[str] {
+					m2[str] = true
+					tc = append(tc, str)
+				}
+			}
+			tmp["topscopeclass"] = tc
+		}
+		//不生索引字段
+		delete(tmp, "package")
+		delete(tmp, "infofield")
+
+		bidopentime := util.Int64All(tmp["bidopentime"]) //开标日期
+		fzb_publishtime := int64(0)                      //记录第一个招标信息的publishtime
+		bidcycle_flag := false                           //判断是否已计算出标书表编制周期
+		list := tmp["list"].([]interface{})
+		for _, m := range list {
+			tmpM := m.(map[string]interface{})
+			//删除purchasing,review_experts
+			delete(tmpM, "purchasing")
+			delete(tmpM, "review_experts")
+			if bidamount, ok := tmpM["bidamount"].(string); ok && len(bidamount) > 0 { //bidamount为string类型，转成float
+				tmpB := util.Float64All(tmpM["bidamount"])
+				tmpM["bidamount"] = tmpB
+			}
+			//projectscope截断
+			listProjectscopeRune := []rune(util.ObjToString(tmpM["projectscope"]))
+			if len(listProjectscopeRune) > 1000 {
+				tmpM["projectscope"] = string(listProjectscopeRune[:1000])
+			}
+			//计算bidcycle标书表编制周期字段
+			if !bidcycle_flag && bidopentime > 0 { //bidopentime>0证明list中有bidopentime，无则不用计算bidcycle
+				if toptype := util.ObjToString(tmpM["toptype"]); toptype == "招标" {
+					zb_bidopentime := util.Int64All(tmpM["bidopentime"])
+					zb_publishtime := util.Int64All(tmpM["publishtime"])
+					if zb_publishtime > 0 {
+						if zb_bidopentime > 0 {
+							if tmpTime := zb_bidopentime - zb_publishtime; tmpTime > 0 {
+								f_day := float64(tmpTime) / float64(86400)
+								day := math.Ceil(f_day)
+								tmp["bidcycle"] = int(day)
+								bidcycle_flag = true
+							}
+						}
+						if fzb_publishtime == 0 { //仅赋值第一个招标信息的publishtime
+							fzb_publishtime = zb_publishtime
+						}
+					}
+				}
+			}
+		}
+		//计算bidcycle标书表编制周期字段
+		//list中招标信息中未能计算出bidcycle,用第一个招标信息的fzb_publishtime和外围bidopentime计算
+		if !bidcycle_flag && bidopentime > 0 && fzb_publishtime > 0 {
+			if tmpTime := bidopentime - fzb_publishtime; tmpTime > 0 {
+				f_day := float64(tmpTime) / float64(86400)
+				day := math.Ceil(f_day)
+				tmp["bidcycle"] = int(day)
+			}
+		}
+		//projectscope截断
+		projectscopeRune := []rune(util.ObjToString(tmp["projectscope"]))
+		if len(projectscopeRune) > 1000 {
+			tmp["projectscope"] = string(projectscopeRune[:1000])
+		}
+		//		if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
+		//			tmp["budget"] = nil
+		//		}
+		//		if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
+		//			tmp["bidamount"] = nil
+		//		}
+		//go IS.Add("project")
+
+		if tmp["bidamount"] != nil && util.Float64All(tmp["bidamount"]) > 1000000000 {
+			delete(tmp, "bidamount")
+		}
+		if tmp["budget"] != nil && util.Float64All(tmp["budget"]) > 1000000000 {
+			delete(tmp, "budget")
+		}
+		tmp["s_projectname"] = tmp["projectname"]
+		saveProjectEsPool <- tmp
+		tmp = make(map[string]interface{})
+	}
+	util.Debug(mapInfo, "create project index...over", n)
+}

udpcreateindex/standardata.go

@@ -0,0 +1,175 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	"log"
+	util "utils"
+	"utils/elastic"
+	"utils/mongodb"
+)
+
+func standardTask(stype string, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	switch stype {
+	case "winnerent":
+		winnerEnt(q)
+	case "buyerent":
+		buyerEnt(q)
+	case "agencyent":
+		agencyEnt(q)
+	}
+}
+
+//winnerent
+func winnerEnt(q map[string]interface{}) {
+	session := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(session)
+	winnerent, _ := standard["winnerent"].(map[string]interface{})
+	c, _ := winnerent["collect"].(string)
+	index, _ := winnerent["index"].(string)
+	itype, _ := winnerent["type"].(string)
+	count, _ := session.DB(standardMgo.DbName).C(c).Find(&q).Count()
+	savepool := make(chan bool, 10)
+
+	log.Println(standardMgo.DbName, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
+
+	query := session.DB(standardMgo.DbName).C(c).Find(q).Iter()
+	arr := make([]map[string]interface{}, EsBulkSize)
+	var n int
+	i := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+		//不生索引字段
+		delete(tmp, "partners")
+		delete(tmp, "wechat_accounts")
+		delete(tmp, "tmp_id")
+		tmp["company"] = tmp["company_name"]
+		arr[i] = tmp
+		n++
+		if i == EsBulkSize-1 {
+			savepool <- true
+			tmps := arr
+			go func(tmpn *[]map[string]interface{}) {
+				defer func() {
+					<-savepool
+				}()
+				elastic.BulkSave(index, itype, tmpn, true)
+			}(&tmps)
+			i = 0
+			arr = make([]map[string]interface{}, EsBulkSize)
+		}
+		if n%EsBulkSize == 0 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	if i > 0 {
+		elastic.BulkSave(index, itype, &arr, true)
+	}
+	log.Println("create winnerent index...over", n)
+}
+
+//buyerent
+func buyerEnt(q map[string]interface{}) {
+	session := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(session)
+	buyerent, _ := standard["buyerent"].(map[string]interface{})
+	c, _ := buyerent["collect"].(string)
+	index, _ := buyerent["index"].(string)
+	itype, _ := buyerent["type"].(string)
+	count, _ := session.DB(standardMgo.DbName).C(c).Find(&q).Count()
+	savepool := make(chan bool, 10)
+
+	log.Println(standardMgo.DbName, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
+
+	query := session.DB(standardMgo.DbName).C(c).Find(q).Iter()
+	arr := make([]map[string]interface{}, EsBulkSize)
+	var n int
+	i := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+		//不生索引字段
+		delete(tmp, "partners")
+		delete(tmp, "wechat_accounts")
+		delete(tmp, "tmp_id")
+		tmp["buyer"] = tmp["buyer_name"]
+		arr[i] = tmp
+		n++
+		if i == EsBulkSize-1 {
+			savepool <- true
+			tmps := arr
+			go func(tmpn *[]map[string]interface{}) {
+				defer func() {
+					<-savepool
+				}()
+				elastic.BulkSave(index, itype, tmpn, true)
+			}(&tmps)
+			i = 0
+			arr = make([]map[string]interface{}, EsBulkSize)
+		}
+		if n%EsBulkSize == 0 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	if i > 0 {
+		elastic.BulkSave(index, itype, &arr, true)
+	}
+	log.Println("create buyerent index...over", n)
+}
+
+//agencyent
+func agencyEnt(q map[string]interface{}) {
+	session := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(session)
+	agencyent, _ := standard["agencyent"].(map[string]interface{})
+	c, _ := agencyent["collect"].(string)
+	index, _ := agencyent["index"].(string)
+	itype, _ := agencyent["type"].(string)
+	count, _ := session.DB(standardMgo.DbName).C(c).Find(&q).Count()
+	savepool := make(chan bool, 10)
+
+	log.Println(standardMgo.DbName, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
+
+	query := session.DB(standardMgo.DbName).C(c).Find(q).Iter()
+	arr := make([]map[string]interface{}, EsBulkSize)
+	var n int
+	i := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+		//不生索引字段
+		delete(tmp, "partners")
+		delete(tmp, "wechat_accounts")
+		delete(tmp, "tmp_id")
+
+		tmp["agency"] = tmp["agency_name"]
+		arr[i] = tmp
+		n++
+		if i == EsBulkSize-1 {
+			savepool <- true
+			tmps := arr
+			go func(tmpn *[]map[string]interface{}) {
+				defer func() {
+					<-savepool
+				}()
+				elastic.BulkSave(index, itype, tmpn, true)
+			}(&tmps)
+			i = 0
+			arr = make([]map[string]interface{}, EsBulkSize)
+		}
+		if n%EsBulkSize == 0 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	if i > 0 {
+		elastic.BulkSave(index, itype, &arr, true)
+	}
+	log.Println("create agencyent index...over", n)
+}

udpcreateindex/task.go

@@ -0,0 +1,40 @@
+// task定时执行项目索引
+package main
+
+import (
+	"github.com/robfig/cron"
+	util "utils"
+)
+
+func task_index() {
+
+	c := cron.New()
+	//c.AddFunc("20 30 5 * * *", func() { task_projects() })
+	//c.AddFunc("0 30 * * * *", func() { task_biddingfile() }) //每30分钟执行一次
+	//c.AddFunc("0 22 14 * * *", func() { task_qyxyindex() })
+
+	_ = c.AddFunc("0 0 0 * * ?", func() { task_winneres() }) //每天凌晨执行一次winner生索引
+	_ = c.AddFunc("0 0 1 * * ?", func() { task_buyeres() })  //每天1点执行一次buyer生索引
+	//_ = c.AddFunc("0 0 2 * * ?", func() { task_biddingAll() }) //每天2点执行 前一天的所有招标数据
+	c.Start()
+}
+func task_winneres() {
+	util.Debug("定时任务，winneres")
+	winnerEsTaskOnce()
+}
+func task_buyeres() {
+	util.Debug("定时任务，buyeres")
+	buyerEsTaskOnce()
+}
+
+//func task_biddingAll() {
+//	qutil.Debug("定时任务，bidding_all")
+//	t := time.Now()
+//	currenttime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
+//	endtime := time.Unix(currenttime.Unix(), 0).AddDate(0, 0, -1)
+//
+//	sid := primitive.NewObjectIDFromTimestamp(endtime)
+//	eid := primitive.NewObjectIDFromTimestamp(currenttime)
+//
+//	//biddingDataTask(nil, map[string]interface{}{"gtid": mongodb.BsonIdToSId(sid), "lteid": mongodb.BsonIdToSId(eid)})
+//}

udpcreateindex/tonumber.go

@@ -0,0 +1,280 @@
+// tonumber
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var contentUnit *regexp.Regexp          //全文检索单位：万元
+var regOperator *regexp.Regexp          //运算符号
+var regNumFloat *regexp.Regexp          //提取整数或浮点数
+var moneyRegChar *regexp.Regexp         //提取中文数字
+var regStrUnit *regexp.Regexp           //提取单位
+var moneyChar = map[string]interface{}{ //"〇": "0", "零": "0",
+	"一": float64(1), "壹": float64(1), "二": float64(2), "贰": float64(2), "三": float64(3), "叁": float64(3), "四": float64(4), "肆": float64(4), "五": float64(5), "伍": float64(5),
+	"六": float64(6), "陆": float64(6), "七": float64(7), "柒": float64(7), "八": float64(8), "捌": float64(8), "九": float64(9), "玖": float64(9), "十": float64(10), "拾": float64(10),
+	"百": float64(100), "佰": float64(100), "千": float64(1000), "仟": float64(1000), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000),
+	"零": float64(0), "点": ".", "角": float64(0.1), "分": float64(0.01),
+}
+var moneyUnit = map[string]float64{
+	"元": float64(1), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000), //单位
+}
+var spaces = []string{"\u3000", "\u2003", "\u00a0"}
+var cutAllSpace = regexp.MustCompile(`\s*`)
+
+func init() {
+	regOperator, _ = regexp.Compile(`[*|+|)*)]`)
+	regNumFloat, _ = regexp.Compile(`([1-9]\d*|0)(\.\d+)?`)
+	regStrUnit, _ = regexp.Compile(`[元|万|亿]`)
+
+	regStrChar := `[〇|零|点|.|．|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]`
+	moneyRegChar, _ = regexp.Compile(regStrChar)
+
+	contentUnit, _ = regexp.Compile(`(万元|单位/万)`)
+}
+
+//转int
+func ObjToInt(data []interface{}) []interface{} {
+	tmp, err := strconv.Atoi(fmt.Sprint(data[0]))
+	if err != nil {
+		data[0] = 0
+		return data
+	} else {
+		data[0] = tmp
+		return data
+	}
+}
+
+//转float,精度小数点4位
+func ObjToFloat(data []interface{}) []interface{} {
+	tmp, err := strconv.ParseFloat(fmt.Sprint(data[0]), 64)
+	if err != nil {
+		return []interface{}{float64(0), data[1]}
+	} else {
+		tmp, err = strconv.ParseFloat(strconv.FormatFloat(tmp, 'f', 4, 64), 64)
+		if err != nil {
+			return []interface{}{float64(0), data[1]}
+		} else {
+			return []interface{}{tmp, data[1]}
+		}
+	}
+}
+
+//金额转换
+func ObjToMoney(data []interface{}) []interface{} {
+	isfindUnit := true
+	ret := capitalMoney(data)[0]
+	if ret.(float64) < float64(10000) || ret.(float64) > float64(50000000000) {
+		ret2, b := numMoney(data)
+		isfindUnit = b
+		if ret2[0].(float64) > ret.(float64) {
+			ret = ret2[0]
+		}
+	}
+	f, _ := strconv.ParseFloat(strconv.FormatFloat(ret.(float64), 'f', 4, 64), 64)
+	if f < 1 {
+		f = 0
+	}
+	//若果金额小于50，全文检索单位：万
+	if f < 50 && f > 0 && isfindUnit {
+		rep := contentUnit.FindAllStringIndex(fmt.Sprint(data[1]), -1)
+		if len(rep) > 0 {
+			f = f * 10000
+		}
+	}
+	data[0] = f
+	return data
+}
+
+//清理所有空白符
+func CutAllSpace(data []interface{}) []interface{} {
+	tmp := cutAllSpace.ReplaceAllString(fmt.Sprint(data[0]), "")
+	tmp = replaceSymbol(tmp, spaces)
+	data[0] = tmp
+	return data
+}
+
+//数字金额转换
+func numMoney(data []interface{}) ([]interface{}, bool) {
+	tmp := fmt.Sprint(data[0])
+	tmp = replaceSymbol(tmp, []string{",", "，", "（", "）", "(", ")", "：", "\n"})
+	tmp = replaceString(tmp, []string{"万元", "亿元", "．"}, []string{"万", "亿", "."})
+	tmp = fmt.Sprint(CutAllSpace([]interface{}{tmp, data[1]})[0])
+	rets := regNumFloat.FindAllString(tmp, -1)
+	fnums := []float64{}
+	unitstrs := []string{}
+	if len(rets) > 0 {
+		pindex := 0 //单位前置
+		for k, v := range rets {
+			f, err := strconv.ParseFloat(v, 64)
+			if err == nil {
+				fnums = append(fnums, f)
+				index := strings.Index(tmp, v)
+				//单位后置
+				start := index + len(v)
+				end := start + 3
+				//log.Println("vvv", tmp, v, pindex, index, start)
+				if k > 0 {
+					if start >= pindex+3 {
+						pstart := pindex + 3
+						if pstart >= index {
+							pstart = index
+						}
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:])
+						}
+					} else {
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				} else {
+					if len(tmp) > end {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:end])
+						}
+					} else {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				}
+				pindex = start
+			}
+		}
+	}
+	//log.Println("unitstrs", fnums, unitstrs)
+	unit := float64(0)
+	fnum := float64(0)
+	for k, v := range fnums {
+		fnum = v
+		units := regStrUnit.FindAllString(unitstrs[k], -1)
+		for _, v := range units {
+			if moneyUnit[v] != 0 {
+				unit = moneyUnit[v]
+				break
+			}
+		}
+		if unit != float64(0) { //取第一个
+			break
+		}
+	}
+	if unit == float64(0) {
+		data[0] = fnum
+	} else {
+		data[0] = fnum * unit
+	}
+	if unit == 10000 {
+		return data, false
+	} else {
+		return data, true
+	}
+}
+
+//大写数子金额转换
+func capitalMoney(data []interface{}) []interface{} {
+	nodes := []float64{}
+	node := float64(0)
+	tmp := float64(0)
+	decimals := 0.0
+	ishaspoint := false //是否含小数点
+	fnum := float64(0)
+	moneyRegChar.ReplaceAllStringFunc(fmt.Sprint(data[0]), func(key string) string {
+		if key == "元" || key == "圆" || key == "点" {
+			ishaspoint = true
+		}
+		if v, ok := moneyChar[key].(float64); ok {
+			if (ishaspoint && v > 10) || key == "整" || key == "正" { //排除后面有其他的单位
+				return ""
+			}
+			//log.Println(key, v, fnum)
+			if v < 10 && v >= 0 {
+				if ishaspoint { //小数部分
+					if v >= 1 {
+						fnum = v
+					} else if v < 1 && v > 0 {
+						decimals += fnum * v
+					}
+				} else {
+					if tmp != float64(0) {
+						node += tmp
+					}
+					tmp = float64(v)
+				}
+			} else if v == 10000 || v == 100000000 { //单位万、亿
+				if tmp != float64(0) {
+					node += tmp
+					tmp = float64(0)
+				}
+				nodes = append(nodes, node*float64(v))
+				node = float64(0)
+			} else {
+				if v == 10 && tmp == 0 {
+					tmp = 1
+				}
+				tmp = tmp * float64(v)
+				node += tmp
+				tmp = float64(0)
+			}
+		}
+		return ""
+	})
+	nodes = append(nodes, node, tmp)
+	ret := float64(0)
+	for _, v := range nodes {
+		ret = ret + v
+	}
+	return []interface{}{ret + decimals, data[1]}
+}
+
+//过滤符号
+func replaceSymbol(con string, rep []string) string {
+	for _, v := range rep {
+		con = strings.Replace(con, v, "", -1)
+	}
+	return con
+}
+
+//符号替换
+func replaceString(con string, ret, rep []string) string {
+	for k, v := range ret {
+		if len(rep) > k {
+			con = strings.Replace(con, v, rep[k], -1)
+		}
+	}
+	return con
+}
+
+//费率转小数
+func RateToFloat(con []interface{}) []interface{} {
+	tmp := fmt.Sprint(CutAllSpace(con)[0])
+	if strings.Contains(tmp, "%") || strings.Contains(tmp, "％") {
+		tmp = strings.Replace(tmp, "%", "", -1)
+		tmp = strings.Replace(tmp, "％", "", -1)
+		rep := ObjToFloat([]interface{}{tmp, con[1]})[0]
+		con[0] = rep.(float64) / 100
+		return con
+	} else {
+		return ObjToFloat([]interface{}{tmp, con[1]})
+	}
+}
+
+//大于一万亿的过滤掉
+func ClearMaxAmount(data []interface{}) []interface{} {
+	value, _ := data[0].(float64)
+	if value >= 1000000000000 {
+		data[0] = float64(0)
+	}
+	return data
+}

udpcreateindex/udptaskmap.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+	"utils/udp"
+)
+
+var udptaskmap = &sync.Map{}
+var tomail string
+var api string
+
+type udpNode struct {
+	data      []byte
+	addr      *net.UDPAddr
+	timestamp int64
+	retry     int
+}
+
+func checkMapJob() {
+	//阿里云内网无法发送邮件
+	jkmail, _ := Sysconfig["jkmail"].(map[string]interface{})
+	if jkmail != nil {
+		tomail, _ = jkmail["to"].(string)
+		api, _ = jkmail["api"].(string)
+	}
+	log.Println("start checkMapJob", tomail, Sysconfig["jkmail"])
+	for {
+		udptaskmap.Range(func(k, v interface{}) bool {
+			now := time.Now().Unix()
+			node, _ := v.(*udpNode)
+			if now-node.timestamp > 120 {
+				node.retry++
+				if node.retry > 5 {
+					log.Println("udp重试失败", k)
+					udptaskmap.Delete(k)
+					res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, tomail, "extract-send-fail", k.(string)))
+					if err == nil {
+						defer res.Body.Close()
+						read, err := ioutil.ReadAll(res.Body)
+						log.Println("邮件发发送：", string(read), err)
+					}
+				} else {
+					log.Println("udp重发", k)
+					udpclient.WriteUdp(node.data, udp.OP_TYPE_DATA, node.addr)
+				}
+			} else if now-node.timestamp > 10 {
+				log.Println("udp任务超时中..", k)
+			}
+			return true
+		})
+		time.Sleep(60 * time.Second)
+	}
+}

udpcreateindex/winnerindex.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	"log"
+	util "utils"
+	"utils/elastic"
+	"utils/mongodb"
+)
+
+func winnerTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	session := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(session)
+	c, _ := winner["collect"].(string)
+	index, _ := winner["index"].(string)
+	itype, _ := winner["type"].(string)
+	count, _ := session.DB(standardMgo.DbName).C(c).Find(&q).Count()
+	savepool := make(chan bool, 10)
+
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	query := session.DB(standardMgo.DbName).C(c).Find(q).Select(bson.M{"pici": 0}).Iter()
+
+	arr := make([]map[string]interface{}, EsBulkSize)
+	var n int
+	i := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+		go IS.Add("winner")
+		arr[i] = tmp
+		n++
+		if i == EsBulkSize-1 {
+			savepool <- true
+			tmps := arr
+			go func(tmpn *[]map[string]interface{}) {
+				defer func() {
+					<-savepool
+				}()
+				elastic.BulkSave(index, itype, tmpn, true)
+			}(&tmps)
+			i = 0
+			arr = make([]map[string]interface{}, EsBulkSize)
+		}
+		if n%EsBulkSize == 0 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	if i > 0 {
+		elastic.BulkSave(index, itype, &arr, true)
+	}
+	log.Println(mapInfo, "create winner index...over", n)
+}

udpcreateindex/winnertask.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"log"
+	"sync"
+	"time"
+	util "utils"
+	"utils/mongodb"
+)
+
+func winnerEsTaskOnce() {
+	defer util.Catch()
+	arrEs := []map[string]interface{}{}
+	winerEsLock := &sync.Mutex{}
+	pool := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+
+	now := time.Now()
+	preTime := time.Date(now.Year(), now.Month(), now.Day()-1, now.Hour(), 0, 0, 0, time.Local)
+	curTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.Local)
+	task_sid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(preTime))
+	task_eid := mongodb.BsonIdToSId(primitive.NewObjectIDFromTimestamp(curTime))
+	//task_sid = "5e6598f82c27dc56292158da"
+	//task_eid = "620576342566c40049f26155"
+	log.Println("winner 区间id:", task_sid, task_eid)
+	//区间id
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			//"$gte": mongodb.StringTOBsonId(task_sid),
+			"$lt": mongodb.StringTOBsonId(task_eid),
+		},
+	}
+	//参数
+	winnerent, _ := standard["winnerent"].(map[string]interface{})
+	win_ent := util.ObjToString(winnerent["collect1"])
+	//win_enterr := qu.ObjToString(winnerent["collect2"])
+	index, _ := winnerent["index"].(string)
+	itype, _ := winnerent["type"].(string)
+	//mongo
+	sess := standardMgo.GetMgoConn()
+	defer standardMgo.DestoryMongoConn(sess)
+
+	log.Println("q:", q, "db:", standardMgo.DbName, "coll:", win_ent)
+	it_1 := sess.DB(standardMgo.DbName).C(win_ent).Find(&q).Sort("_id").Iter()
+	num_1 := 0
+	for tmp := make(map[string]interface{}); it_1.Next(&tmp); num_1++ {
+		if num_1%2000 == 0 && num_1 > 0 {
+			log.Println("当前表:", win_ent, "数量:", num_1)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			savetmp := map[string]interface{}{}
+			tmp_id := mongodb.BsonIdToSId(tmp["_id"])
+			savetmp["_id"] = tmp_id
+			savetmp["name"] = tmp["company_name"]
+			savetmp["winner_name"] = tmp["company_name"]
+			savetmp["pici"] = tmp["updatetime"]
+			if province := util.ObjToString(tmp["province"]); province != "" {
+				savetmp["province"] = province
+			}
+			if city := util.ObjToString(tmp["city"]); city != "" {
+				savetmp["city"] = city
+			}
+			if text := util.ObjToString(tmp["tag_business"]); text != "" {
+				savetmp["tag_business"] = text
+			}
+
+			winerEsLock.Lock()
+			arrEs = append(arrEs, savetmp)
+			if len(arrEs) >= EsBulkSize {
+				tmps := arrEs
+				//Es1.BulkSave(index, itype, &tmps, true)
+				Es2.BulkSave(index, itype, &tmps, true)
+				arrEs = []map[string]interface{}{}
+			}
+			winerEsLock.Unlock()
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+
+	// log.Println("q:", q, "db:", mgostandard.DbName, "coll:", win_enterr)
+	// it_2 := sess.DB(mgostandard.DbName).C(win_enterr).Find(&q).Sort("_id").Iter()
+	// num_2 := 0
+	// for tmp := make(map[string]interface{}); it_2.Next(&tmp); num_2++ {
+	// 	if num_2%100 == 0 && num_2 > 0 {
+	// 		log.Println("当前表:", win_enterr, "数量:", num_2)
+	// 	}
+	// 	pool <- true
+	// 	wg.Add(1)
+	// 	go func(tmp map[string]interface{}) {
+	// 		defer func() {
+	// 			<-pool
+	// 			wg.Done()
+	// 		}()
+	// 		savetmp := map[string]interface{}{}
+	// 		tmp_id := mongodb.BsonIdToSId(tmp["_id"])
+	// 		savetmp["_id"] = tmp_id
+	// 		savetmp["name"] = tmp["name"]
+	//      savetmp["winner_name"] = tmp["name"]
+	// 		savetmp["pici"] = tmp["updatetime"]
+	// 		winerEsLock.Lock()
+	// 		arrEs = append(arrEs, savetmp)
+	// 		if len(arrEs) >= BulkSize {
+	// 			tmps := arrEs
+	// 			elastic.BulkSave(index, itype, &tmps, true)
+	// 			arrEs = []map[string]interface{}{}
+	// 		}
+	// 		winerEsLock.Unlock()
+	// 	}(tmp)
+	// 	tmp = make(map[string]interface{})
+	// }
+
+	wg.Wait()
+	winerEsLock.Lock()
+	if len(arrEs) > 0 {
+		tmps := arrEs
+		//Es1.BulkSave(index, itype, &tmps, true)
+		Es2.BulkSave(index, itype, &tmps, true)
+		arrEs = []map[string]interface{}{}
+	}
+	winerEsLock.Unlock()
+	log.Println("winnerextract  索引完毕！  总计:", num_1)
+}