Merge branch 'dev2.1' of ssh://192.168.3.207:10022/qmx/qfw into dev2.1

common/src/github.com/donnie4w/go-logger/logger/logw.go

@@ -221,7 +221,11 @@ func (this *logBean) console(v ...interface{}) {
 
 func (this *logBean) log(level string, v ...interface{}) {
 	defer catchError()
-	s := fmt.Sprint(v...)
+	//s := fmt.Sprint(v...)
+	s := ""
+	for _, val := range v {
+		s += " " + fmt.Sprint(val)
+	}
 	length := len([]byte(s))
 	var lg *fileBean = fbf.get(this.id)
 	var _level = ALL

common/src/github.com/go-gomail/gomail/smtp.go

@@ -59,6 +59,7 @@ func NewPlainDialer(host string, port int, username, password string) *Dialer {
 // should be closed when done using it.
 func (d *Dialer) Dial() (SendCloser, error) {
 	conn, err := netDialTimeout("tcp", addr(d.Host, d.Port), 10*time.Second)
+	conn.SetDeadline(time.Now().Add(20 * time.Second))
 	if err != nil {
 		return nil, err
 	}

common/src/qfw/util/common.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"log"
 	"math"
+	"math/big"
 	mathRand "math/rand"
 	"net/url"
 	"regexp"
@@ -239,6 +240,9 @@ func Int64All(num interface{}) int64 {
 		return int64(i5)
 	} else if i6, ok6 := num.(int8); ok6 {
 		return int64(i6)
+	} else if i7, ok7 := num.(*big.Int); ok7 {
+		in, _ := strconv.ParseInt(fmt.Sprint(i7), 10, 64)
+		return int64(in)
 	} else {
 		return 0
 	}
@@ -272,6 +276,9 @@ func Float64All(num interface{}) float64 {
 		return float64(i6)
 	} else if i6, ok6 := num.(uint64); ok6 {
 		return float64(i6)
+	} else if i7, ok7 := num.(*big.Float); ok7 {
+		in, _ := strconv.ParseFloat(fmt.Sprint(i7), 64)
+		return float64(in)
 	} else {
 		return 0
 	}
@@ -295,6 +302,9 @@ func IntAllDef(num interface{}, defaultNum int) int {
 		return int(i5)
 	} else if i6, ok6 := num.(int8); ok6 {
 		return int(i6)
+	} else if i7, ok7 := num.(*big.Int); ok7 {
+		in, _ := strconv.Atoi(fmt.Sprint(i7))
+		return int(in)
 	} else {
 		return defaultNum
 	}

common/src/qfw/util/elastic/elasticutil.go

@@ -1003,6 +1003,11 @@ const (
 	FilterQuery_New  = `{"query":{"bool":{"must": [%s%s%s],"should":[]}}}`
 	MatchQueryString = `{"match": {%s: { "query":"%s", "operator": "and"}}}`
 	HL_New           = `"highlight": {"pre_tags": ["<HL>"],"post_tags": ["<HL>"],"fields": {%s}}`
+
+	ik_highlightStr = `%s: {"fragment_size": %d,"number_of_fragments": 1,"require_field_match": true}`
+	IK_pre_tags     = `<font class=\"es-highlight\">`
+	IK_post_tags    = `</font>`
+	HL_IK           = `"highlight": {"pre_tags": ["` + IK_pre_tags + `"],"post_tags": ["` + IK_post_tags + `"],"fields": {%s}}`
 )
 
 //替换了"号
@@ -1154,13 +1159,15 @@ func GetByNgramAll(index, itype string, query interface{}, mustquery, findfields
 		if len(fields) > 0 {
 			qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
 		}
-		if len(order) > 0 {
+		if strings.HasPrefix(order, "CUSTOM_") {
+			qstr = qstr[:len(qstr)-1] + `,` + strings.TrimLeft(order, "CUSTOM_") + `}`
+		} else if len(order) > 0 {
 			qstr = qstr[:len(qstr)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
 		}
 		if start > -1 {
 			qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
 		}
-		//	log.Println("ngram-find", qstr)
+		//log.Println("ngram-find", qstr)
 		return Get(index, itype, qstr)
 	} else {
 		return nil
@@ -1248,7 +1255,7 @@ func GetResForJY(index, itype string, keys []KeyConfig, allquery, findfields, So
 		if start > -1 {
 			qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
 		}
-		log.Println("jy-ngram-find", qstr)
+		//log.Println("jy-ngram-find", qstr)
 		return Get(index, itype, qstr)
 	} else {
 		return nil
@@ -1284,3 +1291,29 @@ func GetAllByNgram(index, itype, qstr, findfields, order, fields string, start,
 		return nil
 	}
 }
+
+//ik 分词
+func GetAllByIk(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
+	if qstr != "" {
+		if highlight {
+			ws := []string{}
+			for _, w := range strings.Split(findfields, ",") {
+				ws = append(ws, fmt.Sprintf(ik_highlightStr, w, count))
+			}
+			qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL_IK, strings.Join(ws, ",")) + `}`
+		}
+		if len(fields) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
+		}
+		if len(order) > 0 {
+			qstr = qstr[:len(qstr)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
+		}
+		if start > -1 {
+			qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
+		}
+		//log.Println("GetAllByNgram：", qstr)
+		return Get(index, itype, qstr)
+	} else {
+		return nil
+	}
+}

common/src/qfw/util/mail/gmail.go

@@ -2,11 +2,10 @@ package mail
 
 import (
 	"log"
+	"qfw/util"
 	"sync"
 	"time"
 
-	"qfw/util"
-
 	"github.com/go-gomail/gomail"
 )
 
@@ -20,11 +19,11 @@ type GmailAuth struct {
 var mailLock = &sync.Mutex{}
 var dialer *gomail.Dialer
 
-func getDialer(flag bool, auth *GmailAuth) *gomail.Dialer {
+func getDialer(flag bool, auth *GmailAuth, to string) *gomail.Dialer {
 	if flag && dialer != nil {
 		return dialer
 	}
-	log.Println("邮件getDialer：get new gomail Dialer")
+	log.Println(to, "发送邮件getDialer：get new gomail Dialer")
 	dialer = gomail.NewPlainDialer(auth.SmtpHost, auth.SmtpPort, auth.User, auth.Pwd) // 发送邮件服务器、端口、发件人账号、发件人密码
 	return dialer
 }
@@ -46,22 +45,21 @@ func GSendMail(from, to, cc, bcc, subject, body, fname, rename string, auth *Gma
 		m.Attach(fname, gomail.Rename(rename), gomail.SetHeader(h)) //添加附件
 		//m.Attach(fname) //添加附件
 	}
-	return gSend(auth, m)
+	return gSend(auth, m, to)
 }
 
 //
-func gSend(auth *GmailAuth, m *gomail.Message) bool {
+func gSend(auth *GmailAuth, m *gomail.Message, to string) bool {
 	mailLock.Lock()
 	defer mailLock.Unlock()
 	defer util.Catch()
-	var err error
-	for i := 0; i < 2; i++ {
-		if err = getDialer(i == 0, auth).DialAndSend(m); err == nil {
-			return true
-		} else {
+	for i := 0; i < 3; i++ {
+		if err := getDialer(i == 0, auth, to).DialAndSend(m); err != nil {
+			log.Println(to, "第", i+1, "次发送邮件gSend error：", err)
 			time.Sleep(200 * time.Millisecond)
+		} else {
+			return true
 		}
 	}
-	log.Println("邮件gSend：", err)
 	return false
 }

common/src/qfw/util/redis/redisutil.go

@@ -473,6 +473,22 @@ func RPUSH(code, list string, val interface{}) bool {
 	return true
 }
 
+func LLEN(code, list string) int64 {
+	defer catch()
+	conn := RedisPool[code].Get()
+	defer conn.Close()
+	ret, err := conn.Do("LLEN", list)
+	if nil != err {
+		log.Println("redisutil-LLENError", err)
+		return 0
+	}
+	if res, ok := ret.(int64); ok {
+		return res
+	} else {
+		return 0
+	}
+}
+
 func catch() {
 	if r := recover(); r != nil {
 		log.Println(r)

core/src/config.json

@@ -8,6 +8,10 @@
     "elasticPoolSize": 30,
     "mongodbPoolSize": 5,
     "mongodbName": "qfw",
+	"cassandra": [
+        "192.168.3.18"
+    ],
+    "cassandrasize":5,
     "smtp": {
         "addr": "smtp.exmail.qq.com",
         "port": 465,

core/src/main.go

@@ -37,6 +37,7 @@ import (
 	_ "qfw/yellowpage"
 	"runtime"
 	"time"
+	ca "ucbsutil/cassandra"
 
 	"github.com/go-xweb/xweb"
 )
@@ -52,6 +53,9 @@ func init() {
 	} else {
 		util.ReadConfig(configfile, &SysConfig)
 	}
+	//初始化cassandra
+	ca.InitCassandra("jianyu", SysConfig.Cassandrasize, SysConfig.Cassandra)
+
 	//mongodb初始化
 	//util.MongodbAddr = SysConfig.Mongodbservers
 	mongodb.InitMongodbPool(SysConfig.MongodbPoolSize, SysConfig.Mongodbservers, SysConfig.MongodbName)

core/src/qfw/coreconfig/SysConfig.go

@@ -11,6 +11,8 @@ type config struct {
 	Mongodbservers     string                 `json:"mongodbServers"` //mongodb节点
 	MongodbPoolSize    int                    `json:"mongodbPoolSize"`
 	MongodbName        string                 `json:"mongodbName"`
+	Cassandra          []string               `json:"cassandra"`
+	Cassandrasize      int                    `json:"cassandrasize"`
 	Elasticsearch      string                 `json:"elasticsearch"`
 	JyEscsearch        string                 `json:"jyescsearch"` //content内容
 	Smtp               smtp                   `json:"smtp"`

core/src/qfw/filter/logfilter.go

@@ -7,13 +7,18 @@
 package filter
 
 import (
-	"github.com/go-xweb/xweb"
 	"net"
 	"net/http"
 	"qfw/util/mongodb"
 	"strings"
 	"sync"
 	"time"
+
+	"encoding/json"
+	uc "ucbsutil"
+	ca "ucbsutil/cassandra"
+
+	"github.com/go-xweb/xweb"
 )
 
 //日志过滤器
@@ -60,38 +65,45 @@ func addLog(l *logfilter, w http.ResponseWriter, req *http.Request) {
 			})
 		}()
 	}
-
+	md, _ := json.Marshal(req.Form)
+	str := string(md)
 	logs := map[string]interface{}{
-		"l_date":     date,
-		"s_ip":       req.Proto,
-		"s_refer":    req.Referer(),
-		"i_year":     timeNow.Year(),
-		"i_month":    timeNow.Month(),
-		"i_day":      timeNow.Day(),
-		"i_hour":     timeNow.Hour(),
-		"i_minutes":  timeNow.Minute(),
-		"s_describe": req.Form,
-		"s_client":   agent,
-		"s_os":       GetOS(agent),
-		"s_browse":   GetBrowse(agent),
-		"s_method":   req.Method,
-		"s_url":      s_url,
+		"id":        uc.DayShortTime(),
+		"date":      date,
+		"ip":        req.Proto,
+		"refer":     req.Referer(),
+		"year":      timeNow.Year(),
+		"month":     timeNow.Month(),
+		"day":       timeNow.Day(),
+		"hour":      timeNow.Hour(),
+		"minutes":   timeNow.Minute(),
+		"mdescribe": str,
+		"client":    agent,
+		"os":        GetOS(agent),
+		"browse":    GetBrowse(agent),
+		"method":    req.Method,
+		"url":       s_url,
 	}
 	if hasSession {
-		logs["s_userid"] = session.Get("userId")
-		logs["s_username"] = userName
-		logs["i_usertype"] = session.Get("userType")
-		logs["s_nickName"] = session.Get("nickName")
-		logs["s_loginName"] = session.Get("loginName")
-		logs["s_loginType"] = session.Get("loginType")
+		logs["uid"] = session.Get("userId")
+		logs["uname"] = userName
+		logs["utype"] = session.Get("userType")
+		logs["nick"] = session.Get("nickName")
+		logs["loginn"] = session.Get("loginName")
+		logs["logint"] = session.Get("loginType")
 	}
 	lock.Lock()
 	arr = append(arr, logs)
 	if len(arr) >= nc {
-		tmp := arr
+		/*tmp := arr
 		arr = make([]map[string]interface{}, 0)
 		go func() {
 			mongodb.SaveBulk("logs", tmp...)
+		}()*/
+		tmp := arr
+		arr = make([]map[string]interface{}, 0)
+		go func() {
+			ca.SaveBatch("qfw_logs", tmp)
 		}()
 	}
 	lock.Unlock()