Merge branch 'develop' of 192.168.3.17:zhanghongbo/qfw into develop

common/src/github.com/surfer/README.md

@@ -0,0 +1,55 @@
+# surfer    [![GoDoc](https://godoc.org/github.com/tsuna/gohbase?status.png)](https://godoc.org/github.com/henrylee2cn/surfer) [![GitHub release](https://img.shields.io/github/release/henrylee2cn/surfer.svg)](https://github.com/henrylee2cn/surfer/releases)
+
+A high level concurrency downloader.
+
+</br>
+surfer是一款Go语言编写的高并发爬虫下载器，拥有surf与phantom两种下载内核。
+
+</br>
+支持固定UserAgent自动保存cookie与随机大量UserAgent禁用cookie两种模式，高度模拟浏览器行为，可实现模拟登录等功能。
+
+</br>
+高并发爬虫[Pholcus](https://github.com/henrylee2cn/pholcus)的专用下载器。（官方QQ群：Go大数据 42731170，欢迎加入我们的讨论）
+</br>
+
+
+
+### Usage
+
+```
+package main
+
+import (
+    "github.com/henrylee2cn/surfer"
+    "io/ioutil"
+    "log"
+)
+
+func main() {
+    // 默认使用surf内核下载
+    resp, err := surfer.Download(&surfer.DefaultRequest{
+        Url: "http://github.com/henrylee2cn/surfer",
+    })
+    if err != nil {
+        log.Fatal(err)
+    }
+    b, err := ioutil.ReadAll(resp.Body)
+    log.Println(string(b), err)
+
+    // 指定使用phantomjs内核下载
+    resp, err = surfer.Download(&surfer.DefaultRequest{
+        Url:          "http://github.com/henrylee2cn",
+        DownloaderID: 1,
+    })
+    if err != nil {
+        log.Fatal(err)
+    }
+    b, err = ioutil.ReadAll(resp.Body)
+    log.Println(string(b), err)
+
+    resp.Body.Close()
+    surfer.DestroyJsFiles()
+}
+```
+
+详情参考：[example.go](https://github.com/henrylee2cn/surfer/blob/master/example/example.go)

common/src/github.com/surfer/agent/agent.go

@@ -0,0 +1,320 @@
+// Package agent generates user agents strings for well known browsers
+// and for custom browsers.
+//
+// When submitting patches to add user agents formats, please *always* include
+// "{{.Coms}}" between the opening ( and closing ) braces, even if you're
+// sure the browser would never have additional comments.
+package agent
+
+import (
+	"bytes"
+	"math/rand"
+	"runtime"
+	"strings"
+	"text/template"
+	"time"
+)
+
+const (
+	// Windows operating system.
+	Windows int = iota
+	// Linux based operating system.
+	Linux
+	// Macintosh/OS X operating system.
+	Macintosh
+)
+
+// TemplateData structure for template data.
+type TemplateData struct {
+	Name string
+	Ver  string
+	OSN  string
+	OSV  string
+	Coms string
+}
+
+// OSAttributes stores OS attributes.
+type OSAttributes struct {
+	// OSName is the operating system name.
+	OSName string
+	// OSVersion is the operating system version.
+	OSVersion string
+	// Comments are additional comments to add to a user agent string.
+	Comments []string
+}
+
+// DefaultOSAttributes stores default OS attributes.
+var DefaultOSAttributes = map[int]OSAttributes{
+	Windows:   {"Windows NT", "6.3", []string{"x64"}},
+	Linux:     {"Linux", "3.16.1", []string{"x64"}},
+	Macintosh: {"Intel Mac OS X", "10_6_8", []string{}},
+}
+
+// Formats is a collection of UA format strings.
+// key is the browser version.
+// value is the browser info.
+type Formats map[string]string
+
+// UAData stores information on a browser user agent.
+type UAData struct {
+	TopVersion string
+	DefaultOS  int
+	Formats    Formats
+}
+
+// UATable is a collection of UAData values.
+// key is the name of the browser.
+type UATable map[string]UAData
+
+// Database is the "database" of user agents.
+var Database = UATable{
+	"chrome": {
+		"37.0.2049.0",
+		Windows,
+		Formats{
+			"37": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) Chrome/{{.Ver}} Safari/537.36",
+			"36": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) Chrome/{{.Ver}} Safari/537.36",
+			"35": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) Chrome/{{.Ver}} Safari/537.36",
+			"34": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) Chrome/{{.Ver}} Safari/537.36",
+			"33": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) Chrome/{{.Ver}} Safari/537.36",
+			"32": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) Chrome/{{.Ver}} Safari/537.36",
+			"31": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) Chrome/{{.Ver}} Safari/537.36",
+			"30": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) Chrome/{{.Ver}} Safari/537.36",
+		},
+	},
+	"firefox": {
+		"31.0",
+		Windows,
+		Formats{
+			"31": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:31.0) Gecko/20100101 Firefox/{{.Ver}}",
+			"30": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:30.0) Gecko/20120101 Firefox/{{.Ver}}",
+			"29": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:29.0) Gecko/20120101 Firefox/{{.Ver}}",
+			"28": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:28.0) Gecko/20100101 Firefox/{{.Ver}}",
+			"27": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:27.0) Gecko/20130101 Firefox/{{.Ver}}",
+			"26": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:26.0) Gecko/20121011 Firefox/{{.Ver}}",
+			"25": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:25.0) Gecko/20100101 Firefox/{{.Ver}}",
+		},
+	},
+	"msie": {
+		"10.0",
+		Windows,
+		Formats{
+			"10": "Mozilla/5.0 (compatible; MSIE 10.0; {{.OSN}} {{.OSV}}{{if .Coms}}{{.Coms}}; {{end}}Trident/5.0; .NET CLR 3.5.30729)",
+			"9":  "Mozilla/5.0 (compatible; MSIE 9.0; {{.OSN}} {{.OSV}}{{if .Coms}}{{.Coms}}; {{end}}Trident/5.0; .NET CLR 3.0.30729)",
+			"8":  "Mozilla/5.0 (compatible; MSIE 8.0; {{.OSN}} {{.OSV}}{{if .Coms}}{{.Coms}}; {{end}}Trident/4.0; .NET CLR 3.0.04320)",
+			"7":  "Mozilla/4.0 (compatible; MSIE 7.0; {{.OSN}} {{.OSV}}{{if .Coms}}{{.Coms}}; {{end}}.NET CLR 2.0.50727)",
+		},
+	},
+	"opera": {
+		"12.14",
+		Windows,
+		Formats{
+			"12": "Opera/9.80 ({{.OSN}} {{.OSV}}; U{{.Coms}}) Presto/2.9.181 Version/{{.Ver}}",
+			"11": "Opera/9.80 ({{.OSN}} {{.OSV}}; U{{.Coms}}) Presto/2.7.62 Version/{{.Ver}}",
+			"10": "Opera/9.80 ({{.OSN}} {{.OSV}}; U{{.Coms}}) Presto/2.2.15 Version/{{.Ver}}",
+			"9":  "Opera/9.00 ({{.OSN}} {{.OSV}}; U{{.Coms}})",
+		},
+	},
+	"safari": {
+		"6.0",
+		Macintosh,
+		Formats{
+			"6": "Mozilla/5.0 (Macintosh; {{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/536.26 (KHTML, like Gecko) Version/{{.Ver}} Safari/8536.25",
+			"5": "Mozilla/5.0 (Macintosh; {{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/531.2+ (KHTML, like Gecko) Version/{{.Ver}} Safari/531.2+",
+			"4": "Mozilla/5.0 (Macintosh; {{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/528.16 (KHTML, like Gecko) Version/{{.Ver}} Safari/528.16",
+		},
+	},
+	"itunes": {
+		"9.1.1",
+		Macintosh,
+		Formats{
+			"9": "iTunes/{{.Ver}}",
+			"8": "iTunes/{{.Ver}}",
+			"7": "iTunes/{{.Ver}} (Macintosh; U; PPC Mac OS X 10.4.7{{.Coms}})",
+			"6": "iTunes/{{.Ver}} (Macintosh; U; PPC Mac OS X 10.4.5{{.Coms}})",
+		},
+	},
+	"aol": {
+		"9.7",
+		Windows,
+		Formats{
+			"9": "Mozilla/5.0 (compatible; MSIE 9.0; AOL {{.Ver}}; AOLBuild 4343.19; {{.OSN}} {{.OSV}}; WOW64; Trident/5.0; FunWebProducts{{.Coms}})",
+			"8": "Mozilla/4.0 (compatible; MSIE 7.0; AOL {{.Ver}}; {{.OSN}} {{.OSV}}; GTB5; .NET CLR 1.1.4322; .NET CLR 2.0.50727{{.Coms}})",
+			"7": "Mozilla/4.0 (compatible; MSIE 7.0; AOL {{.Ver}}; {{.OSN}} {{.OSV}}; FunWebProducts{{.Coms}})",
+			"6": "Mozilla/4.0 (compatible; MSIE 6.0; AOL {{.Ver}}; {{.OSN}} {{.OSV}}{{.Coms}})",
+		},
+	},
+	"konqueror": {
+		"4.9",
+		Linux,
+		Formats{
+			"4": "Mozilla/5.0 (compatible; Konqueror/4.0; {{.OSN}}{{.Coms}}) KHTML/4.0.3 (like Gecko)",
+			"3": "Mozilla/5.0 (compatible; Konqueror/3.0-rc6; i686 {{.OSN}}; 20021127{{.Coms}})",
+			"2": "Mozilla/5.0 (compatible; Konqueror/2.1.1; {{.OSN}}{{.Coms}})",
+		},
+	},
+	"netscape": {
+		"9.1.0285",
+		Windows,
+		Formats{
+			"9": "Mozilla/5.0 ({{.OSN}}; U; {{.OSN}} {{.OSV}}; rv:1.9.2.4{{.Coms}}) Gecko/20070321 Netscape/{{.Ver}}",
+			"8": "Mozilla/5.0 ({{.OSN}}; U; {{.OSN}} {{.OSV}}; rv:1.7.5{{.Coms}}) Gecko/20050519 Netscape/{{.Ver}}",
+			"7": "Mozilla/5.0 ({{.OSN}}; U; {{.OSN}} {{.OSV}}; rv:1.0.1{{.Coms}}) Gecko/20020921 Netscape/{{.Ver}}",
+		},
+	},
+	"lynx": {
+		"2.8.8dev.3",
+		Linux,
+		Formats{
+			"2": "Lynx/{{.Ver}} libwww-FM/2.14 SSL-MM/1.4.1",
+			"1": "Lynx (textmode)",
+		},
+	},
+	"googlebot": {
+		"2.1",
+		Linux,
+		Formats{
+			"2": "Mozilla/5.0 (compatible; Googlebot/{{.Ver}}; +http://www.google.com/bot.html{{.Coms}})",
+			"1": "Googlebot/{{.Ver}} (+http://www.google.com/bot.html{{.Coms}})",
+		},
+	},
+	"bingbot": {
+		"2.0",
+		Windows,
+		Formats{
+			"2": "Mozilla/5.0 (compatible; bingbot/{{.Ver}}; +http://www.bing.com/bingbot.htm{{.Coms}})",
+		},
+	},
+	"yahoobot": {
+		"2.0",
+		Linux,
+		Formats{
+			"2": "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp{{.Coms}})",
+		},
+	},
+	"default": {
+		"1.0",
+		Linux,
+		Formats{
+			"1": "{{.Name}}/{{.Ver}} ({{.OSN}} {{.OSV}}{{.Coms}})",
+		},
+	},
+}
+
+// 全部UserAgent
+var UserAgents = map[string][]string{}
+
+func init() {
+	for browser, userAgentData := range Database {
+		if browser == "default" {
+			continue
+		}
+		os := userAgentData.DefaultOS
+		osAttribs := DefaultOSAttributes[os]
+		for version, _ := range userAgentData.Formats {
+			ua := createFromDetails(
+				browser,
+				version,
+				osAttribs.OSName,
+				osAttribs.OSVersion,
+				osAttribs.Comments)
+			UserAgents["all"] = append(UserAgents["all"], ua)
+
+			if browser != "itunes" && browser != "lynx" && browser != "googlebot" && browser != "bingbot" && browser != "yahoobot" {
+				UserAgents["common"] = append(UserAgents["common"], ua)
+			}
+		}
+	}
+	l := len(UserAgents["common"])
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	idx := r.Intn(l)
+	UserAgents["all"][0], UserAgents["all"][idx] = UserAgents["all"][idx], UserAgents["all"][0]
+	UserAgents["common"][0], UserAgents["common"][idx] = UserAgents["common"][idx], UserAgents["common"][0]
+}
+
+// Create generates and returns a complete user agent string.
+func CreateReal() string {
+	return createFromDetails("Surfer", "1.0", osName(), osVersion(), []string{runtime.Version()})
+}
+
+// CreateDefault returns a user agent string using default values.
+func CreateDefault(browser string) string {
+	bn := strings.ToLower(browser)
+	data := Database[bn]
+	os := data.DefaultOS
+	osAttribs := DefaultOSAttributes[os]
+
+	return createFromDetails(
+		browser,
+		data.TopVersion,
+		osAttribs.OSName,
+		osAttribs.OSVersion,
+		osAttribs.Comments)
+}
+
+// CreateVersion generates and returns a complete user agent string for a specific browser version.
+func CreateVersion(browser, version string) string {
+	bn := strings.ToLower(browser)
+	data := Database[bn]
+	os := data.DefaultOS
+	osAttribs := DefaultOSAttributes[os]
+
+	return createFromDetails(
+		browser,
+		version,
+		osAttribs.OSName,
+		osAttribs.OSVersion,
+		osAttribs.Comments)
+}
+
+// TopVersion returns the most recent version for the given browser name.
+func TopVersion(bname string) string {
+	bname = strings.ToLower(bname)
+	data, ok := Database[bname]
+	if ok {
+		return data.TopVersion
+	}
+	return Database["default"].TopVersion
+}
+
+// Format returns the format string for the given browser name and version.
+//
+// When a format can't be found for a version, the first format string for the browser
+// is returned. When a format can't be found for the browser the default format is
+// returned.
+func Format(bname, bver string) string {
+	bname = strings.ToLower(bname)
+	majVer := strings.Split(bver, ".")[0]
+	data, ok := Database[bname]
+	if ok {
+		format, ok := data.Formats[majVer]
+		if ok {
+			return format
+		} else {
+			top := TopVersion(bname)
+			majVer = strings.Split(top, ".")[0]
+			return data.Formats[majVer]
+		}
+	}
+
+	return Database["default"].Formats["1"]
+}
+
+// createFromDetails generates and returns a complete user agent string.
+func createFromDetails(bname, bver, osname, osver string, c []string) string {
+	if bver == "" {
+		bver = TopVersion(bname)
+	}
+	comments := strings.Join(c, "; ")
+	if comments != "" {
+		comments = "; " + comments
+	}
+
+	data := TemplateData{bname, bver, osname, osver, comments}
+	buff := &bytes.Buffer{}
+	t := template.New("formatter")
+	t.Parse(Format(bname, bver))
+	t.Execute(buff, data)
+
+	return buff.String()
+}

common/src/github.com/surfer/agent/agent_bsd.go

@@ -0,0 +1,26 @@
+// +build darwin dragonfly freebsd netbsd openbsd
+
+package agent
+
+import (
+	"runtime"
+	"syscall"
+)
+
+// osName returns the name of the OS.
+func osName() string {
+	name, err := syscall.Sysctl("kern.ostype")
+	if err != nil {
+		return runtime.GOOS
+	}
+	return name
+}
+
+// osVersion returns the OS version.
+func osVersion() string {
+	release, err := syscall.Sysctl("kern.osrelease")
+	if err != nil {
+		return "0.0"
+	}
+	return release
+}

common/src/github.com/surfer/agent/agent_linux.go

@@ -0,0 +1,41 @@
+// +build linux
+
+package agent
+
+import (
+	"runtime"
+	"syscall"
+)
+
+// osName returns the name of the OS.
+func osName() string {
+	buf := &syscall.Utsname{}
+	err := syscall.Uname(buf)
+	if err != nil {
+		return runtime.GOOS
+	}
+	return charsToString(buf.Sysname)
+}
+
+// osVersion returns the OS version.
+func osVersion() string {
+	buf := &syscall.Utsname{}
+	err := syscall.Uname(buf)
+	if err != nil {
+		return "0.0"
+	}
+	return charsToString(buf.Release)
+}
+
+// charsToString converts a [65]int8 byte array into a string.
+func charsToString(ca [65]int8) string {
+	s := make([]byte, len(ca))
+	var lens int
+	for ; lens < len(ca); lens++ {
+		if ca[lens] == 0 {
+			break
+		}
+		s[lens] = uint8(ca[lens])
+	}
+	return string(s[0:lens])
+}

common/src/github.com/surfer/agent/agent_windows.go

@@ -0,0 +1,25 @@
+// +build windows
+
+package agent
+
+import (
+	"fmt"
+	"runtime"
+	"syscall"
+)
+
+// osName returns the name of the OS.
+func osName() string {
+	return runtime.GOOS
+}
+
+// osVersion returns the OS version.
+func osVersion() string {
+	v, err := syscall.GetVersion()
+	if err != nil {
+		return "0.0"
+	}
+	major := uint8(v)
+	minor := uint8(v >> 8)
+	return fmt.Sprintf("%d.%d", major, minor)
+}

common/src/github.com/surfer/example/example.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"github.com/henrylee2cn/surfer"
+	"io/ioutil"
+	"log"
+	"time"
+)
+
+func main() {
+	// 默认使用surf内核下载
+	log.Println("********************************************* surf内核GET下载测试开始 *********************************************")
+	resp, err := surfer.Download(&surfer.DefaultRequest{
+		Url: "http://www.baidu.com/",
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Println(resp.Header)
+
+	b, err := ioutil.ReadAll(resp.Body)
+	log.Println(string(b), err)
+
+	log.Println("********************************************* surf内核GET下载测试完毕 *********************************************")
+
+	// 默认使用surf内核下载
+	log.Println("********************************************* surf内核POST下载测试开始 *********************************************")
+	resp, err = surfer.Download(&surfer.DefaultRequest{
+		Url:      "http://accounts.lewaos.com/",
+		Method:   "POST",
+		PostData: "username=123456@qq.com&password=123456&login_btn=login_btn&submit=login_btn",
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Println(resp.Header)
+
+	b, err = ioutil.ReadAll(resp.Body)
+	log.Println(string(b), err)
+
+	log.Println("********************************************* surf内核POST下载测试完毕 *********************************************")
+
+	log.Println("********************************************* phantomjs内核GET下载测试开始 *********************************************")
+
+	// 指定使用phantomjs内核下载
+	resp, err = surfer.Download(&surfer.DefaultRequest{
+		Url:          "http://www.baidu.com/",
+		DownloaderID: 1,
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Println(resp.Header)
+
+	b, err = ioutil.ReadAll(resp.Body)
+	log.Println(string(b), err)
+
+	log.Println("********************************************* phantomjs内核GET下载测试完毕 *********************************************")
+
+	log.Println("********************************************* phantomjs内核POST下载测试开始 *********************************************")
+
+	// 指定使用phantomjs内核下载
+	resp, err = surfer.Download(&surfer.DefaultRequest{
+		DownloaderID: 1,
+		Url:          "http://accounts.lewaos.com/",
+		Method:       "POST",
+		PostData:     "username=123456@qq.com&password=123456&login_btn=login_btn&submit=login_btn",
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Println(resp.Header)
+
+	b, err = ioutil.ReadAll(resp.Body)
+	log.Println(string(b), err)
+
+	log.Println("********************************************* phantomjs内核POST下载测试完毕 *********************************************")
+
+	resp.Body.Close()
+
+	surfer.DestroyJsFiles()
+
+	time.Sleep(600e9)
+}

common/src/github.com/surfer/param.go

@@ -0,0 +1,141 @@
+package surfer
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/surfer/agent"
+	"github.com/surfer/util"
+)
+
+type Param struct {
+	method        string
+	url           *url.URL
+	proxy         *url.URL
+	contentType   string
+	body          io.Reader
+	header        http.Header
+	enableCookie  bool
+	dialTimeout   time.Duration
+	connTimeout   time.Duration
+	tryTimes      int
+	retryPause    time.Duration
+	redirectTimes int
+	client        *http.Client
+}
+
+func NewParam(req Request) (param *Param, err error) {
+	param = new(Param)
+	param.url, err = util.UrlEncode(req.GetUrl())
+	if err != nil {
+		return nil, err
+	}
+
+	if req.GetProxy() != "" {
+		if param.proxy, err = url.Parse(req.GetProxy()); err != nil {
+			return nil, err
+		}
+	}
+
+	switch method := strings.ToUpper(req.GetMethod()); method {
+	case "GET", "HEAD":
+		param.method = method
+	case "POST":
+		param.method = method
+		param.contentType = "application/x-www-form-urlencoded"
+		param.body = strings.NewReader(req.GetPostData())
+	case "POST-M":
+		param.method = "POST"
+		body := &bytes.Buffer{}
+		writer := multipart.NewWriter(body)
+		values, _ := url.ParseQuery(req.GetPostData())
+		for k, vs := range values {
+			for _, v := range vs {
+				writer.WriteField(k, v)
+			}
+		}
+		err := writer.Close()
+		if err != nil {
+			return nil, err
+		}
+		param.contentType = writer.FormDataContentType()
+		param.body = body
+
+	default:
+		param.method = "GET"
+	}
+
+	param.header = make(http.Header)
+
+	if param.contentType != "" {
+		param.header.Set("Content-Type", param.contentType)
+	}
+
+	for k, v := range req.GetHeader() {
+		for _, vv := range v {
+			param.header.Add(k, vv)
+		}
+	}
+
+	param.enableCookie = req.GetEnableCookie()
+
+	if len(param.header.Get("User-Agent")) == 0 {
+		if param.enableCookie {
+			param.header.Set("User-Agent", agent.UserAgents["common"][0])
+		} else {
+			l := len(agent.UserAgents["common"])
+			r := rand.New(rand.NewSource(time.Now().UnixNano()))
+			param.header.Set("User-Agent", agent.UserAgents["common"][r.Intn(l)])
+		}
+	}
+
+	param.dialTimeout = req.GetDialTimeout()
+	if param.dialTimeout < 0 {
+		param.dialTimeout = 0
+	}
+
+	param.connTimeout = req.GetConnTimeout()
+	param.tryTimes = req.GetTryTimes()
+	param.retryPause = req.GetRetryPause()
+	param.redirectTimes = req.GetRedirectTimes()
+	return
+}
+
+// 回写Request内容
+func (self *Param) writeback(resp *http.Response) *http.Response {
+	if resp == nil {
+		resp = new(http.Response)
+		resp.Request = new(http.Request)
+	} else if resp.Request == nil {
+		resp.Request = new(http.Request)
+	}
+
+	resp.Request.Method = self.method
+	resp.Request.Header = self.header
+	resp.Request.Host = self.url.Host
+
+	return resp
+}
+
+// checkRedirect is used as the value to http.Client.CheckRedirect
+// when redirectTimes equal 0, redirect times is ∞
+// when redirectTimes less than 0, not allow redirects
+func (self *Param) checkRedirect(req *http.Request, via []*http.Request) error {
+	if self.redirectTimes == 0 {
+		return nil
+	}
+	if len(via) >= self.redirectTimes {
+		if self.redirectTimes < 0 {
+			return fmt.Errorf("not allow redirects.")
+		}
+		return fmt.Errorf("stopped after %v redirects.", self.redirectTimes)
+	}
+	return nil
+}

common/src/github.com/surfer/request.go

@@ -0,0 +1,188 @@
+package surfer
+
+import (
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+)
+
+type Request interface {
+	// url
+	GetUrl() string
+	// GET POST POST-M HEAD
+	GetMethod() string
+	// POST values
+	GetPostData() string
+	// http header
+	GetHeader() http.Header
+	// enable http cookies
+	GetEnableCookie() bool
+	// dial tcp: i/o timeout
+	GetDialTimeout() time.Duration
+	// WSARecv tcp: i/o timeout
+	GetConnTimeout() time.Duration
+	// the max times of download
+	GetTryTimes() int
+	// the pause time of retry
+	GetRetryPause() time.Duration
+	// the download ProxyHost
+	GetProxy() string
+	// max redirect times
+	GetRedirectTimes() int
+	// select Surf ro PhomtomJS
+	GetDownloaderID() int
+}
+
+const (
+	SurfID             = 0               // Surf下载器标识符
+	PhomtomJsID        = 1               // PhomtomJs下载器标识符
+	DefaultMethod      = "GET"           // 默认请求方法
+	DefaultDialTimeout = 2 * time.Minute // 默认请求服务器超时
+	DefaultConnTimeout = 2 * time.Minute // 默认下载超时
+	DefaultTryTimes    = 3               // 默认最大下载次数
+	DefaultRetryPause  = 2 * time.Second // 默认重新下载前停顿时长
+)
+
+// 默认实现的Request
+type DefaultRequest struct {
+	// url (必须填写)
+	Url string
+	// GET POST POST-M HEAD (默认为GET)
+	Method string
+	// http header
+	Header http.Header
+	// 是否使用cookies，在Spider的EnableCookie设置
+	EnableCookie bool
+	// POST values
+	PostData string
+	// dial tcp: i/o timeout
+	DialTimeout time.Duration
+	// WSARecv tcp: i/o timeout
+	ConnTimeout time.Duration
+	// the max times of download
+	TryTimes int
+	// how long pause when retry
+	RetryPause time.Duration
+	// max redirect times
+	// when RedirectTimes equal 0, redirect times is ∞
+	// when RedirectTimes less than 0, redirect times is 0
+	RedirectTimes int
+	// the download ProxyHost
+	Proxy string
+
+	// 指定下载器ID
+	// 0为Surf高并发下载器，各种控制功能齐全
+	// 1为PhantomJS下载器，特点破防力强，速度慢，低并发
+	DownloaderID int
+
+	// 保证prepare只调用一次
+	once sync.Once
+}
+
+func (self *DefaultRequest) prepare() {
+	if self.Method == "" {
+		self.Method = DefaultMethod
+	}
+	self.Method = strings.ToUpper(self.Method)
+
+	if self.Header == nil {
+		self.Header = make(http.Header)
+	}
+
+	if self.DialTimeout < 0 {
+		self.DialTimeout = 0
+	} else if self.DialTimeout == 0 {
+		self.DialTimeout = DefaultDialTimeout
+	}
+
+	if self.ConnTimeout < 0 {
+		self.ConnTimeout = 0
+	} else if self.ConnTimeout == 0 {
+		self.ConnTimeout = DefaultConnTimeout
+	}
+
+	if self.TryTimes == 0 {
+		self.TryTimes = DefaultTryTimes
+	}
+
+	if self.RetryPause <= 0 {
+		self.RetryPause = DefaultRetryPause
+	}
+
+	if self.DownloaderID != PhomtomJsID {
+		self.DownloaderID = SurfID
+	}
+}
+
+// url
+func (self *DefaultRequest) GetUrl() string {
+	self.once.Do(self.prepare)
+	return self.Url
+}
+
+// GET POST POST-M HEAD
+func (self *DefaultRequest) GetMethod() string {
+	self.once.Do(self.prepare)
+	return self.Method
+}
+
+// POST values
+func (self *DefaultRequest) GetPostData() string {
+	self.once.Do(self.prepare)
+	return self.PostData
+}
+
+// http header
+func (self *DefaultRequest) GetHeader() http.Header {
+	self.once.Do(self.prepare)
+	return self.Header
+}
+
+// enable http cookies
+func (self *DefaultRequest) GetEnableCookie() bool {
+	self.once.Do(self.prepare)
+	return self.EnableCookie
+}
+
+// dial tcp: i/o timeout
+func (self *DefaultRequest) GetDialTimeout() time.Duration {
+	self.once.Do(self.prepare)
+	return self.DialTimeout
+}
+
+// WSARecv tcp: i/o timeout
+func (self *DefaultRequest) GetConnTimeout() time.Duration {
+	self.once.Do(self.prepare)
+	return self.ConnTimeout
+}
+
+// the max times of download
+func (self *DefaultRequest) GetTryTimes() int {
+	self.once.Do(self.prepare)
+	return self.TryTimes
+}
+
+// the pause time of retry
+func (self *DefaultRequest) GetRetryPause() time.Duration {
+	self.once.Do(self.prepare)
+	return self.RetryPause
+}
+
+// the download ProxyHost
+func (self *DefaultRequest) GetProxy() string {
+	self.once.Do(self.prepare)
+	return self.Proxy
+}
+
+// max redirect times
+func (self *DefaultRequest) GetRedirectTimes() int {
+	self.once.Do(self.prepare)
+	return self.RedirectTimes
+}
+
+// select Surf ro PhomtomJS
+func (self *DefaultRequest) GetDownloaderID() int {
+	self.once.Do(self.prepare)
+	return self.DownloaderID
+}

common/src/github.com/surfer/surf.go

@@ -0,0 +1,117 @@
+package surfer
+
+import (
+	"crypto/tls"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/http/cookiejar"
+	"strings"
+	"time"
+
+	"github.com/surfer/agent"
+)
+
+// Default is the default Download implementation.
+type Surf struct {
+	cookieJar *cookiejar.Jar
+}
+
+func New() Surfer {
+	s := new(Surf)
+	s.cookieJar, _ = cookiejar.New(nil)
+	return s
+}
+
+func (self *Surf) Download(req Request) (resp *http.Response, err error) {
+	param, err := NewParam(req)
+	if err != nil {
+		return nil, err
+	}
+	param.client = self.buildClient(param)
+	resp, err = self.httpRequest(param)
+	resp = param.writeback(resp)
+	// if err != nil {
+	// 	resp.Status = "200 OK"
+	// 	resp.StatusCode = 200
+	// }
+	return
+}
+
+// buildClient creates, configures, and returns a *http.Client type.
+func (self *Surf) buildClient(param *Param) *http.Client {
+	client := &http.Client{
+		CheckRedirect: param.checkRedirect,
+	}
+
+	if param.enableCookie {
+		client.Jar = self.cookieJar
+	}
+
+	transport := &http.Transport{
+		DisableKeepAlives: true,
+		Dial: func(network, addr string) (net.Conn, error) {
+			c, err := net.DialTimeout(network, addr, param.dialTimeout)
+			if err != nil {
+				return nil, err
+			}
+			if param.connTimeout > 0 {
+				c.SetDeadline(time.Now().Add(param.connTimeout))
+			}
+			return c, nil
+		},
+	}
+
+	if param.proxy != nil {
+		transport.Proxy = http.ProxyURL(param.proxy)
+	}
+
+	if strings.ToLower(param.url.Scheme) == "https" {
+		transport.TLSClientConfig = &tls.Config{RootCAs: nil, InsecureSkipVerify: true}
+		transport.DisableCompression = true
+	}
+	client.Transport = transport
+	return client
+}
+
+// send uses the given *http.Request to make an HTTP request.
+func (self *Surf) httpRequest(param *Param) (resp *http.Response, err error) {
+	req, err := http.NewRequest(param.method, param.url.String(), param.body)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header = param.header
+
+	if param.tryTimes <= 0 {
+		for {
+			resp, err = param.client.Do(req)
+			if err != nil {
+				if !param.enableCookie {
+					l := len(agent.UserAgents["common"])
+					r := rand.New(rand.NewSource(time.Now().UnixNano()))
+					req.Header.Set("User-Agent", agent.UserAgents["common"][r.Intn(l)])
+				}
+				time.Sleep(param.retryPause)
+				continue
+			}
+			break
+		}
+	} else {
+		for i := 0; i < param.tryTimes; i++ {
+			resp, err = param.client.Do(req)
+			if err != nil {
+				if !param.enableCookie {
+					l := len(agent.UserAgents["common"])
+					r := rand.New(rand.NewSource(time.Now().UnixNano()))
+					req.Header.Set("User-Agent", agent.UserAgents["common"][r.Intn(l)])
+				}
+				time.Sleep(param.retryPause)
+				continue
+			}
+			break
+		}
+	}
+
+	return resp, err
+}

common/src/github.com/surfer/surfer.go

@@ -0,0 +1,36 @@
+// surfer是一款Go语言编写的高并发爬虫下载器，支持 GET/POST/HEAD 方法及 http/https 协议，同时支持固定UserAgent自动保存cookie与随机大量UserAgent禁用cookie两种模式，高度模拟浏览器行为，可实现模拟登录等功能。
+package surfer
+
+import (
+	"net/http"
+	"os"
+	"sync"
+)
+
+var (
+	surf          Surfer
+	phantom       Surfer
+	once_surf     sync.Once
+	once_phantom  sync.Once
+	tempJsDir     = "./tmp"
+	phantomjsFile = os.Getenv("GOPATH") + `\src\github.com\surfer\phantomjs\phantomjs`
+)
+
+func Download(req Request) (resp *http.Response, err error) {
+	switch req.GetDownloaderID() {
+	case SurfID:
+		once_surf.Do(func() { surf = New() })
+		resp, err = surf.Download(req)
+
+	}
+	return
+}
+
+// Downloader represents an core of HTTP web browser for crawler.
+type Surfer interface {
+	// GET @param url string, header http.Header, cookies []*http.Cookie
+	// HEAD @param url string, header http.Header, cookies []*http.Cookie
+	// POST PostForm @param url, referer string, values url.Values, header http.Header, cookies []*http.Cookie
+	// POST-M PostMultipart @param url, referer string, values url.Values, header http.Header, cookies []*http.Cookie
+	Download(Request) (resp *http.Response, err error)
+}

common/src/github.com/surfer/util/util.go

@@ -0,0 +1,94 @@
+// Package util contains some utility methods used by other packages.
+package util
+
+import (
+	"fmt"
+	"hash/crc32"
+	"log"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// 返回编码后的url.URL指针、及解析错误
+func UrlEncode(urlStr string) (*url.URL, error) {
+	urlObj, err := url.Parse(urlStr)
+	urlObj.RawQuery = urlObj.Query().Encode()
+	return urlObj, err
+}
+
+// 制作特征值
+func MakeHash(s string) string {
+	const IEEE = 0xedb88320
+	var IEEETable = crc32.MakeTable(IEEE)
+	hash := fmt.Sprintf("%x", crc32.Checksum([]byte(s), IEEETable))
+	return hash
+}
+
+// The GetWDPath gets the work directory path.
+func GetWDPath() string {
+	wd := os.Getenv("GOPATH")
+	if wd == "" {
+		panic("GOPATH is not setted in env.")
+	}
+	return wd
+}
+
+// The IsDirExists judges path is directory or not.
+func IsDirExists(path string) bool {
+	fi, err := os.Stat(path)
+
+	if err != nil {
+		return os.IsExist(err)
+	} else {
+		return fi.IsDir()
+	}
+
+	panic("util isDirExists not reached")
+}
+
+// The IsFileExists judges path is file or not.
+func IsFileExists(path string) bool {
+	fi, err := os.Stat(path)
+
+	if err != nil {
+		return os.IsExist(err)
+	} else {
+		return !fi.IsDir()
+	}
+
+	panic("util isFileExists not reached")
+}
+
+// 遍历目录，可指定后缀
+func WalkDir(targpath string, suffixes ...string) (dirlist []string) {
+	if !filepath.IsAbs(targpath) {
+		targpath, _ = filepath.Abs(targpath)
+	}
+	err := filepath.Walk(targpath, func(retpath string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !f.IsDir() {
+			return nil
+		}
+		if len(suffixes) == 0 {
+			dirlist = append(dirlist, retpath)
+			return nil
+		}
+		for _, suffix := range suffixes {
+			if strings.HasSuffix(retpath, suffix) {
+				dirlist = append(dirlist, retpath)
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		log.Printf("utils.WalkDir: %v\n", err)
+		return
+	}
+
+	return
+}

common/src/qfw/util/credit/credit.go

@@ -67,7 +67,7 @@ func init() {
 }
 
 //一次性任务积分
-func InCreditA(userId, code string, credit_a int) (bool, int, int, error) {
+func InCreditA(userId, code string, credit_a int64) (bool, int64, int, error) {
 	result := false
 	if len(userId) < 1 {
 		return result, credit_a, 0, nil
@@ -87,7 +87,7 @@ func InCreditA(userId, code string, credit_a int) (bool, int, int, error) {
 }
 
 //剑鱼一次性任务
-func CheckSword(userId, code string, credit_a int, xb *xweb.Action) bool {
+func CheckSword(userId, code string, credit_a int64, xb *xweb.Action) bool {
 	result := false
 	if !AIsHasDo(code, credit_a) {
 		result, credit_a = UpuserCreditA(code, userId, credit_a)
@@ -116,18 +116,18 @@ func InCreditB(userId, code string, param map[string]interface{}) (bool, int, er
 }
 
 //更新用户一次性积分状态
-func UpuserCreditA(code, userId string, credit_a int) (bool, int) {
+func UpuserCreditA(code, userId string, credit_a int64) (bool, int64) {
 	var ret uint64
 	if tmp, ok := CreditA[code]; ok {
 		ret = 1 << (tmp - 1)
 	}
 	n_credit_a := uint64(credit_a) + ret
 	b := mogo.Update("user", `{"_id":"`+userId+`"}`, &map[string]interface{}{"$set": &map[string]interface{}{"credit_a": int64(n_credit_a)}}, true, false)
-	return b, int(n_credit_a)
+	return b, int64(n_credit_a)
 }
 
 //判断一次性积分是否完成
-func AIsHasDo(code string, num int) bool {
+func AIsHasDo(code string, num int64) bool {
 	b := false
 	var ret uint64
 	if tmp, ok := CreditA[code]; ok {
@@ -140,7 +140,7 @@ func AIsHasDo(code string, num int) bool {
 }
 
 //判断A是否完成
-func AAllIsHasDo(num int) bool {
+func AAllIsHasDo(num int64) bool {
 	b := false
 	var ret uint64
 	for k, v := range CreditA {
@@ -187,7 +187,7 @@ func UpuserCreditSession(userId, code, dtype string, param map[string]interface{
 	score := 0
 	util.Try(func() {
 		if dtype == "A" {
-			credit_a := util.IntAll(xb.GetSession("credit_a"))
+			credit_a := util.Int64All(xb.GetSession("credit_a"))
 			b, credit_a, score, _ = InCreditA(userId, code, credit_a)
 			if b {
 				xb.Session().UpdateByCustomField("id", userId, "credit_a", credit_a)

common/src/qfw/util/mongodb/mongodbutil_test.go

@@ -172,3 +172,14 @@ func Test_range(t *testing.T){
 		}
 }
 
+
+func Test_Credit(t *testing.T) {
+	//InitMongodbPool(1, "10.116.86.154:27080", "qfw")
+	//u:=FindById("user","55a30cdfaf53744fca000b51",nil)
+	//cre:=util.IntAll((*u)["credit_a"])
+	//cre2:=util.Int64All((*u)["credit_a"])
+	 s :=int64(1152921504606848907)
+	log.Println(s,int(s))
+
+}
+

core/src/qfw/manage/auditing.go

@@ -804,7 +804,7 @@ func (s *SystemManage) Updateaudit() error {
 		}
 		//认证送积分
 		if flag == "true" && _status == 1 {
-			b, credit_a, score, _ := credit.InCreditA(s_submitid, credit.A_RZ, util.IntAll((*f)["credit_a"]))
+			b, credit_a, score, _ := credit.InCreditA(s_submitid, credit.A_RZ, util.Int64All((*f)["credit_a"]))
 			if b {
 				s.Session().UpdateByCustomField("id", s_submitid, "i_credit", util.IntAll((*f)["i_credit"])+score)
 				s.Session().UpdateByCustomField("id", s_submitid, "credit_a", credit_a)

core/src/qfw/member/bidmanage.go

@@ -206,7 +206,7 @@ func (d *BidManage) ChangeStatus() error {
 			if status == 3 {
 				param := make(map[string]interface{})
 				param["objid"] = wtb["s_userid"].(string)
-				credit_a := util.IntAll(d.GetSession("credit_a"))
+				credit_a := util.Int64All(d.GetSession("credit_a"))
 				//log.Println("选标", credit.AIsHasDo(credit.A_WCJY, credit_a), userId, param["objid"])
 				if credit.AIsHasDo(credit.A_WCJY, credit_a) {
 					credit.UpuserCreditSession(userId, credit.C_JY, "B", param, d.Action)
@@ -216,8 +216,8 @@ func (d *BidManage) ChangeStatus() error {
 				}
 				//判断对方是否完成一次性任务
 				rr := *FindById("user", param["objid"].(string), nil)
-				if !credit.AIsHasDo(credit.A_WCJY, util.IntAll(rr["credit_a"])) {
-					b, credit_a, score, _ := credit.InCreditA(param["objid"].(string), credit.A_WCJY, util.IntAll(rr["credit_a"]))
+				if !credit.AIsHasDo(credit.A_WCJY, util.Int64All(rr["credit_a"])) {
+					b, credit_a, score, _ := credit.InCreditA(param["objid"].(string), credit.A_WCJY, util.Int64All(rr["credit_a"]))
 					if b {
 						d.Session().UpdateByCustomField("id", param["objid"].(string), "credit_a", credit_a)
 						d.Session().UpdateByCustomField("id", param["objid"].(string), "i_credit", util.IntAll(rr["i_credit"])+score)

core/src/qfw/member/credit/creditdetail.go

@@ -33,7 +33,7 @@ func (c *credit) MyCredit() error {
 	if len(userId) > 0 {
 		user := *mongodb.FindById("user", userId, nil)
 		c.T["user"] = user
-		credit_a := util.IntAll(user["credit_a"])
+		credit_a := util.Int64All(user["credit_a"])
 		b := cd.AAllIsHasDo(credit_a)
 		u := c.Header("User-Agent")
 		if strings.Index(u, "Mobile") > -1 {
@@ -104,7 +104,7 @@ func (c *credit) InCreditAjx() error {
 	result := make(M)
 	result["result"] = "n"
 	if len(userId) > 0 {
-		credit_a := util.IntAll(c.GetSession("credit_a"))
+		credit_a := util.Int64All(c.GetSession("credit_a"))
 		param := c.GetString("param")
 		//分享服务
 		if param == "fx" {

core/src/qfw/member/membermanager.go

@@ -868,7 +868,7 @@ func (m *Member) Bindmail() error {
 				bol := Update("user", M{"_id": ObjectIdHex(m.GetSession("userId").(string))}, M{"$set": updateMap}, false, false)
 				if bol {
 					//绑邮箱送积分
-					if !credit.AIsHasDo(credit.A_BYX, IntAll(m.GetSession("credit_a"))) {
+					if !credit.AIsHasDo(credit.A_BYX, Int64All(m.GetSession("credit_a"))) {
 						credit.UpuserCreditSession(usid, credit.A_BYX, "A", nil, m.Action)
 					}
 					//邮箱绑定认证
@@ -878,7 +878,7 @@ func (m *Member) Bindmail() error {
 						if len(r) > 0 && ac == "" {
 							autoAuth(usid, uname, avatar, entid, proid)
 							//认证送积分
-							credit.InCreditA(usid, credit.A_RZ, IntAll(m.GetSession("credit_a")))
+							credit.InCreditA(usid, credit.A_RZ, Int64All(m.GetSession("credit_a")))
 							//发系统消息
 							msg := &msg.Msg{
 								Msgtype:   1,
@@ -956,7 +956,7 @@ func (m *Member) Bindphone() error {
 				bol := Update("user", M{"_id": ObjectIdHex(m.GetSession("userId").(string))}, M{"$set": updateMap}, false, false)
 				if bol {
 					//绑手机送积分
-					if !credit.AIsHasDo(credit.A_BSJ, IntAll(m.GetSession("credit_a"))) {
+					if !credit.AIsHasDo(credit.A_BSJ, Int64All(m.GetSession("credit_a"))) {
 						credit.UpuserCreditSession(m.GetSession("userId").(string), credit.A_BSJ, "A", nil, m.Action)
 					}
 					UpdateCookieSession(m.Action, m.GetSession("loginType").(string), false, *FindById("user", m.GetSession("userId").(string), nil))
@@ -1121,7 +1121,7 @@ func UpdateSession(action *xweb.Action, r map[string]interface{}) {
 		setSessMap["identType"] = IntAll(r["i_identificationtype"])
 		setSessMap["identWay"] = IntAll(r["i_identificationway"])
 		setSessMap["opLocDistrict"] = r["opLocDistrict"]
-		setSessMap["credit_a"] = IntAll(r["credit_a"])
+		setSessMap["credit_a"] = Int64All(r["credit_a"])
 		setSessMap["i_credit"] = IntAll(r["i_credit"])
 		if r["s_phone"] == nil || r["s_phone"].(string) == "" {
 			setSessMap["phone"] = ""

core/src/qfw/member/ordermanage.go

@@ -76,7 +76,7 @@ func (o *OrderManage) ChangeStatus() error {
 		if flag && status == 3 {
 			param := make(map[string]interface{})
 			param["objid"] = (*r)["s_editorid"]
-			credit_a := util.IntAll(o.GetSession("credit_a"))
+			credit_a := util.Int64All(o.GetSession("credit_a"))
 			userId := util.ObjToString(o.GetSession("userId"))
 			//log.Println("预约", credit.AIsHasDo(credit.A_WCJY, credit_a), userId, param["objid"])
 			if credit.AIsHasDo(credit.A_WCJY, credit_a) {
@@ -87,8 +87,8 @@ func (o *OrderManage) ChangeStatus() error {
 			}
 			//判断对方是否完成一次性任务
 			rr := *FindById("user", param["objid"].(string), nil)
-			if !credit.AIsHasDo(credit.A_WCJY, util.IntAll(rr["credit_a"])) {
-				b, credit_a, score, _ := credit.InCreditA(param["objid"].(string), credit.A_WCJY, util.IntAll(rr["credit_a"]))
+			if !credit.AIsHasDo(credit.A_WCJY, util.Int64All(rr["credit_a"])) {
+				b, credit_a, score, _ := credit.InCreditA(param["objid"].(string), credit.A_WCJY, util.Int64All(rr["credit_a"]))
 				if b {
 					o.Session().UpdateByCustomField("id", param["objid"].(string), "credit_a", credit_a)
 					o.Session().UpdateByCustomField("id", param["objid"].(string), "i_credit", util.IntAll(rr["i_credit"])+score)

core/src/qfw/member/yellowpage.go

@@ -237,7 +237,7 @@ func (yp *Yellowpage) Dosave() error {
 		status = "n"
 		info = "保存信息失败"
 	} else { //首次创建企业名片，送积分
-		if !credit.AIsHasDo(credit.A_CJMP, util.IntAll(yp.GetSession("credit_a"))) {
+		if !credit.AIsHasDo(credit.A_CJMP, util.Int64All(yp.GetSession("credit_a"))) {
 			credit.UpuserCreditSession(util.ObjToString(yp.GetSession("userId")), credit.A_CJMP, "A", nil, yp.Action)
 		}
 	}
@@ -435,7 +435,7 @@ func (yp *Yellowpage) AddService() error {
 		tempFlag = len(_id) > 0
 		//发服务送积分
 		if tempFlag {
-			if credit.AIsHasDo(credit.A_FFW, util.IntAll(yp.GetSession("credit_a"))) {
+			if credit.AIsHasDo(credit.A_FFW, util.Int64All(yp.GetSession("credit_a"))) {
 				credit.UpuserCreditSession(yp.GetSession("userId").(string), credit.B_FFW, "B", nil, yp.Action)
 			} else {
 				credit.UpuserCreditSession(yp.GetSession("userId").(string), credit.A_FFW, "A", nil, yp.Action)

core/src/qfw/mobile/wxmenu.go

@@ -207,7 +207,7 @@ func (m *Mobile) SwordfishPay() error {
 						}
 						//i_credit -= len(isPay) * 1000
 						if i_credit >= 0 {
-							credit_a := util.IntAll(m.GetSession("credit_a"))
+							credit_a := util.Int64All(m.GetSession("credit_a"))
 							b := false
 							if b, _ = credit.OutCreditB(userId.(string), strings.Join(mapCode, ","), util.ObjToString(m.GetSession("s_m_openid")), 0, isPay, m.Action); b {
 								//先扣分，然后更新，然后返回结果
@@ -324,9 +324,11 @@ func (m *Mobile) AjaxReq() error {
 			}
 			//更新数据库
 			msgset["l_modifydate"] = time.Now().Unix()
+
 			if mongodb.Update("user", `{"_id":"`+userId.(string)+`"}`, &map[string]interface{}{"$set": set}, false, false) {
 				flag = "y"
-				credit_a := util.IntAll(m.GetSession("credit_a"))
+				/**
+				credit_a := util.Int64All(m.GetSession("credit_a"))
 				if !credit.AIsHasDo(credit.A_JYSCTS, credit_a) && len(mapPush) > 0 {
 					util.Try(func() {
 						//对用户进行推送
@@ -346,7 +348,9 @@ func (m *Mobile) AjaxReq() error {
 						log.Println("给用户推送出错", e)
 					})
 				}
+				**/
 			}
+
 		}
 		break
 	}

core/src/qfw/search/searchService.go

@@ -164,7 +164,7 @@ func (n *Search) GetEnterpriseList(reqType, param /*参数*/ string) error {
 		if len(queryStr) > 0 {
 			userId := ObjToString(n.GetSession("userId"))
 			if len(userId) > 0 {
-				credit_a := IntAll(n.GetSession("credit_a"))
+				credit_a := Int64All(n.GetSession("credit_a"))
 				if credit.AIsHasDo(credit.A_QYCX, credit_a) {
 					credit.UpuserCreditSession(userId, credit.B_QYCX, "B", nil, n.Action)
 				} else {

core/src/qfw/searchmarket/demand.go

@@ -176,7 +176,7 @@ func (d *Demand) AddDemand() error {
 			b := UpdateNewDoc("demand", "demand", mongo.FindById("demand", rs, ""))
 			//发布需求送积分
 			if b {
-				credit_a := util.IntAll(d.GetSession("credit_a"))
+				credit_a := util.Int64All(d.GetSession("credit_a"))
 				if credit.AIsHasDo(credit.A_FXQ, credit_a) {
 					credit.UpuserCreditSession(util.ObjToString(s_userid), credit.B_FXQ, "B", nil, d.Action)
 				} else {

core/src/qfw/searchmarket/service.go

@@ -281,7 +281,7 @@ func (s *Service) Comment() error {
 						//评价送积分
 						param := make(map[string]interface{})
 						param["serviceid"] = s_serviceid
-						credit_a := util.IntAll(s.GetSession("credit_a"))
+						credit_a := util.Int64All(s.GetSession("credit_a"))
 						if credit.AIsHasDo(credit.A_WCJYPJ, credit_a) {
 							credit.UpuserCreditSession(s_userid, credit.C_PJ, "B", param, s.Action)
 						} else {

core/src/qfw/yellowpage/yellowpagemanager.go

@@ -460,7 +460,7 @@ func (yp *Yellowpage) GetRelation() error {
 		entName := yp.GetString("entName")
 		if regNo != "" && entName != "" {
 			relation, flag = getRelation(regNo, entName)
-			if flag && !credit.AIsHasDo(credit.A_CKGXW, util.IntAll(yp.GetSession("credit_a"))) {
+			if flag && !credit.AIsHasDo(credit.A_CKGXW, util.Int64All(yp.GetSession("credit_a"))) {
 				credit.UpuserCreditSession(yp.GetSession("userId").(string), credit.A_CKGXW, "A", nil, yp.Action)
 			}
 		}

push/src/main.go

@@ -42,7 +42,8 @@ var jobs []push.PushJobFace = []push.PushJobFace{
 var bJobs = true
 
 func runJob() {
-	if bJobs {
+	h:=time.Now().Hour()
+	if bJobs && h>5 &&h <23 {
 		bJobs = false
 		for _, job := range jobs {
 			//单个job内部实现多并发，job间不再支持并发，网络压力太大了

push/src/qfw/push/cache.go

@@ -93,7 +93,6 @@ func InitCache(flag, m_openid string, bview bool) map[string]*[]*MemberInterest
 					Interest:    a_key,
 					Openid:      util.ObjToString(tmp["s_m_openid"]),
 				}
-				log.Println("find-openid:", user.Openid)
 				date := flagModule["l_modifydate"]
 				if date == nil {
 					date = o_msgset["l_modifydate"]

push/src/qfw/push/dopush/dopush.go

@@ -18,7 +18,8 @@ import (
 	"strings"
 	"time"
 )
-var wxsendpool = make(chan bool,50)
+
+var wxsendpool = make(chan bool, 50)
 var se util.SimpleEncrypt
 var re *regexp.Regexp
 
@@ -269,8 +270,7 @@ L1:
 		vv := *v
 		time.Sleep(50 * time.Millisecond)
 		res = true
-		log.Println("send-openid:",kk.Openid)
-		wxsendpool<-true
+		wxsendpool <- true
 		go Send(&kk, &vv, now, TITLEA, ShortTitle, stype, MaxPushSize)
 	}
 	return res
@@ -290,8 +290,8 @@ func Send(k *push.MemberInterest, v *list.List, now time.Time, TITLEA, ShortTitl
 			}
 		}
 	}()
-	
-	defer func(){
+
+	defer func() {
 		<-wxsendpool
 	}()
 	str := fmt.Sprintf("<div>根据您设置的关键词（%s）,给您推送以下信息:</div>", strings.Join(k.Interest, ";"))

push/src/qfw/push/rpcpush/rpcpush.go

@@ -2,13 +2,11 @@ package rpcpush
 
 import (
 	"encoding/json"
-	"fmt"
 	"log"
 	"qfw/push/bid"
 	"qfw/push/dopush"
 	"qfw/util"
 	qrpc "qfw/util/rpc"
-	"time"
 )
 
 type PushInfo struct {
@@ -41,6 +39,8 @@ func (p *PushInfo) ResultView(data *qrpc.PushData, Reply *[]byte) error {
 //RPC调用推送信息
 func (p *PushInfo) PushMsg(data *qrpc.PushData, Reply *int) error {
 	util.Try(func() {
+		*Reply = 1
+		/**
 		for k, v := range data.PushType {
 			//昨天到今天的数据
 			pj := dopush.Pjob{
@@ -57,6 +57,7 @@ func (p *PushInfo) PushMsg(data *qrpc.PushData, Reply *int) error {
 				*Reply = 0
 			}
 		}
+		**/
 	}, func(e interface{}) {})
 	return nil
 }