package main import ( "crypto/md5" "encoding/hex" "encoding/json" "fmt" "io/ioutil" "log" "net" "net/http" "net/url" "qfw/util" "qfw/util/mysql" "strconv" "strings" "time" ) var ( config map[string]string appid, key, apiurl, day string httpClient *http.Client MaxIdleCons int = 100 MaxIdleConsPerHost int = 100 IdleConnTimeout int = 2048 ConnectTimeOut int = 30 KeepAlive int = 30 Tidb *mysql.Mysql ) func init() { httpClient = createHttpClient() Tidb = &mysql.Mysql{ Address: "192.168.3.217:4000", UserName: "root", PassWord: "=PDT49#80Z!RVv52_z", DBName: "createxzh", MaxOpenConns: 20, MaxIdleConns: 40, } Tidb.Init() } func main() { util.ReadConfig(&config) appid = config["appid"] key = config["key"] apiurl = config["apiurl"] day = config["day"] getData() } func createHttpClient() *http.Client { client := &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: time.Duration(ConnectTimeOut) * time.Second, //TCP连接超时30s KeepAlive: time.Duration(KeepAlive) * time.Second, //TCP keepalive保活检测定时30s }).DialContext, MaxIdleConns: MaxIdleCons, MaxIdleConnsPerHost: MaxIdleConsPerHost, IdleConnTimeout: time.Duration(IdleConnTimeout) * time.Second, //闲置连接超时2048s ResponseHeaderTimeout: time.Second * 60, }, } return client } func getToken() (token string) { tm := fmt.Sprintf("%d", time.Now().Unix()) res := post(apiurl+"/user/access_token", map[string]string{ "appid": appid, "timestamp": tm, "signature": MD5(appid + tm + key), //"key": "6PzV0CUa", }, nil) log.Println(tm, MD5(appid+tm+key), res) if res != nil && res["access_token"] != "" { token, _ = res["access_token"].(string) } return } func getData() { // token := getToken() log.Println("开始任务") next := "0" for { tm := fmt.Sprint(time.Now().Unix()) data := post(apiurl+"/data/getalldata", map[string]string{ // "access_token": token, "day": day, "next": next, "appid": appid, }, map[string]string{ "timestamp": tm, "signature": MD5(appid + tm + key), }) if data != nil { log.Println(data["next"]) datas := util.ObjArrToMapArr(data["data"].([]interface{})) for _, v := range datas { filehref, _ := json.Marshal(v["filehref"]) v["filehref"] = string(filehref) purchasinglist, _ := json.Marshal(v["purchasinglist"]) v["purchasinglist"] = string(purchasinglist) id := util.ObjToString(v["id"]) if Tidb.Count("info", map[string]interface{}{"id": id}) == 0 { did := Tidb.Insert("info", v) if did > 0 { log.Println("保存成功 ", id) } } else { log.Println("库中已有 ", id) } } nexts := util.IntAll(data["next"]) if nexts == 0 { break } else { tm = fmt.Sprint(time.Now().Unix()) next = strconv.Itoa(nexts) log.Println("第", next, "条") } } else { break } } } func MD5(str string) string { h := md5.New() h.Write([]byte(str)) return strings.ToUpper(hex.EncodeToString(h.Sum(nil))) } func post(urls string, form, header map[string]string) (data map[string]interface{}) { formValues := make(url.Values) for k, v := range form { formValues[k] = []string{v} } log.Println(formValues) header["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8" request, err := http.NewRequest("POST", urls, strings.NewReader(formValues.Encode())) if err != nil { return } for k, v := range header { request.Header.Add(k, v) } response, err := httpClient.Do(request) //前面预处理一些参数,状态,Do执行发送;处理返回结果;Do:发送请求, if err != nil { return } defer response.Body.Close() replay, err := ioutil.ReadAll(response.Body) if err != nil { log.Println("read reply error:", err) return } // 创建一个空的map来存放解析后的数据 // 使用json.Unmarshal来解析[]byte到map json.Unmarshal(replay, &data) return }