package spider_com import ( "crypto/sha256" "encoding/base64" "encoding/json" "fmt" "io" "io/ioutil" "math/big" "math/rand" "net/http" "os" "regexp" "time" "github.com/yuin/gopher-lua" ) var Reg = regexp.MustCompile("[^0-9A-Za-z\u4e00-\u9fa5]+") var Filter = regexp.MustCompile("<[^>]*?>|[\\s\u3000\u2003\u00a0]") //time.AfterFunc 加锁 func TimeAfterFunc(td time.Duration, f func(), ch chan bool) { ch <- true time.Sleep(10 * time.Millisecond) <-ch time.AfterFunc(td, func() { f() }) } //time.AfterFunc 加锁 func TimeSleepFunc(td time.Duration, ch chan bool) { ch <- true time.Sleep(10 * time.Millisecond) <-ch time.Sleep(td) } func ParseHttpCookie(obj interface{}) (ret []*http.Cookie) { if arr, ok := obj.([]interface{}); ok { for _, v := range arr { item := v.(map[string]interface{}) maxAge := 0 if item["MaxAge"] != nil { maxAge = int(item["MaxAge"].(float64)) } secure := false if item["Secure"] != nil { secure = item["Secure"].(bool) } httpOnly := false if item["HttpOnly"] != nil { httpOnly = item["HttpOnly"].(bool) } exp := item["Expires"].(string) expt, _ := time.Parse("2006-01-02T15:04:05Z", exp) ret = append(ret, &http.Cookie{ Value: item["Value"].(string), Name: item["Name"].(string), Domain: item["Domain"].(string), Path: item["Path"].(string), Expires: expt, MaxAge: maxAge, Secure: secure, HttpOnly: httpOnly, }) } } return } func TableToMap(tab *lua.LTable) map[string]interface{} { tmp := make(map[string]interface{}) tab.ForEach(func(k, v lua.LValue) { key := fmt.Sprint(k) if val, ok := v.(*lua.LTable); ok { tmp[key] = TableToMap(val) } else { if val, ok := v.(lua.LString); ok { tmp[key] = string(val) } else if val, ok := v.(lua.LNumber); ok { tmp[key] = int64(val) } else if val, ok := v.(lua.LBool); ok { tmp[key] = bool(val) } else { tmp[key] = fmt.Sprint(v) } } }) return tmp } func MapToTable(l *lua.LState, obj []interface{}) *lua.LTable { listtb := l.NewTable() for i := 0; i < len(obj); i++ { tb := l.NewTable() if tmp, ok := obj[i].(map[string]string); ok { for k, v := range tmp { tb.RawSet(lua.LString(k), lua.LString(v)) } listtb.Insert((i + 1), tb) } else if tmp, ok := obj[i].(string); ok { listtb.Insert(i, lua.LString(tmp)) } } return listtb } func MapToLuaTable(l *lua.LState, obj map[string]interface{}) *lua.LTable { tab := l.NewTable() for k, v := range obj { if val, ok := v.(string); ok { tab.RawSet(lua.LString(k), lua.LString(val)) } else if val, ok := v.(int64); ok { tab.RawSet(lua.LString(k), lua.LNumber(val)) } else if val, ok := v.(int32); ok { tab.RawSet(lua.LString(k), lua.LNumber(val)) } else if val, ok := v.(float64); ok { tab.RawSet(lua.LString(k), lua.LNumber(val)) } else if val, ok := v.(float32); ok { tab.RawSet(lua.LString(k), lua.LNumber(val)) } else if val, ok := v.(bool); ok { tab.RawSet(lua.LString(k), lua.LBool(val)) } else if val, ok := v.(map[string]interface{}); ok { tab.RawSet(lua.LString(k), MapToLuaTable(l, val)) } else if val, ok := v.([]interface{}); ok { bs, _ := json.Marshal(val) tab.RawSet(lua.LString(k), lua.LString(string(bs))) } } return tab } func GetTable(param *lua.LTable) map[string]interface{} { tmp := map[string]interface{}{} param.ForEach(func(key, val lua.LValue) { k := fmt.Sprint(key) if v, ok := val.(lua.LString); ok { tmp[k] = string(v) } else if v, ok := val.(*lua.LTable); ok { tmp[k] = GetTable(v) } else { tmp[k] = val } }) return tmp } func GetTableEx(param *lua.LTable) map[string]interface{} { rep := make(map[string]interface{}) param.ForEach(func(key, val lua.LValue) { rep[key.String()] = val }) return rep } func ParseDate2Int64(str string) int64 { t, err := time.ParseInLocation("2006-01-02 15:04:05", str, time.Local) if err != nil { return time.Now().Unix() } else { return t.Unix() } } func EncodeB64(message string) (retour string) { base64Text := make([]byte, base64.StdEncoding.EncodedLen(len(message))) base64.StdEncoding.Encode(base64Text, []byte(message)) return string(base64Text) } func DecodeB64(message string) (retour string) { base64Text := make([]byte, base64.StdEncoding.DecodedLen(len(message))) base64.StdEncoding.Decode(base64Text, []byte(message)) return string(base64Text) } // 获取随机数 func GetRandMath(num int) int { r := rand.New(rand.NewSource(time.Now().UnixNano())) return r.Intn(num) } // 对href哈希取模 func HexToBigIntMod(href string) int { //取哈希值 t := sha256.New() io.WriteString(t, href) hex := fmt.Sprintf("%x", t.Sum(nil)) //取模 n := new(big.Int) n, _ = n.SetString(hex[2:], 16) return int(n.Mod(n, big.NewInt(16)).Int64()) } // 求hash func HexText(href string) string { h := sha256.New() h.Write([]byte(href)) return fmt.Sprintf("%x", h.Sum(nil)) } func HexTextByte(text []byte) string { h := sha256.New() h.Write(text) return fmt.Sprintf("%x", h.Sum(nil)) } func Sha(con string) string { h := sha256.New() con = Reg.ReplaceAllString(Filter.ReplaceAllString(con, ""), "") h.Write([]byte(con)) return fmt.Sprintf("%x", h.Sum(nil)) } func FilterDetail(con string) string { return Reg.ReplaceAllString(Filter.ReplaceAllString(con, ""), "") } // 判断当前时间是否是工作时间,工作时间周一至周五早7点至晚7点 func IsWorkTime() bool { tt := time.Now() //"Monday,Tuesday,Wednesday,Thursday,Friday,Saturday,Sunday" if tt.Weekday().String() == "Saturday" || tt.Weekday().String() == "Sunday" { return false } else { if tt.Hour() > 7 && tt.Hour() < 19 { return true } else { return false } } } // 生成文件 func CreateFile(code, script string) (string, error) { filepath := "res/" + time.Now().Format("2006/01/02") err := os.MkdirAll(filepath, 0777) filepath = filepath + "/spider_" + code + ".lua" f, err := os.Create(filepath) defer f.Close() if err == nil { f.WriteString(script) return filepath, nil } else { return "", err } } func GetProxyAddr(proxyAddr, roxyAuthor string) string { //获取代理 req, err := http.NewRequest(http.MethodGet, proxyAddr, nil) if err != nil { fmt.Println("get proxy request err:", err) return "" } //添加请求头 req.Header.Add("Authorization", roxyAuthor) client := http.Client{} //发送请求 resp, err := client.Do(req) if err != nil { fmt.Println("get proxy client err:", err) return "" } defer resp.Body.Close() bodyByte, err := ioutil.ReadAll(resp.Body) if err != nil { fmt.Println("get proxy read body err:", err) return "" } tmp := map[string]interface{}{} if json.Unmarshal(bodyByte, &tmp) != nil { return "" } if data, ok := tmp["data"].(map[string]interface{}); ok && len(data) > 0 { if httpProxy, ok := data["http"].(string); ok { return httpProxy } else if httpsProxy, ok := data["https"].(string); ok { return httpsProxy } } return "" }