// util package spiderutil import ( "crypto/md5" "encoding/base64" "encoding/hex" "encoding/json" "fmt" "math/rand" "net/http" "os" "qfw/util" "time" "github.com/yuin/gopher-lua" ) //time.AfterFunc 加锁 func TimeAfterFunc(td time.Duration, f func(), ch chan bool) { ch <- true time.Sleep(10 * time.Millisecond) <-ch time.AfterFunc(td, func() { f() }) } //time.AfterFunc 加锁 func TimeSleepFunc(td time.Duration, ch chan bool) { ch <- true time.Sleep(10 * time.Millisecond) <-ch time.Sleep(td) } func ParseHttpCookie(obj interface{}) (ret []*http.Cookie) { if arr, ok := obj.([]interface{}); ok { for _, v := range arr { item := v.(map[string]interface{}) maxAge := 0 if item["MaxAge"] != nil { maxAge = int(item["MaxAge"].(float64)) } secure := false if item["Secure"] != nil { secure = item["Secure"].(bool) } httpOnly := false if item["HttpOnly"] != nil { httpOnly = item["HttpOnly"].(bool) } exp := item["Expires"].(string) expt, _ := time.Parse("2006-01-02T15:04:05Z", exp) ret = append(ret, &http.Cookie{ Value: item["Value"].(string), Name: item["Name"].(string), Domain: item["Domain"].(string), Path: item["Path"].(string), Expires: expt, MaxAge: maxAge, Secure: secure, HttpOnly: httpOnly, }) } } return } func TableToMap(tab *lua.LTable) map[string]interface{} { tmp := make(map[string]interface{}) tab.ForEach(func(k, v lua.LValue) { key := fmt.Sprint(k) if val, ok := v.(*lua.LTable); ok { tmp[key] = TableToMap(val) } else { if val, ok := v.(lua.LString); ok { tmp[key] = string(val) } else if val, ok := v.(lua.LNumber); ok { tmp[key] = int64(val) } else if val, ok := v.(lua.LBool); ok { tmp[key] = bool(val) } else { tmp[key] = fmt.Sprint(v) } } }) return tmp } func MapToTable(l *lua.LState, obj []interface{}) *lua.LTable { listtb := l.NewTable() for i := 0; i < len(obj); i++ { tb := l.NewTable() if tmp, ok := obj[i].(map[string]string); ok { for k, v := range tmp { tb.RawSet(lua.LString(k), lua.LString(v)) } listtb.Insert((i + 1), tb) } } return listtb } func MapToLuaTable(l *lua.LState, obj map[string]interface{}) *lua.LTable { tab := l.NewTable() for k, v := range obj { if val, ok := v.(string); ok { tab.RawSet(lua.LString(k), lua.LString(val)) } else if val, ok := v.(int64); ok { tab.RawSet(lua.LString(k), lua.LNumber(val)) } else if val, ok := v.(int32); ok { tab.RawSet(lua.LString(k), lua.LNumber(val)) } else if val, ok := v.(float64); ok { tab.RawSet(lua.LString(k), lua.LNumber(val)) } else if val, ok := v.(float32); ok { tab.RawSet(lua.LString(k), lua.LNumber(val)) } else if val, ok := v.(bool); ok { tab.RawSet(lua.LString(k), lua.LBool(val)) } else if val, ok := v.(map[string]interface{}); ok { tab.RawSet(lua.LString(k), MapToLuaTable(l, val)) } else if val, ok := v.([]interface{}); ok { bs, _ := json.Marshal(val) tab.RawSet(lua.LString(k), lua.LString(string(bs))) } } return tab } func GetTable(param *lua.LTable) map[string]interface{} { tmp := map[string]interface{}{} param.ForEach(func(key, val lua.LValue) { k := fmt.Sprint(key) if v, ok := val.(lua.LString); ok { tmp[k] = string(v) } else if v, ok := val.(*lua.LTable); ok { tmp[k] = GetTable(v) } else { tmp[k] = val } }) return tmp } func GetTableEx(param *lua.LTable) map[string]interface{} { rep := make(map[string]interface{}) param.ForEach(func(key, val lua.LValue) { rep[key.String()] = val }) return rep } func ParseDate2Int64(str string) int64 { t, err := time.ParseInLocation("2006-01-02 15:04:05", str, time.Local) if err != nil { return time.Now().Unix() } else { return t.Unix() } } func EncodeB64(message string) (retour string) { base64Text := make([]byte, base64.StdEncoding.EncodedLen(len(message))) base64.StdEncoding.Encode(base64Text, []byte(message)) return string(base64Text) } func DecodeB64(message string) (retour string) { base64Text := make([]byte, base64.StdEncoding.DecodedLen(len(message))) base64.StdEncoding.Decode(base64Text, []byte(message)) return string(base64Text) } //获取随机数 func GetRandMath(num int) int { r := rand.New(rand.NewSource(time.Now().UnixNano())) return r.Intn(num) } //判断当前时间是否是工作时间,工作时间周一至周五早7点至晚7点 func IsWorkTime() bool { tt := time.Now() //"Monday,Tuesday,Wednesday,Thursday,Friday,Saturday,Sunday" if tt.Weekday().String() == "Saturday" || tt.Weekday().String() == "Sunday" { return false } else { if tt.Hour() > 7 && tt.Hour() < 19 { return true } else { return false } } } //生成文件 func CreateFile(code, script string) (string, error) { filepath := "res/" + time.Now().Format("2006/01/02") err := os.MkdirAll(filepath, 0777) filepath = filepath + "/spider_" + code + ".lua" f, err := os.Create(filepath) defer f.Close() if err == nil { f.WriteString(script) return filepath, nil } else { return "", err } } func GetMd5String(s string) string { h := md5.New() h.Write([]byte(s)) return hex.EncodeToString(h.Sum(nil)) } var Se = util.SimpleEncrypt{Key: "topnet#2017@editor"}