websiteMonitor/util/util.go

package spider_com

import (
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/big"
	"math/rand"
	"net/http"
	"os"
	"regexp"
	"time"

	"github.com/yuin/gopher-lua"
)

var Reg = regexp.MustCompile("[^0-9A-Za-z\u4e00-\u9fa5]+")
var Filter = regexp.MustCompile("<[^>]*?>|[\\s\u3000\u2003\u00a0]")

//time.AfterFunc 加锁

func TimeAfterFunc(td time.Duration, f func(), ch chan bool) {
	ch <- true
	time.Sleep(10 * time.Millisecond)
	<-ch
	time.AfterFunc(td, func() {
		f()
	})
}

//time.AfterFunc 加锁

func TimeSleepFunc(td time.Duration, ch chan bool) {
	ch <- true
	time.Sleep(10 * time.Millisecond)
	<-ch
	time.Sleep(td)
}

func ParseHttpCookie(obj interface{}) (ret []*http.Cookie) {
	if arr, ok := obj.([]interface{}); ok {
		for _, v := range arr {
			item := v.(map[string]interface{})
			maxAge := 0
			if item["MaxAge"] != nil {
				maxAge = int(item["MaxAge"].(float64))
			}
			secure := false
			if item["Secure"] != nil {
				secure = item["Secure"].(bool)
			}
			httpOnly := false
			if item["HttpOnly"] != nil {
				httpOnly = item["HttpOnly"].(bool)
			}
			exp := item["Expires"].(string)
			expt, _ := time.Parse("2006-01-02T15:04:05Z", exp)
			ret = append(ret, &http.Cookie{
				Value:   item["Value"].(string),
				Name:    item["Name"].(string),
				Domain:  item["Domain"].(string),
				Path:    item["Path"].(string),
				Expires: expt,

				MaxAge:   maxAge,
				Secure:   secure,
				HttpOnly: httpOnly,
			})
		}
	}
	return
}

func TableToMap(tab *lua.LTable) map[string]interface{} {
	tmp := make(map[string]interface{})
	tab.ForEach(func(k, v lua.LValue) {
		key := fmt.Sprint(k)
		if val, ok := v.(*lua.LTable); ok {
			tmp[key] = TableToMap(val)
		} else {
			if val, ok := v.(lua.LString); ok {
				tmp[key] = string(val)
			} else if val, ok := v.(lua.LNumber); ok {
				tmp[key] = int64(val)
			} else if val, ok := v.(lua.LBool); ok {
				tmp[key] = bool(val)
			} else {
				tmp[key] = fmt.Sprint(v)
			}
		}
	})
	return tmp
}

func MapToTable(l *lua.LState, obj []interface{}) *lua.LTable {
	listtb := l.NewTable()
	for i := 0; i < len(obj); i++ {
		tb := l.NewTable()
		if tmp, ok := obj[i].(map[string]string); ok {
			for k, v := range tmp {
				tb.RawSet(lua.LString(k), lua.LString(v))
			}
			listtb.Insert((i + 1), tb)
		} else if tmp, ok := obj[i].(string); ok {
			listtb.Insert(i, lua.LString(tmp))
		}
	}
	return listtb
}

func MapToLuaTable(l *lua.LState, obj map[string]interface{}) *lua.LTable {
	tab := l.NewTable()
	for k, v := range obj {
		if val, ok := v.(string); ok {
			tab.RawSet(lua.LString(k), lua.LString(val))
		} else if val, ok := v.(int64); ok {
			tab.RawSet(lua.LString(k), lua.LNumber(val))
		} else if val, ok := v.(int32); ok {
			tab.RawSet(lua.LString(k), lua.LNumber(val))
		} else if val, ok := v.(float64); ok {
			tab.RawSet(lua.LString(k), lua.LNumber(val))
		} else if val, ok := v.(float32); ok {
			tab.RawSet(lua.LString(k), lua.LNumber(val))
		} else if val, ok := v.(bool); ok {
			tab.RawSet(lua.LString(k), lua.LBool(val))
		} else if val, ok := v.(map[string]interface{}); ok {
			tab.RawSet(lua.LString(k), MapToLuaTable(l, val))
		} else if val, ok := v.([]interface{}); ok {
			bs, _ := json.Marshal(val)
			tab.RawSet(lua.LString(k), lua.LString(string(bs)))
		}
	}
	return tab
}
func GetTable(param *lua.LTable) map[string]interface{} {
	tmp := map[string]interface{}{}
	param.ForEach(func(key, val lua.LValue) {
		k := fmt.Sprint(key)
		if v, ok := val.(lua.LString); ok {
			tmp[k] = string(v)
		} else if v, ok := val.(*lua.LTable); ok {
			tmp[k] = GetTable(v)
		} else {
			tmp[k] = val
		}
	})
	return tmp
}
func GetTableEx(param *lua.LTable) map[string]interface{} {
	rep := make(map[string]interface{})
	param.ForEach(func(key, val lua.LValue) {
		rep[key.String()] = val
	})
	return rep
}

func ParseDate2Int64(str string) int64 {
	t, err := time.ParseInLocation("2006-01-02 15:04:05", str, time.Local)
	if err != nil {
		return time.Now().Unix()
	} else {
		return t.Unix()
	}
}

func EncodeB64(message string) (retour string) {
	base64Text := make([]byte, base64.StdEncoding.EncodedLen(len(message)))
	base64.StdEncoding.Encode(base64Text, []byte(message))
	return string(base64Text)
}

func DecodeB64(message string) (retour string) {
	base64Text := make([]byte, base64.StdEncoding.DecodedLen(len(message)))
	base64.StdEncoding.Decode(base64Text, []byte(message))
	return string(base64Text)
}

// 获取随机数
func GetRandMath(num int) int {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	return r.Intn(num)
}

// 对href哈希取模
func HexToBigIntMod(href string) int {
	//取哈希值
	t := sha256.New()
	io.WriteString(t, href)
	hex := fmt.Sprintf("%x", t.Sum(nil))
	//取模
	n := new(big.Int)
	n, _ = n.SetString(hex[2:], 16)
	return int(n.Mod(n, big.NewInt(16)).Int64())
}

// 求hash
func HexText(href string) string {
	h := sha256.New()
	h.Write([]byte(href))
	return fmt.Sprintf("%x", h.Sum(nil))
}

func HexTextByte(text []byte) string {
	h := sha256.New()
	h.Write(text)
	return fmt.Sprintf("%x", h.Sum(nil))
}

func Sha(con string) string {
	h := sha256.New()
	con = Reg.ReplaceAllString(Filter.ReplaceAllString(con, ""), "")
	h.Write([]byte(con))
	return fmt.Sprintf("%x", h.Sum(nil))
}

func FilterDetail(con string) string {
	return Reg.ReplaceAllString(Filter.ReplaceAllString(con, ""), "")
}

// 判断当前时间是否是工作时间，工作时间周一至周五早7点至晚7点
func IsWorkTime() bool {
	tt := time.Now()
	//"Monday,Tuesday,Wednesday,Thursday,Friday,Saturday,Sunday"
	if tt.Weekday().String() == "Saturday" || tt.Weekday().String() == "Sunday" {
		return false
	} else {
		if tt.Hour() > 7 && tt.Hour() < 19 {
			return true
		} else {
			return false
		}
	}
}

// 生成文件
func CreateFile(code, script string) (string, error) {
	filepath := "res/" + time.Now().Format("2006/01/02")
	err := os.MkdirAll(filepath, 0777)
	filepath = filepath + "/spider_" + code + ".lua"
	f, err := os.Create(filepath)
	defer f.Close()
	if err == nil {
		f.WriteString(script)
		return filepath, nil
	} else {
		return "", err
	}
}

func GetProxyAddr(proxyAddr, roxyAuthor string) string {
	//获取代理
	req, err := http.NewRequest(http.MethodGet, proxyAddr, nil)
	if err != nil {
		fmt.Println("get proxy request err:", err)
		return ""
	}
	//添加请求头
	req.Header.Add("Authorization", roxyAuthor)
	client := http.Client{}
	//发送请求
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("get proxy client err:", err)
		return ""
	}
	defer resp.Body.Close()
	bodyByte, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("get proxy read body err:", err)
		return ""
	}
	tmp := map[string]interface{}{}
	if json.Unmarshal(bodyByte, &tmp) != nil {
		return ""
	}
	if data, ok := tmp["data"].(map[string]interface{}); ok && len(data) > 0 {
		if httpProxy, ok := data["http"].(string); ok {
			return httpProxy
		} else if httpsProxy, ok := data["https"].(string); ok {
			return httpsProxy
		}
	}
	return ""
}