spider_editor/src/spider/msclient.go

// msclient
package spider

import (
	"math/rand"
	mu "mfw/util"
	"time"
)

type DynamicIPMap struct {
	Code        string
	InvalidTime int64
}

var Msclient *mu.Client
var MsclientFile *mu.Client
var MsclientBid *mu.Client
var MsclientTest *mu.Client
var MsclientChromedp *mu.Client
var MsclientChromedpTest *mu.Client
var Alldownloader map[string]DynamicIPMap = make(map[string]DynamicIPMap)
var AlldownloaderBid map[string]DynamicIPMap = make(map[string]DynamicIPMap)
var AlldownloaderFile map[string]DynamicIPMap = make(map[string]DynamicIPMap)
var AlldownloaderTest map[string]DynamicIPMap = make(map[string]DynamicIPMap)
var AlldownloaderChromedp map[string]DynamicIPMap = make(map[string]DynamicIPMap)
var AlldownloaderChromedpTest map[string]DynamicIPMap = make(map[string]DynamicIPMap)

func processevent(p *mu.Packet) {
	defer mu.Catch()
	var data []byte
	switch p.Event {
	case mu.SERVICE_DOWNLOAD_APPEND_NODE:
		data = p.GetBusinessData()
		//log.Println("获取动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			Alldownloader[code] = DynamicIPMap{
				Code:        code,
				InvalidTime: time.Now().Unix() + 60*10,
			}
		}
	case mu.SERVICE_DOWNLOAD_DELETE_NODE:
		data = p.GetBusinessData()
		//log.Println("删除动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			delete(Alldownloader, code)
		}
	}
}
func processeventbid(p *mu.Packet) {
	defer mu.Catch()
	var data []byte
	switch p.Event {
	case mu.SERVICE_DOWNLOAD_APPEND_NODE:
		data = p.GetBusinessData()
		//log.Println("获取动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			AlldownloaderBid[code] = DynamicIPMap{
				Code:        code,
				InvalidTime: time.Now().Unix() + 60*10,
			}
		}
	case mu.SERVICE_DOWNLOAD_DELETE_NODE:
		data = p.GetBusinessData()
		//log.Println("删除动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			delete(AlldownloaderBid, code)
		}
	}
}
func processeventFile(p *mu.Packet) {
	defer mu.Catch()
	var data []byte
	switch p.Event {
	case mu.SERVICE_DOWNLOAD_APPEND_NODE:
		data = p.GetBusinessData()
		//log.Println("获取动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			AlldownloaderFile[code] = DynamicIPMap{
				Code:        code,
				InvalidTime: time.Now().Unix() + 60*10,
			}
		}
	case mu.SERVICE_DOWNLOAD_DELETE_NODE:
		data = p.GetBusinessData()
		//log.Println("删除动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			delete(AlldownloaderFile, code)
		}
	}
}
func processeventTest(p *mu.Packet) {
	defer mu.Catch()
	var data []byte
	switch p.Event {
	case mu.SERVICE_DOWNLOAD_APPEND_NODE:
		data = p.GetBusinessData()
		//log.Println("获取动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			AlldownloaderTest[code] = DynamicIPMap{
				Code:        code,
				InvalidTime: time.Now().Unix() + 60*10,
			}
		}
	case mu.SERVICE_DOWNLOAD_DELETE_NODE:
		data = p.GetBusinessData()
		//log.Println("删除动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			delete(AlldownloaderTest, code)
		}
	}
}
func processeventChromedp(p *mu.Packet) {
	defer mu.Catch()
	var data []byte
	switch p.Event {
	case mu.SERVICE_DOWNLOAD_APPEND_NODE:
		data = p.GetBusinessData()
		//log.Println("获取动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			AlldownloaderChromedp[code] = DynamicIPMap{
				Code:        code,
				InvalidTime: time.Now().Unix() + 60*10,
			}
		}
	case mu.SERVICE_DOWNLOAD_DELETE_NODE:
		data = p.GetBusinessData()
		//log.Println("删除动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			delete(AlldownloaderChromedp, code)
		}
	}
}
func processeventChromedpTest(p *mu.Packet) {
	defer mu.Catch()
	var data []byte
	switch p.Event {
	case mu.SERVICE_DOWNLOAD_APPEND_NODE:
		data = p.GetBusinessData()
		//log.Println("获取动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			AlldownloaderChromedp[code] = DynamicIPMap{
				Code:        code,
				InvalidTime: time.Now().Unix() + 60*10,
			}
		}
	case mu.SERVICE_DOWNLOAD_DELETE_NODE:
		data = p.GetBusinessData()
		//log.Println("删除动态地址:", len(data), string(data))
		for i := 0; i < len(data)/8; i++ {
			code := string(data[i*8 : (i+1)*8])
			delete(AlldownloaderChromedp, code)
		}
	}
}

func gc4Alldownloader() {
	n := time.Now().Unix()
	for _, v := range Alldownloader {
		if v.InvalidTime < n {
			delete(Alldownloader, v.Code)
		}
	}
	time.AfterFunc(1*time.Minute, gc4Alldownloader)
}
func gc4AlldownloaderBid() {
	n := time.Now().Unix()
	for _, v := range AlldownloaderBid {
		if v.InvalidTime < n {
			delete(AlldownloaderBid, v.Code)
		}
	}
	time.AfterFunc(1*time.Minute, gc4AlldownloaderBid)
}
func gc4AlldownloaderFile() {
	n := time.Now().Unix()
	for _, v := range AlldownloaderFile {
		if v.InvalidTime < n {
			delete(AlldownloaderFile, v.Code)
		}
	}
	time.AfterFunc(1*time.Minute, gc4AlldownloaderFile)
}
func gc4AlldownloaderTest() {
	n := time.Now().Unix()
	for _, v := range AlldownloaderTest {
		if v.InvalidTime < n {
			delete(AlldownloaderTest, v.Code)
		}
	}
	time.AfterFunc(1*time.Minute, gc4AlldownloaderTest)
}
func gc4AlldownloaderChromedp() {
	n := time.Now().Unix()
	for _, v := range AlldownloaderChromedp {
		if v.InvalidTime < n {
			delete(AlldownloaderChromedp, v.Code)
		}
	}
	time.AfterFunc(1*time.Minute, gc4AlldownloaderChromedp)
}
func gc4AlldownloaderChromedpTest() {
	n := time.Now().Unix()
	for _, v := range AlldownloaderChromedp {
		if v.InvalidTime < n {
			delete(AlldownloaderChromedp, v.Code)
		}
	}
	time.AfterFunc(1*time.Minute, gc4AlldownloaderChromedpTest)
}

func GetOneDownloader() string {
	if len(AlldownloaderTest) < 1 {
		return ""
	}
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	pos := r.Intn(len(AlldownloaderTest))
	index := 0
	retcode := ""
	for k, _ := range AlldownloaderTest {
		if index == pos {
			retcode = k
			break
		}
		index++
	}
	return retcode
}

// 初始化，启动消息客户端
func InitMsgClient(serveraddr, serveraddrbid, serveraddrtest, name, namebid, nametest string) {
	Msclient, _ = mu.NewClient(&mu.ClientConfig{ClientName: name,
		MsgServerAddr:   serveraddr,
		EventHandler:    processevent,
		CanHandleEvents: []int{mu.SERVICE_DOWNLOAD_APPEND_NODE, mu.SERVICE_DOWNLOAD_DELETE_NODE},
		ReadBufferSize:  500,
		WriteBufferSize: 500,
	})
	go gc4Alldownloader()

	MsclientBid, _ = mu.NewClient(&mu.ClientConfig{ClientName: namebid,
		MsgServerAddr:   serveraddrbid,
		EventHandler:    processeventbid,
		CanHandleEvents: []int{mu.SERVICE_DOWNLOAD_APPEND_NODE, mu.SERVICE_DOWNLOAD_DELETE_NODE},
		ReadBufferSize:  500,
		WriteBufferSize: 500,
	})
	go gc4AlldownloaderBid()

	MsclientTest, _ = mu.NewClient(&mu.ClientConfig{ClientName: nametest,
		MsgServerAddr:   serveraddrtest,
		EventHandler:    processeventTest,
		CanHandleEvents: []int{mu.SERVICE_DOWNLOAD_APPEND_NODE, mu.SERVICE_DOWNLOAD_DELETE_NODE},
		ReadBufferSize:  500,
		WriteBufferSize: 500,
	})
	go gc4AlldownloaderTest()
}

// 初始chrome启动消息客户端
func InitChromeMsgClient(chromeaddr, chrometestaddr, name, nametest string) {
	InitMsgClientChromedp(chromeaddr, name)
	InitMsgClientChromedpTest(chrometestaddr, nametest)
}

func InitMsgClientFile(serveraddr, name string) {
	MsclientFile, _ = mu.NewClient(&mu.ClientConfig{ClientName: name,
		MsgServerAddr:   serveraddr,
		EventHandler:    processeventFile,
		CanHandleEvents: []int{mu.SERVICE_DOWNLOAD_APPEND_NODE, mu.SERVICE_DOWNLOAD_DELETE_NODE},
		ReadBufferSize:  500,
		WriteBufferSize: 500,
	})
	go gc4AlldownloaderFile()
}

func InitMsgClientChromedp(serveraddr, name string) {
	MsclientChromedp, _ = mu.NewClient(&mu.ClientConfig{ClientName: name,
		MsgServerAddr:   serveraddr,
		EventHandler:    processeventChromedp,
		CanHandleEvents: []int{mu.SERVICE_DOWNLOAD_APPEND_NODE, mu.SERVICE_DOWNLOAD_DELETE_NODE},
		ReadBufferSize:  200,
		WriteBufferSize: 200,
	})
	go gc4AlldownloaderChromedp()
}
func InitMsgClientChromedpTest(serveraddr, name string) {
	MsclientChromedpTest, _ = mu.NewClient(&mu.ClientConfig{ClientName: name,
		MsgServerAddr:   serveraddr,
		EventHandler:    processeventChromedpTest,
		CanHandleEvents: []int{mu.SERVICE_DOWNLOAD_APPEND_NODE, mu.SERVICE_DOWNLOAD_DELETE_NODE},
		ReadBufferSize:  200,
		WriteBufferSize: 200,
	})
	go gc4AlldownloaderChromedpTest()
}