package main import ( "encoding/json" "fmt" "github.com/donnie4w/go-logger/logger" "io/ioutil" "os" "regexp" util "spider_chromedp/chromedp/mfw" "time" ) var ( timeReg = regexp.MustCompile("[0-9]{4}-[0-9]{2}-[0-9]{2}") client *util.Client Channel chan bool ) func InitChromedpConfig() { //初始浏览器实例 BrowserGroup = make(chan *Browser, BrowserSize) Channel = make(chan bool, BrowserSize) } func main() { //日志相关 logger.SetConsole(false) logger.SetLevel(logger.LEVEL_DEBUG) logger.SetRollingDaily("./logs", "chrome.log") go clearLogs() // InitChromedpConfig() //初始化参数 //InitRedisClient(ChromedpConfig["redisaddr"].(string)) //初始化Redis InitBrowserGroup() //初始化浏览器实例组 InitServer() //初始化消息服务 go sendIdleMsg() ch := make(chan bool) <-ch } func sendIdleMsg() { for { time.Sleep(5 * time.Second) if chlen := len(Channel); chlen < BrowserSize { myid := client.GetMyclient() client.WriteObj(myid, "", util.SENDTO_TYPE_IDLE_SERVER, -1, map[string]interface{}{myid: BrowserSize - chlen}) } } } func clearLogs() { fmt.Println("=======clearLogs========") timeInt := time.Now().AddDate(0, 0, -30).Unix() dirs, err := ioutil.ReadDir("./logs") if err == nil { for _, f := range dirs { fname := f.Name() logTimeStr := timeReg.FindString(fname) if logTimeStr == "" { continue } logTimeInt, _ := time.ParseInLocation("2006-01-02", logTimeStr, time.Local) if logTimeInt.Unix() < timeInt { os.Remove("./logs/" + fname) } } } time.AfterFunc(24*time.Hour, clearLogs) } func InitServer() { cf := &util.ClientConfig{ ClientName: ServerName, EventHandler: processevent, MsgServerAddr: ServerAddr, CanHandleEvents: []int{util.SERVICE_DOWNLOAD}, //OnRequestConnect: func() {}, OnConnectSuccess: func() { fmt.Println("join...") }, ReadBufferSize: 200, WriteBufferSize: 200, } client, _ = util.NewClient(cf) } func processevent(p *util.Packet) { defer Catch() Channel <- true event := int(p.Event) switch event { case util.SERVICE_DOWNLOAD: //监听下载服务 task := &ChromedpTask{} json.Unmarshal(p.GetBusinessData(), &task) var ret []string fmt.Println(*task) if task.TimeOut <= 0 { task.TimeOut = ChromeTaskTimeOut } if task.Flow { ret = DownloadHtmlByChromedpForFlow(task) //chromedp下载页面(列表页、详情页顺序采集) } else { ret = DownloadHtmlByChromedp(task) //chromedp下载页面(列表页、详情页分开采集) } //if len(ret) == 0 { //给默认值 // ret = []string{""} //} client.WriteObj(p.From, p.Msgid, util.EVENT_RECIVE_CALLBACK, util.SENDTO_TYPE_P2P, ret) //写入,返回 <-Channel default: <-Channel } }