chromedp渲染下载服务等其他服务更新

chromedp/browser.go

@@ -19,6 +19,7 @@ import (
 const (
 	BROWSER_DEVICE_TYPE_PC     = 0
 	BROWSER_DEVICE_TYPE_MOBILE = 1
+	BROWSER_RESET_ERROR_LIMIT  = 10
 )
 
 var (
@@ -127,6 +128,7 @@ var (
 type (
 	//Browser 浏览器
 	Browser struct {
+		timeoutAmount                                int
 		showImages                                   bool
 		incognito                                    bool
 		socks5proxy, proxyChangeIpUrl, clearTraceUrl string
@@ -346,31 +348,56 @@ func (b *Browser) RunWithTimeout(fn func(ctx context.Context, exit chan<- bool),
 	go fn(ctx, exit)
 	select {
 	case <-exit:
+		b.timeoutAmount = 0
 		cancelFn()
 		return
 	case <-time.After(time.Duration(timeout) * time.Second):
+		fmt.Print("ooo")
+		b.timeoutAmount += 1
 		cancelFn()
 		return
 	}
 }
 
 // CloseTab 过滤，关闭Tab页
-func (b *Browser) CloseTab(filter func(url, title string) bool) {
-	defer Catch()
-	ts, err := chromedp.Targets(b.incCtx)
+func (b *Browser) CloseTab(ctx context.Context, filter func(title, url string) bool) {
+	if ctx == nil {
+		ctx = b.incCtx
+	}
+	ctx, _ = context.WithTimeout(ctx, 5*time.Second)
+	ts, err := chromedp.Targets(ctx)
 	if err != nil {
 		return
 	}
 	for _, t := range ts {
 		if filter(t.Title, t.URL) {
-			ctx, _ := chromedp.NewContext(b.incCtx, chromedp.WithTargetID(t.TargetID))
-			chromedp.Run(ctx,
+			newCtx, _ := chromedp.NewContext(ctx, chromedp.WithTargetID(t.TargetID))
+			chromedp.Run(newCtx,
 				page.Close(),
 			)
 		}
 	}
 }
 
+func (b *Browser) Revert() *Browser {
+	if b.timeoutAmount > BROWSER_RESET_ERROR_LIMIT {
+		//超时太多了，可能是内部错了，重新创建浏览器实例
+		fmt.Print("[Reset browser]")
+		b.Close()
+		browser := NewBrowser(Headless,
+			SetUserDataDir(UserDataDir),
+			SetSocks5proxy(ProxyAddr),
+			SetProxyChangeIpUrl(ProxyChangeIpUrl),
+			SetDiskCacheDir(DiskCacheDir),
+			SetExtesionDir(ExtesionDir),
+			SetShowImage(ShowImage),
+			//SetIncognito(env.Incognito),
+		)
+		return browser
+	}
+	return b
+}
+
 // Close
 func (b *Browser) Close() {
 	defer Catch()

chromedp/config.json

@@ -1,6 +1,6 @@
 {
   "server": {
-    "addr": "127.0.0.1:807",
+    "addr": "127.0.0.1:801",
     "name": "chromedp下载服务"
   },
   "headless": false,
@@ -10,7 +10,7 @@
   "userdatadir": "",
   "diskcachedir": "",
   "extesiondir": "",
-  "socks5proxy": "",
-  "changeipurl": "",
+  "socks5proxy": "127.0.0.1:18080",
+  "changeipurl": "http://127.0.0.1:18081/changeip",
   "redisaddr": "list=192.168.3.166:1579"
 }

chromedp/server.go

@@ -79,7 +79,7 @@ func DownloadHtmlByChromedp(task *ChromedpTask) (result []string) {
 	//获取一个浏览器实例
 	browser := <-BrowserGroup
 	defer func() {
-		BrowserGroup <- browser
+		BrowserGroup <- browser.Revert()
 	}()
 	//封装浏览器动作
 	acts, _ := task.AssembleTasks(task.Actions, &result, nil, browser)
@@ -91,7 +91,10 @@ func DownloadHtmlByChromedp(task *ChromedpTask) (result []string) {
 		ChangeBrowserDevice(ctx, BROWSER_DEVICE_TYPE_PC) //浏览器类型，pc端
 		err := chromedp.Run(ctx, acts...)
 		if err != nil {
-			fmt.Println("Chromedp Run Error :", err)
+			logger.Info("Chromedp Run Task Error :", err)
+			//if strings.Contains(err.Error(), "ERR_PROXY_CONNECTION_FAILED") { //代理异常,切换IP
+			browser.ChangeIp()
+			//}
 		}
 	}, task.TimeOut)
 	return
@@ -109,7 +112,7 @@ func DownloadHtmlByChromedpForFlow(task *ChromedpTask) (result []string) {
 	//获取一个浏览器实例
 	browser := <-BrowserGroup
 	defer func() {
-		BrowserGroup <- browser
+		BrowserGroup <- browser.Revert()
 	}()
 	result = []string{}             //记录动作集返回的html
 	resultMap := map[int64]string{} //记录列表页nodes信息，用于匹配详情页tab
@@ -125,6 +128,9 @@ func DownloadHtmlByChromedpForFlow(task *ChromedpTask) (result []string) {
 		err := chromedp.Run(ctx, acts...)
 		if err != nil {
 			logger.Info("Chromedp Run Task Error :", err)
+			//if strings.Contains(err.Error(), "ERR_PROXY_CONNECTION_FAILED") { //代理异常
+			browser.ChangeIp()
+			//}
 		}
 	}, task.TimeOut)
 	return

ipcenter/ipconfig.json

@@ -1,5 +1,5 @@
 {
-  "port": ":8091",
+  "port": ":60910",
   "proxyaddr": "http://proxy.spdata.jianyu360.com/proxy/getallip",
   "chansize": 1,
   "ippoollowsize": 10,

main.go

@@ -32,7 +32,7 @@ var Alldownloader map[string]DynamicIPMap = make(map[string]DynamicIPMap)
 var Msclient *util.Client
 
 func main() {
-	InitMsgClient("127.0.0.1:807", "123")
+	InitMsgClient("127.0.0.1:801", "123")
 	go Download6()
 	//go Download5()
 	//go Download1()

tundrawboard/main.go

@@ -19,7 +19,7 @@ var (
 	ipcenteraddr      = flag.String("centeraddr", "127.0.0.1:60910", "IP代理中心地址") //IP代理中心地址
 	addr              = flag.String("addr", ":18080", "跳板socks5监听地址")
 	monitorAddr       = flag.String("maddr", ":18081", "监控地址")
-	connectTimeout    = flag.Int64("cto", 360, "网络链接超时")
+	connectTimeout    = flag.Int64("cto", 90, "网络链接超时")
 	currentTargetAddr = "" //当前环境IP
 	cache             = list.New()
 	diskCacheDir      = flag.String("cachedir", "./cache_dir", "用户缓存目录位置")
@@ -62,8 +62,9 @@ func processClient(client net.Conn) {
 	var targetConn net.Conn
 	var err error
 	for {
-		//默认复用上一次对可用IP
+		//默认复用上一次可用IP
 		targetConn, err = net.DialTimeout("tcp", currentTargetAddr, time.Duration(*connectTimeout)*time.Second)
+		//fmt.Println("check ip:", currentTargetAddr)
 		if err == nil {
 			break
 		}
@@ -147,6 +148,7 @@ func main() {
 	conn, err := net.Listen("tcp", *addr)
 	if err != nil {
 		log.Fatal(err)
+		return
 	}
 	//
 	go monitor()