package browser import ( "context" "log" "strings" "time" "KeyWebsiteMonitor/spider/util" "github.com/chromedp/cdproto/cdp" "github.com/chromedp/cdproto/fetch" "github.com/chromedp/cdproto/input" "github.com/chromedp/cdproto/network" "github.com/chromedp/cdproto/page" "github.com/chromedp/chromedp" ) // CloseTabs关闭页面 func (b *Browser) CloseTabs(tabTitle, tabUrl string, timeoutInt64 int64) (err error) { if timeoutInt64 == 0 { timeoutInt64 = 5 } timeout := time.Duration(timeoutInt64) * time.Millisecond ts, err := chromedp.Targets(b.Ctx) if err != nil { return err } for _, t := range ts { bl := false if tabUrl != "" && t.URL != "" { tu := strings.ReplaceAll(strings.ReplaceAll(tabUrl, "https://", ""), "http://", "") tURL := strings.ReplaceAll(strings.ReplaceAll(t.URL, "https://", ""), "http://", "") if strings.Contains(tURL, tu) { bl = true } } log.Println(tabTitle != "", strings.Contains(t.Title, tabTitle), tabUrl != "", strings.Contains(t.URL, tabUrl), bl) if (tabTitle != "" && strings.Contains(t.Title, tabTitle)) || (tabUrl != "" && strings.Contains(t.URL, tabUrl) || bl) { newCtx, _ := chromedp.NewContext(b.Ctx, chromedp.WithTargetID(t.TargetID)) ctx, _ := context.WithTimeout(newCtx, timeout) chromedp.Run( ctx, page.Close(), ) } else { log.Printf("tabTile:%s ,taburl:%s ,无法关闭。", tabTitle, tabUrl) } } return nil } // CloseTabs关闭页面 func (b *Browser) CloseTabsWithout(tabTitle, tabUrl string, timeoutInt64 int64) (err error) { if timeoutInt64 == 0 { timeoutInt64 = 5 } timeout := time.Duration(timeoutInt64) * time.Millisecond ts, err := chromedp.Targets(b.Ctx) if err != nil { return err } for _, t := range ts { if (tabTitle != "" && !strings.Contains(t.Title, tabTitle)) || (tabUrl != "" && !strings.Contains(t.URL, tabUrl)) { newCtx, _ := chromedp.NewContext(b.Ctx, chromedp.WithTargetID(t.TargetID)) ctx, _ := context.WithTimeout(newCtx, timeout) chromedp.Run( ctx, page.Close(), ) } } return nil } // Navigate 导航到指定网址 func (b *Browser) Navigate(tabTitle string, tabUrl string, isNewTab bool, targetUrl string, timeout int64) (err error) { ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout) if err != nil { return err } //defer fn() //新标签页 if isNewTab { ctx, _ = chromedp.NewContext(ctx) } // er := chromedp.Run(ctx, //这里作反爬检测 chromedp.ActionFunc(func(cxt context.Context) error { _, err := page.AddScriptToEvaluateOnNewDocument("Object.defineProperty(navigator, 'webdriver', { get: () => false, });").Do(cxt) return err }), chromedp.Navigate(targetUrl)) return er } func DisableFetchExceptScripts(ctx context.Context, abortUrls []string) func(event interface{}) { return func(event interface{}) { switch ev := event.(type) { case *fetch.EventRequestPaused: go func() { c := chromedp.FromContext(ctx) ctx := cdp.WithExecutor(ctx, c.Target) if util.StrHasFlags(ev.Request.URL, abortUrls) { //fmt.Println("Abort url", ev.Request.URL) fetch.FailRequest(ev.RequestID, network.ErrorReasonBlockedByClient).Do(ctx) } else { fetch.ContinueRequest(ev.RequestID).Do(ctx) } }() } } } // Navigate 导航到指定网址,可以中断一些不必要的请求 func (b *Browser) NavigateWithAbortFlags(tabTitle string, tabUrl string, isNewTab bool, targetUrl string, abortUrlFlag string, timeout int64) (err error) { ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout) if err != nil { return err } //defer fn() //新标签页 if isNewTab { ctx, _ = chromedp.NewContext(ctx) } flagArr := strings.Split(abortUrlFlag, ";") chromedp.ListenTarget(ctx, DisableFetchExceptScripts(ctx, flagArr)) // return chromedp.Run(ctx, fetch.Enable(), //这里作反爬检测 chromedp.ActionFunc(func(cxt context.Context) error { _, err := page.AddScriptToEvaluateOnNewDocument("Object.defineProperty(navigator, 'webdriver', { get: () => false, });").Do(cxt) return err }), chromedp.Navigate(targetUrl)) } // Reload 重新加载tab页地址 func (b *Browser) Reload(tabTitle string, tabUrl string, timeout int64) (err error) { ctx, fn, err := b.findTabContext(tabTitle, tabUrl, timeout) if err != nil { return err } defer fn() return chromedp.Run(ctx, chromedp.Reload()) } // Back 回退 func (b *Browser) Back(tabTitle string, tabUrl string, timeout int64) (err error) { ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout) if err != nil { return err } //defer fn() // return chromedp.Run(ctx, chromedp.NavigateBack()) } // Forward 向前 func (b *Browser) Forward(tabTitle string, tabUrl string, timeout int64) (err error) { ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout) if err != nil { return err } //defer fn() // return chromedp.Run(ctx, chromedp.NavigateForward()) } // ExecuteJS 执行脚本 //某些js需要延迟关闭 needsleep func (b *Browser) ExecuteJS(tabTitle, tabUrl, script string, ret interface{}, timeout int64, needSleep bool) (err error) { b.ExecuteJSChan <- true defer func() { <-b.ExecuteJSChan }() // log.Println("timeout", timeout) // timeout = 15 * 1000 ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout+5000) if err != nil { return err } // defer fn() // return chromedp.Run(ctx, // chromedp.Evaluate(script, ret)) if needSleep { err = chromedp.Run(ctx, chromedp.Evaluate(script, ret), chromedp.Sleep(time.Millisecond*time.Duration(timeout))) } else { err = chromedp.Run(ctx, chromedp.Evaluate(script, ret)) } if err != nil { log.Println(needSleep, timeout, "err", err) } return err } // Click 点击 func (b *Browser) Click(tabTitle, tabUrl, selector string, selectorType int, timeout int64) (err error) { ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout) if err != nil { return err } //defer fn() var act chromedp.QueryAction switch selectorType { case selector_type_id: act = chromedp.Click(selector, chromedp.ByID) case selector_type_query: act = chromedp.Click(selector, chromedp.ByQuery) case selector_type_search: act = chromedp.Click(selector, chromedp.BySearch) case selector_type_jspath: act = chromedp.Click(selector, chromedp.ByJSPath) default: act = chromedp.Click(selector, chromedp.ByQueryAll) } return chromedp.Run(ctx, act) } // keysend 键盘输入 func (b *Browser) KeySend(tabTitle, tabUrl, selector, sendStr string, selectorType int, timeout int64) (err error) { ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout) if err != nil { return err } //defer fn() var act chromedp.QueryAction switch selectorType { case selector_type_id: act = chromedp.SendKeys(selector, sendStr, chromedp.ByID) case selector_type_query: act = chromedp.SendKeys(selector, sendStr, chromedp.ByQuery) case selector_type_search: act = chromedp.SendKeys(selector, sendStr, chromedp.BySearch) case selector_type_jspath: act = chromedp.SendKeys(selector, sendStr, chromedp.ByJSPath) default: act = chromedp.SendKeys(selector, sendStr, chromedp.ByQueryAll) } return chromedp.Run(ctx, act) } // WaitVisible 等待元素可见 func (b *Browser) WaitVisible(tabTitle, tabUrl, selector string, selectorType int, timeout int64) error { ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout) if err != nil { return err } //defer fn() var act chromedp.QueryAction switch selectorType { case selector_type_id: act = chromedp.WaitVisible(selector, chromedp.ByID) case selector_type_query: act = chromedp.WaitVisible(selector, chromedp.ByQuery) case selector_type_search: act = chromedp.WaitVisible(selector, chromedp.BySearch) case selector_type_jspath: act = chromedp.WaitVisible(selector, chromedp.ByJSPath) case 22: act = chromedp.WaitReady(selector, chromedp.ByID) default: act = chromedp.WaitVisible(selector, chromedp.ByQueryAll) } return chromedp.Run(ctx, act) } // 鼠标点击 双击 func (b *Browser) DoubleClick(tabTitle, tabUrl, selector string, selectorType int, timeout int64) (err error) { ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout) if err != nil { return err } //defer fn() var act chromedp.QueryAction switch selectorType { case selector_type_id: act = chromedp.DoubleClick(selector, chromedp.ByID) case selector_type_query: act = chromedp.DoubleClick(selector, chromedp.ByQuery) case selector_type_search: act = chromedp.DoubleClick(selector, chromedp.BySearch) case selector_type_jspath: act = chromedp.DoubleClick(selector, chromedp.ByJSPath) default: act = chromedp.DoubleClick(selector, chromedp.ByQueryAll) } return chromedp.Run(ctx, act) } // 鼠标拖拽 func (b *Browser) MouseDrag(tabTitle, tabUrl string, points [][2]float64, delay []int64, timeout int64) (err error) { ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout) if err != nil { return err } //defer fn() //鼠标拖拽 actionsLen := len(points)*2 + 2 pointsLen := len(points) var acts = make([]chromedp.Action, actionsLen, actionsLen) //第一个动作 acts[0] = chromedp.MouseEvent(input.MousePressed, points[0][0], points[0][1], chromedp.ButtonLeft) for i := 1; i <= len(points); i++ { acts[i*2] = chromedp.MouseEvent(input.MouseMoved, points[i-1][0], points[i-1][1], chromedp.ButtonLeft) acts[i*2+1] = chromedp.Sleep(time.Duration(delay[i]) * time.Millisecond) } acts[actionsLen-1] = chromedp.MouseEvent(input.MouseReleased, points[pointsLen-1][0], points[pointsLen-1][1], chromedp.ButtonLeft) return chromedp.Run(ctx, acts...) } // 等待准备好 func (b *Browser) WaitReady(tabTitle, tabUrl, selector string, selectorType int, timeout int64) (err error) { ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout) if err != nil { return err } //defer fn() var act chromedp.QueryAction switch selectorType { case selector_type_id: act = chromedp.WaitReady(selector, chromedp.ByID) case selector_type_query: act = chromedp.WaitReady(selector, chromedp.ByQuery) case selector_type_search: act = chromedp.WaitReady(selector, chromedp.BySearch) case selector_type_jspath: act = chromedp.WaitReady(selector, chromedp.ByJSPath) default: act = chromedp.WaitReady(selector, chromedp.ByQueryAll) } return chromedp.Run(ctx, act) }