websiteMonitor/spider/browser/baseaction.go

package browser

import (
	"context"
	"log"
	"strings"
	"time"

	"KeyWebsiteMonitor/spider/util"

	"github.com/chromedp/cdproto/cdp"
	"github.com/chromedp/cdproto/fetch"
	"github.com/chromedp/cdproto/input"
	"github.com/chromedp/cdproto/network"
	"github.com/chromedp/cdproto/page"
	"github.com/chromedp/chromedp"
)

// CloseTabs关闭页面
func (b *Browser) CloseTabs(tabTitle, tabUrl string, timeoutInt64 int64) (err error) {
	if timeoutInt64 == 0 {
		timeoutInt64 = 5
	}
	timeout := time.Duration(timeoutInt64) * time.Millisecond

	ts, err := chromedp.Targets(b.Ctx)
	if err != nil {
		return err
	}
	for _, t := range ts {
		bl := false
		if tabUrl != "" && t.URL != "" {
			tu := strings.ReplaceAll(strings.ReplaceAll(tabUrl, "https://", ""), "http://", "")
			tURL := strings.ReplaceAll(strings.ReplaceAll(t.URL, "https://", ""), "http://", "")
			if strings.Contains(tURL, tu) {
				bl = true
			}
		}
		log.Println(tabTitle != "", strings.Contains(t.Title, tabTitle), tabUrl != "", strings.Contains(t.URL, tabUrl), bl)
		if (tabTitle != "" && strings.Contains(t.Title, tabTitle)) ||
			(tabUrl != "" && strings.Contains(t.URL, tabUrl) ||
				bl) {
			newCtx, _ := chromedp.NewContext(b.Ctx, chromedp.WithTargetID(t.TargetID))
			ctx, _ := context.WithTimeout(newCtx, timeout)
			chromedp.Run(
				ctx,
				page.Close(),
			)
		} else {
			log.Printf("tabTile:%s ，taburl：%s ，无法关闭。", tabTitle, tabUrl)
		}
	}
	return nil
}

// CloseTabs关闭页面
func (b *Browser) CloseTabsWithout(tabTitle, tabUrl string, timeoutInt64 int64) (err error) {
	if timeoutInt64 == 0 {
		timeoutInt64 = 5
	}
	timeout := time.Duration(timeoutInt64) * time.Millisecond

	ts, err := chromedp.Targets(b.Ctx)
	if err != nil {
		return err
	}
	for _, t := range ts {
		if (tabTitle != "" && !strings.Contains(t.Title, tabTitle)) ||
			(tabUrl != "" && !strings.Contains(t.URL, tabUrl)) {
			newCtx, _ := chromedp.NewContext(b.Ctx, chromedp.WithTargetID(t.TargetID))
			ctx, _ := context.WithTimeout(newCtx, timeout)
			chromedp.Run(
				ctx,
				page.Close(),
			)
		}
	}
	return nil
}

// Navigate 导航到指定网址
func (b *Browser) Navigate(tabTitle string,
	tabUrl string, isNewTab bool,
	targetUrl string, timeout int64) (err error) {
	ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout)
	if err != nil {
		return err
	}
	//defer fn()
	//新标签页
	if isNewTab {
		ctx, _ = chromedp.NewContext(ctx)
	}
	//
	er := chromedp.Run(ctx,
		//这里作反爬检测
		chromedp.ActionFunc(func(cxt context.Context) error {
			_, err := page.AddScriptToEvaluateOnNewDocument("Object.defineProperty(navigator, 'webdriver', { get: () => false, });").Do(cxt)
			return err
		}),
		chromedp.Navigate(targetUrl))
	return er
}

func DisableFetchExceptScripts(ctx context.Context, abortUrls []string) func(event interface{}) {
	return func(event interface{}) {
		switch ev := event.(type) {
		case *fetch.EventRequestPaused:
			go func() {
				c := chromedp.FromContext(ctx)
				ctx := cdp.WithExecutor(ctx, c.Target)
				if util.StrHasFlags(ev.Request.URL, abortUrls) {
					//fmt.Println("Abort url", ev.Request.URL)
					fetch.FailRequest(ev.RequestID, network.ErrorReasonBlockedByClient).Do(ctx)
				} else {
					fetch.ContinueRequest(ev.RequestID).Do(ctx)
				}
			}()
		}
	}
}

// Navigate 导航到指定网址,可以中断一些不必要的请求
func (b *Browser) NavigateWithAbortFlags(tabTitle string,
	tabUrl string, isNewTab bool,
	targetUrl string, abortUrlFlag string, timeout int64) (err error) {
	ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout)
	if err != nil {
		return err
	}
	//defer fn()
	//新标签页
	if isNewTab {
		ctx, _ = chromedp.NewContext(ctx)
	}
	flagArr := strings.Split(abortUrlFlag, ";")
	chromedp.ListenTarget(ctx, DisableFetchExceptScripts(ctx, flagArr))
	//
	return chromedp.Run(ctx,
		fetch.Enable(),
		//这里作反爬检测
		chromedp.ActionFunc(func(cxt context.Context) error {
			_, err := page.AddScriptToEvaluateOnNewDocument("Object.defineProperty(navigator, 'webdriver', { get: () => false, });").Do(cxt)
			return err
		}),
		chromedp.Navigate(targetUrl))
}

// Reload 重新加载tab页地址
func (b *Browser) Reload(tabTitle string,
	tabUrl string, timeout int64) (err error) {
	ctx, fn, err := b.findTabContext(tabTitle, tabUrl, timeout)
	if err != nil {
		return err
	}
	defer fn()
	return chromedp.Run(ctx,
		chromedp.Reload())
}

// Back 回退
func (b *Browser) Back(tabTitle string,
	tabUrl string, timeout int64) (err error) {
	ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout)
	if err != nil {
		return err
	}
	//defer fn()
	//
	return chromedp.Run(ctx,
		chromedp.NavigateBack())
}

// Forward 向前
func (b *Browser) Forward(tabTitle string,
	tabUrl string, timeout int64) (err error) {
	ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout)
	if err != nil {
		return err
	}
	//defer fn()
	//
	return chromedp.Run(ctx,
		chromedp.NavigateForward())
}

// ExecuteJS 执行脚本
//某些js需要延迟关闭 needsleep
func (b *Browser) ExecuteJS(tabTitle, tabUrl, script string, ret interface{}, timeout int64, needSleep bool) (err error) {
	b.ExecuteJSChan <- true
	defer func() {
		<-b.ExecuteJSChan
	}()
	// log.Println("timeout", timeout)
	// timeout = 15 * 1000
	ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout+5000)
	if err != nil {
		return err
	}
	// defer fn()
	// return chromedp.Run(ctx,
	// 	chromedp.Evaluate(script, ret))
	if needSleep {
		err = chromedp.Run(ctx,
			chromedp.Evaluate(script, ret), chromedp.Sleep(time.Millisecond*time.Duration(timeout)))

	} else {
		err = chromedp.Run(ctx,
			chromedp.Evaluate(script, ret))
	}
	if err != nil {
		log.Println(needSleep, timeout, "err", err)
	}
	return err
}

// Click 点击
func (b *Browser) Click(tabTitle, tabUrl, selector string, selectorType int, timeout int64) (err error) {
	ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout)
	if err != nil {
		return err
	}
	//defer fn()
	var act chromedp.QueryAction
	switch selectorType {
	case selector_type_id:
		act = chromedp.Click(selector, chromedp.ByID)
	case selector_type_query:
		act = chromedp.Click(selector, chromedp.ByQuery)
	case selector_type_search:
		act = chromedp.Click(selector, chromedp.BySearch)
	case selector_type_jspath:
		act = chromedp.Click(selector, chromedp.ByJSPath)
	default:
		act = chromedp.Click(selector, chromedp.ByQueryAll)
	}
	return chromedp.Run(ctx,
		act)
}

// keysend 键盘输入
func (b *Browser) KeySend(tabTitle, tabUrl, selector, sendStr string,
	selectorType int, timeout int64) (err error) {
	ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout)
	if err != nil {
		return err
	}
	//defer fn()
	var act chromedp.QueryAction
	switch selectorType {
	case selector_type_id:
		act = chromedp.SendKeys(selector, sendStr, chromedp.ByID)
	case selector_type_query:
		act = chromedp.SendKeys(selector, sendStr, chromedp.ByQuery)
	case selector_type_search:
		act = chromedp.SendKeys(selector, sendStr, chromedp.BySearch)
	case selector_type_jspath:
		act = chromedp.SendKeys(selector, sendStr, chromedp.ByJSPath)
	default:
		act = chromedp.SendKeys(selector, sendStr, chromedp.ByQueryAll)
	}
	return chromedp.Run(ctx,
		act)
}

// WaitVisible 等待元素可见
func (b *Browser) WaitVisible(tabTitle, tabUrl, selector string,
	selectorType int, timeout int64) error {
	ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout)
	if err != nil {
		return err
	}
	//defer fn()
	var act chromedp.QueryAction
	switch selectorType {
	case selector_type_id:
		act = chromedp.WaitVisible(selector, chromedp.ByID)
	case selector_type_query:
		act = chromedp.WaitVisible(selector, chromedp.ByQuery)
	case selector_type_search:
		act = chromedp.WaitVisible(selector, chromedp.BySearch)
	case selector_type_jspath:
		act = chromedp.WaitVisible(selector, chromedp.ByJSPath)
	case 22:
		act = chromedp.WaitReady(selector, chromedp.ByID)
	default:
		act = chromedp.WaitVisible(selector, chromedp.ByQueryAll)
	}
	return chromedp.Run(ctx,
		act)
}

// 鼠标点击 双击
func (b *Browser) DoubleClick(tabTitle, tabUrl, selector string, selectorType int, timeout int64) (err error) {
	ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout)
	if err != nil {
		return err
	}
	//defer fn()
	var act chromedp.QueryAction
	switch selectorType {
	case selector_type_id:
		act = chromedp.DoubleClick(selector, chromedp.ByID)
	case selector_type_query:
		act = chromedp.DoubleClick(selector, chromedp.ByQuery)
	case selector_type_search:
		act = chromedp.DoubleClick(selector, chromedp.BySearch)
	case selector_type_jspath:
		act = chromedp.DoubleClick(selector, chromedp.ByJSPath)
	default:
		act = chromedp.DoubleClick(selector, chromedp.ByQueryAll)
	}
	return chromedp.Run(ctx,
		act)
}

// 鼠标拖拽
func (b *Browser) MouseDrag(tabTitle, tabUrl string, points [][2]float64, delay []int64, timeout int64) (err error) {
	ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout)
	if err != nil {
		return err
	}
	//defer fn()
	//鼠标拖拽
	actionsLen := len(points)*2 + 2
	pointsLen := len(points)
	var acts = make([]chromedp.Action, actionsLen, actionsLen)
	//第一个动作
	acts[0] = chromedp.MouseEvent(input.MousePressed, points[0][0], points[0][1], chromedp.ButtonLeft)
	for i := 1; i <= len(points); i++ {
		acts[i*2] = chromedp.MouseEvent(input.MouseMoved, points[i-1][0], points[i-1][1], chromedp.ButtonLeft)
		acts[i*2+1] = chromedp.Sleep(time.Duration(delay[i]) * time.Millisecond)
	}
	acts[actionsLen-1] = chromedp.MouseEvent(input.MouseReleased, points[pointsLen-1][0], points[pointsLen-1][1], chromedp.ButtonLeft)
	return chromedp.Run(ctx,
		acts...)
}

// 等待准备好
func (b *Browser) WaitReady(tabTitle, tabUrl, selector string, selectorType int, timeout int64) (err error) {
	ctx, _, err := b.findTabContext(tabTitle, tabUrl, timeout)
	if err != nil {
		return err
	}
	//defer fn()
	var act chromedp.QueryAction
	switch selectorType {
	case selector_type_id:
		act = chromedp.WaitReady(selector, chromedp.ByID)
	case selector_type_query:
		act = chromedp.WaitReady(selector, chromedp.ByQuery)
	case selector_type_search:
		act = chromedp.WaitReady(selector, chromedp.BySearch)
	case selector_type_jspath:
		act = chromedp.WaitReady(selector, chromedp.ByJSPath)
	default:
		act = chromedp.WaitReady(selector, chromedp.ByQueryAll)
	}
	return chromedp.Run(ctx,
		act)
}