spider_creater/backend/vm/single.go

package vm

import (
	"container/list"
	_ "embed"
	"fmt"
	"github.com/chromedp/chromedp"
	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	be "spider_creator/backend"
	"spider_creator/backend/ai"
	"strconv"
	"time"
)

// NewVM
func NewVM(attachesDir string, dnf be.EventNotifyFace) *VM {
	return &VM{
		attachesDir, dnf,
	}
}

func (vm *VM) RunSpiderTmp(url string, maxPages int, listDealy, trunPageDelay, contentDelay int64, headless bool, showImage bool, proxyServe string, exit chan bool, cssMark map[string]interface{}) {
	sc, err := be.NewSpiderConfig(cssMark)
	if err != nil {
		qu.Debug("标注信息传输失败！")
		vm.dnf.Dispatch("debug_event", "标注信息传输失败！")
		return
	}
	if url != "" {
		sc.Href = url
	}
	_, baseCancel, _, _, ctx, cancel := be.NewBrowser(headless, showImage, proxyServe)
	qu.Debug("1浏览器打开", *sc)
	vm.dnf.Dispatch("debug_event", "1 浏览器打开")
	defer func() {
		cancel()
		baseCancel()
		qu.Debug("0浏览器已经销毁")
		vm.dnf.Dispatch("debug_event", "0 浏览器已经销毁")
		close(exit)
	}()
	chromedp.Run(ctx, chromedp.Tasks{
		chromedp.Navigate(sc.Href),                                  //打开页面
		chromedp.WaitReady("document.body", chromedp.ByJSPath),      //等待body加载完毕
		chromedp.Sleep(time.Duration(listDealy) * time.Millisecond), //列表页等待
	})
	vm.dnf.Dispatch("debug_event", "2 页面已经打开")
	qu.Debug("2页面打开")
	var runJs string = sc.ListJSCode

	//TODO 2. 执行JS代码，获取列表页信息
	if be.RegSpace.ReplaceAllString(runJs, "") == "" {
		runJs = renderJavascriptCoder(loadListItemsJS, sc)
	}
	qu.Debug("列表页执行JS:", runJs)
	currentResult := list.New()
	be.DataResults[sc.Code] = currentResult
	no := 1
	for j := 0; j < maxPages; j++ {
		qu.Debug("开始检查第" + fmt.Sprint(j+1) + "页...")
		listResult := make(be.ResultItems, 0)
		err = chromedp.Run(ctx, chromedp.Tasks{
			chromedp.Evaluate(runJs, &listResult),
		})
		if err != nil {
			qu.Debug("执行JS代码失败", err.Error())
			vm.dnf.Dispatch("debug_event", "2 第"+fmt.Sprint(j+1)+"页执行JS代码失败")
			continue
		}
		qu.Debug("第"+fmt.Sprint(j+1)+"页列表采集条数：", len(listResult))
		vm.dnf.Dispatch("debug_event", "3 第"+fmt.Sprint(j+1)+"获取列表完成")
		for _, v := range listResult {
			select {
			case <-exit:
				return
			default:
				v.Site = sc.Site
				v.Channel = sc.Channel
				v.Title = v.ListTitle
				v.Content = "详见正文"
				v.No = no
				qu.Debug(v.No, v.ListTitle, v.Href)
				no++
				currentResult.PushBack(v)
			}
		}
		vm.dnf.Dispatch("debug_event", "4 第"+fmt.Sprint(j+1)+"页采集完成，准备执行翻页")
		if j < maxPages-1 {
			if err = trunPage(sc, trunPageDelay, ctx); err != nil {
				qu.Debug("翻页失败", err.Error())
				vm.dnf.Dispatch("debug_event", "5 第"+fmt.Sprint(j+1)+"页翻页失败")
				time.Sleep(3 * time.Second)
				break
			}
		}
	}
	vm.dnf.Dispatch("debug_event", "6 采集测试完成")
	qu.Debug("6采集测试完成")
}

// RunSpider
func (vm *VM) RunSpider(url string, maxPages int, listDealy int64, contentDelay int64, headless bool, showImage bool, proxyServe string, exit chan bool, cssMark map[string]interface{}) {
	sc, err := be.NewSpiderConfig(cssMark)
	if err != nil {
		qu.Debug("标注信息传输失败！")
		vm.dnf.Dispatch("debug_event", "标注信息传输失败！")
		return
	}
	if url != "" {
		sc.Href = url
	}
	_, baseCancel, _, _, ctx, cancel := be.NewBrowser(headless, showImage, proxyServe)
	qu.Debug("1浏览器打开", *sc)
	vm.dnf.Dispatch("debug_event", "1 浏览器打开")
	defer func() {
		cancel()
		baseCancel()
		qu.Debug("0浏览器已经销毁")
		vm.dnf.Dispatch("debug_event", "0 浏览器已经销毁")
		close(exit)
	}()
	chromedp.Run(ctx, chromedp.Tasks{
		chromedp.Navigate(sc.Href),                                  //打开页面
		chromedp.WaitReady("document.body", chromedp.ByJSPath),      //等待body加载完毕
		chromedp.Sleep(time.Duration(listDealy) * time.Millisecond), //列表页等待
	})
	vm.dnf.Dispatch("debug_event", "2 页面已经打开")
	qu.Debug("2页面打开")
	var runJs string = sc.ListJSCode
	listResult := make(be.ResultItems, 0)
	//TODO 2. 执行JS代码，获取列表页信息
	if be.RegSpace.ReplaceAllString(runJs, "") == "" {
		runJs = renderJavascriptCoder(loadListItemsJS, sc)
	}
	qu.Debug("execute list jscode", runJs)
	err = chromedp.Run(ctx, chromedp.Tasks{
		chromedp.Evaluate(runJs, &listResult),
	})
	if err != nil {
		qu.Debug("执行JS代码失败", err.Error())
		vm.dnf.Dispatch("debug_event", "2 执行JS代码失败")
		return
	}
	vm.dnf.Dispatch("debug_event", "3 获取列表完成")
	qu.Debug("3获取列表完成", len(listResult))

	//TODO 3. 打开详情页 ,最多打开10条
	runJs = sc.ContentJSCode
	if be.RegSpace.ReplaceAllString(runJs, "") == "" {
		runJs = renderJavascriptCoder(loadContentJS, sc)
	}
	currentResult := list.New()
	be.DataResults[sc.Code] = currentResult
	qu.Debug("execute content js", runJs)
	no := 1
	for _, v := range listResult {
		select {
		case <-exit:
			return
		default:
			qu.Debug(v.No, v.ListTitle, v.Href)
			vm.dnf.Dispatch("debug_event", fmt.Sprintf("4. %d- 待 下载详情页 %s ", v.No, v.ListTitle))
			var result string = ""
			err = chromedp.Run(ctx, chromedp.Tasks{
				chromedp.Navigate(v.Href),
				chromedp.WaitReady(`document.body`, chromedp.ByJSPath),
				chromedp.Sleep(time.Duration(contentDelay) * time.Millisecond),
				chromedp.Evaluate(runJs, v),
			})
			v.No = no
			no++
			v.Site = sc.Site
			v.Channel = sc.Channel
			if err != nil {
				qu.Debug("执行JS代码失败", err.Error())
			}
			if len(v.AttachLinks) > 0 { //有附件
				vm.dnf.Dispatch("debug_event", fmt.Sprintf("4. 下载附件"))
				//TODO 下载附件
				downloadAttaches(v, vm.attachesDir)
			}
			//关闭当前TAB页
			chromedp.Run(ctx, chromedp.Tasks{
				chromedp.Evaluate(`var ret="";window.close();ret`, &result),
			})
			vm.dnf.Dispatch("debug_event", fmt.Sprintf("4. %d- 下载详情页 %s 完成", v.No, v.Title))
			currentResult.PushBack(v)
		}
	}
	vm.dnf.Dispatch("debug_event", "5 采集测试完成")
	qu.Debug("5采集测试完成")
}

// CountYestodayArts 统计昨日信息发布量
func (vm *VM) CountYestodayArts(url string, listDealy int64, trunPageDelay int64,
	headless bool, showImage bool, exit chan bool, currentSpiderConfig *be.SpiderConfig) (count int) {
	sc := be.MergeSpiderConfig(currentSpiderConfig, &be.SpiderConfig{Href: url})
	_, baseCancel, _, _, ctx, cancel := be.NewBrowser(headless, showImage, "")
	qu.Debug("1浏览器打开")
	vm.dnf.Dispatch("debug_event", "1 浏览器打开")
	defer func() {
		cancel()
		baseCancel()
		qu.Debug("0浏览器已经销毁")
		vm.dnf.Dispatch("debug_event", "0 浏览器已经销毁")
		vm.dnf.Dispatch("debug_event", fmt.Sprintf("99 昨日信息发布量:%d ", count))
		close(exit)
	}()

	//时间比较
	now := time.Now()
	yesterday := now.AddDate(0, 0, -1) // 获取昨天的日期
	startOfYesterday := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, now.Location())
	endOfYesterday := startOfYesterday.AddDate(0, 0, 1).Add(-time.Nanosecond)

	//TODO 1.
	chromedp.Run(ctx, chromedp.Tasks{
		chromedp.Navigate(sc.Href),
		chromedp.WaitReady("document.body", chromedp.ByJSPath),
		chromedp.Sleep(time.Duration(listDealy) * time.Millisecond),
	})
	vm.dnf.Dispatch("debug_event", "2 页面已经打开")
	qu.Debug("2页面打开")
	//TODO 2. 执行JS代码，获取列表页信息
	runJs := renderJavascriptCoder(loadListItemsJS, sc)
	tmp := map[string]bool{}
	//最多翻页1000页
	for i := 0; i < MAX_TRUN_PAGE; i++ {
		select {
		case <-exit:
			return
		default:
			vm.dnf.Dispatch("debug_event", "3 执行列表页JS")
			listResult := make(be.ResultItems, 0)
			err := chromedp.Run(ctx, chromedp.Tasks{
				chromedp.Evaluate(runJs, &listResult),
			})
			if err != nil {
				qu.Debug("执行JS代码失败", err.Error())
				vm.dnf.Dispatch("debug_event", "3 执行JS代码失败")
				return
			}
			//TODO 人工智能转换采集到的日期
			callAIState := false
			for j := 0; j < 5; j++ {
				vm.dnf.Dispatch("debug_event", "3 执行AI提取列表发布时间"+strconv.Itoa(j+1))
				err := ai.UpdateResultDateStr(listResult)
				if err == nil {
					callAIState = true
					break
				}
			}
			if !callAIState {
				vm.dnf.Dispatch("debug_event", "3 多轮次调用AI均未得到合理结果")
				return
			}
			//TODO 日期统计
			for _, r := range listResult {
				day, err := time.Parse("2006-01-02", r.ListPubTime)
				if err != nil {
					continue
				}
				if _, ok := tmp[r.Href]; ok { //去重
					continue
				}
				if day.After(startOfYesterday) && day.Before(endOfYesterday) {
					count += 1
				} else if day.Before(startOfYesterday) {
					return
				}
			}
			vm.dnf.Dispatch("debug_event", fmt.Sprintf("4 当前观测昨日信息发布量:%d ", count))
			//TODO 翻页
			//fmt.Println("下一页CSS选择器", currentSpiderConfig.ListNextPageCss)
			chromedp.Run(ctx, chromedp.Tasks{
				chromedp.Click(fmt.Sprintf(`document.querySelector("%s")`, currentSpiderConfig.ListNextPageCss),
					chromedp.ByJSPath),
				chromedp.Sleep(time.Duration(trunPageDelay) * time.Millisecond),
			})
		}
	}
	return
}