package vm import ( "container/list" _ "embed" "fmt" "log" be "spidercreator/backend" ai "spidercreator/backend/ai" "strconv" "time" "github.com/chromedp/chromedp" ) // NewVM func NewVM(attachesDir string, dnf be.EventNotifyFace) *VM { return &VM{ attachesDir, dnf, } } // RunSpider func (vm *VM) RunSpider(url string, maxPages int, listDealy int64, contentDelay int64, headless bool, showImage bool, proxyServe string, exit chan bool, currentSpiderConfig *be.SpiderConfig, currentResult *list.List) { sc := be.MergeSpiderConfig(currentSpiderConfig, &be.SpiderConfig{Url: url}) _, baseCancel, _, _, ctx, cancel := be.NewBrowser(headless, showImage, proxyServe) log.Println("1浏览器打开", *sc) vm.dnf.Dispatch("debug_event", "1 浏览器打开") defer func() { cancel() baseCancel() log.Println("0浏览器已经销毁") vm.dnf.Dispatch("debug_event", "0 浏览器已经销毁") close(exit) }() chromedp.Run(ctx, chromedp.Tasks{ chromedp.Navigate(sc.Url), chromedp.WaitReady("document.body", chromedp.ByJSPath), chromedp.Sleep(time.Duration(listDealy) * time.Millisecond), }) vm.dnf.Dispatch("debug_event", "2 页面已经打开") log.Println("2页面打开") var runJs string = sc.ListJSCode listResult := make(be.ResultItems, 0) //TODO 2. 执行JS代码,获取列表页信息 if runJs == "" { runJs = renderJavascriptCoder(loadListItemsJS, sc) } //log.Println("execute list jscode", runJs) err := chromedp.Run(ctx, chromedp.Tasks{ chromedp.Evaluate(runJs, &listResult), }) if err != nil { log.Println("执行JS代码失败", err.Error()) vm.dnf.Dispatch("debug_event", "2 执行JS代码失败") return } vm.dnf.Dispatch("debug_event", "3 获取列表完成") log.Println("3获取列表完成") //TODO 3. 打开详情页 ,最多打开10条 runJs = sc.ContentJSCode if runJs == "" { runJs = renderJavascriptCoder(loadContentJS, sc) } //log.Println("execute content js", runJs) for _, v := range listResult { select { case <-exit: return default: vm.dnf.Dispatch("debug_event", fmt.Sprintf("4. %d- 待 下载详情页 %s ", v.No, v.ListTitle)) var result string = "" err = chromedp.Run(ctx, chromedp.Tasks{ chromedp.Navigate(v.Href), chromedp.WaitReady(`document.body`, chromedp.ByJSPath), chromedp.Sleep(time.Duration(contentDelay) * time.Millisecond), chromedp.Evaluate(runJs, v), }) if err != nil { log.Println("执行JS代码失败", err.Error()) } if len(v.AttachLinks) > 0 { //有附件 vm.dnf.Dispatch("debug_event", fmt.Sprintf("4. 下载附件")) //TODO 下载附件 downloadAttaches(v, vm.attachesDir) } //关闭当前TAB页 chromedp.Run(ctx, chromedp.Tasks{ chromedp.Evaluate(`var ret="";window.close();ret`, &result), }) vm.dnf.Dispatch("debug_event", fmt.Sprintf("4. %d- 下载详情页 %s 完成", v.No, v.Title)) currentResult.PushBack(v) } } vm.dnf.Dispatch("debug_event", "5 采集测试完成") log.Println("5采集测试完成") } // CountYestodayArts 统计昨日信息发布量 func (vm *VM) CountYestodayArts(url string, listDealy int64, trunPageDelay int64, headless bool, showImage bool, exit chan bool, currentSpiderConfig *be.SpiderConfig) (count int) { sc := be.MergeSpiderConfig(currentSpiderConfig, &be.SpiderConfig{Url: url}) _, baseCancel, _, _, ctx, cancel := be.NewBrowser(headless, showImage, "") log.Println("1浏览器打开") vm.dnf.Dispatch("debug_event", "1 浏览器打开") defer func() { cancel() baseCancel() log.Println("0浏览器已经销毁") vm.dnf.Dispatch("debug_event", "0 浏览器已经销毁") vm.dnf.Dispatch("debug_event", fmt.Sprintf("99 昨日信息发布量:%d ", count)) close(exit) }() //时间比较 now := time.Now() yesterday := now.AddDate(0, 0, -1) // 获取昨天的日期 startOfYesterday := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, now.Location()) endOfYesterday := startOfYesterday.AddDate(0, 0, 1).Add(-time.Nanosecond) //TODO 1. chromedp.Run(ctx, chromedp.Tasks{ chromedp.Navigate(sc.Url), chromedp.WaitReady("document.body", chromedp.ByJSPath), chromedp.Sleep(time.Duration(listDealy) * time.Millisecond), }) vm.dnf.Dispatch("debug_event", "2 页面已经打开") log.Println("2页面打开") //TODO 2. 执行JS代码,获取列表页信息 runJs := renderJavascriptCoder(loadListItemsJS, sc) tmp := map[string]bool{} //最多翻页1000页 for i := 0; i < MAX_TRUN_PAGE; i++ { select { case <-exit: return default: vm.dnf.Dispatch("debug_event", "3 执行列表页JS") listResult := make(be.ResultItems, 0) err := chromedp.Run(ctx, chromedp.Tasks{ chromedp.Evaluate(runJs, &listResult), }) if err != nil { log.Println("执行JS代码失败", err.Error()) vm.dnf.Dispatch("debug_event", "3 执行JS代码失败") return } //TODO 人工智能转换采集到的日期 callAIState := false for j := 0; j < 5; j++ { vm.dnf.Dispatch("debug_event", "3 执行AI提取列表发布时间"+strconv.Itoa(j+1)) err := ai.UpdateResultDateStr(listResult) if err == nil { callAIState = true break } } if !callAIState { vm.dnf.Dispatch("debug_event", "3 多轮次调用AI均未得到合理结果") return } //TODO 日期统计 for _, r := range listResult { day, err := time.Parse("2006-01-02", r.ListPubTime) if err != nil { continue } if _, ok := tmp[r.Href]; ok { //去重 continue } if day.After(startOfYesterday) && day.Before(endOfYesterday) { count += 1 } else if day.Before(startOfYesterday) { return } } vm.dnf.Dispatch("debug_event", fmt.Sprintf("4 当前观测昨日信息发布量:%d ", count)) //TODO 翻页 //fmt.Println("下一页CSS选择器", currentSpiderConfig.ListNextPageCss) chromedp.Run(ctx, chromedp.Tasks{ chromedp.Click(fmt.Sprintf(`document.querySelector("%s")`, currentSpiderConfig.ListNextPageCss), chromedp.ByJSPath), chromedp.Sleep(time.Duration(trunPageDelay) * time.Millisecond), }) } } return }