Selaa lähdekoodia

详情页下载、测试功能修改

mxs 9 kuukautta sitten
vanhempi
commit
e2a0b22435

+ 133 - 142
backend/vm/check.go

@@ -11,29 +11,144 @@ import (
 )
 )
 
 
 // VerifySpiderConfig 验证爬虫配置,支持翻页,列表项数据只提取2条
 // VerifySpiderConfig 验证爬虫配置,支持翻页,列表项数据只提取2条
+func (vm *VM) VerifySpiderConfig(sc *be.SpiderConfig) (*be.SpiderConfigVerifyResult, error) {
+	qu.Debug("sc---", *sc)
+	verifyResult := list.New()
+	be.DataResults[sc.Code] = verifyResult
+	ret := &be.SpiderConfigVerifyResult{false, false, false, false, false, false, false}
+	_, baseCancelFn, _, _, ctx, incCancelFn := be.NewBrowser(false, false, "")    //列表页使用
+	_, baseCancelFn2, _, _, ctx2, incCancelFn2 := be.NewBrowser(false, false, "") //详情页使用
+	defer func() {
+		incCancelFn2()
+		baseCancelFn2()
+		incCancelFn()
+		baseCancelFn()
+	}()
+
+	listRunJs, contentRunJs := sc.ListJSCode, sc.ContentJSCode
+	//2. 执行JS代码,获取列表页信息
+	if be.RegSpace.ReplaceAllString(listRunJs, "") == "" {
+		listRunJs = renderJavascriptCoder(loadListItemsJS, sc)
+	}
+	if be.RegSpace.ReplaceAllString(contentRunJs, "") == "" {
+		contentRunJs = renderJavascriptCoder(loadContentJS, sc)
+	}
+	qu.Debug("获取列表页JS代码:", listRunJs)
+	qu.Debug("获取详情页JS代码:", contentRunJs)
+	//TODO 3.打开列表,获取条目清单
+	chromedp.Run(ctx, chromedp.Tasks{
+		chromedp.Navigate(sc.Href),
+		chromedp.WaitReady("document.body", chromedp.ByJSPath),
+		//chromedp.Sleep(1000 * time.Millisecond),
+		chromedp.Sleep(time.Duration(sc.ListDelayTime) * time.Millisecond),
+	})
+	no := 1
+T:
+	for j := 0; j < 2; j++ { //最多检查2页
+		qu.Debug("开始检查第" + fmt.Sprint(j+1) + "页...")
+		listResult := make(be.ResultItems, 0)
+		err := chromedp.Run(ctx, chromedp.Tasks{
+			chromedp.Evaluate(listRunJs, &listResult),
+		})
+		if err != nil {
+			qu.Debug("执行列表页JS代码失败", err.Error())
+			continue
+		}
+		//TODO 5.操作详情页
+		qu.Debug("列表采集条数:", len(listResult))
+		for contentIndex, r := range listResult {
+			qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条")
+			if contentIndex > 1 { //每页只校验2条
+				break
+			}
+			//打开详情页
+			err = chromedp.Run(ctx2, chromedp.Tasks{
+				chromedp.Navigate(r.Href),
+				chromedp.WaitReady("document.body", chromedp.ByJSPath),
+				//chromedp.Sleep(2000 * time.Millisecond),
+				chromedp.Sleep(time.Duration(sc.ContentDelayTime) * time.Millisecond),
+			})
+			if err != nil {
+				qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条详情页打开异常")
+				continue
+			}
+			//获取详情页内容
+			err = chromedp.Run(ctx2, chromedp.Tasks{
+				chromedp.Evaluate(contentRunJs, r),
+			})
+			if err != nil {
+				qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条详情页内容获取失败")
+				continue
+			}
+			//下载附件
+			if sc.AttachCss != "" {
+				downloadAttaches(r, vm.attachesDir)
+			}
+			r.Site = sc.Site
+			r.Channel = sc.Channel
+			if r.Title == "" {
+				r.Title = r.ListTitle
+			}
+			if r.PublishTime == "" {
+				r.PublishTime = r.ListPubTime
+			}
+			r.No = no
+			no += 1
+			//结果放入缓存
+			verifyResult.PushBack(r)
+		}
+		qu.Debug("第"+fmt.Sprint(j+1)+"页校验成功数据条数:", verifyResult.Len())
+		//TODO 6.翻页
+		if verifyResult.Len() > 0 {
+			if sc.MaxPages == 1 { //最大页为1,不校验翻页
+				ret.ListTrunPage = true
+				break
+			} else if sc.MaxPages > 1 { //&& !ret.ListTrunPage {
+				if err = trunPage(sc, sc.ListTurnDelayTime, ctx); err != nil { //翻页失败
+					qu.Debug("第" + fmt.Sprint(j+1) + "页翻页失败")
+					break T
+				} else {
+					ret.ListTrunPage = true
+				}
+			}
+		}
+	}
+	//检查
+	for el := verifyResult.Front(); el != nil; el = el.Next() {
+		r, _ := el.Value.(*be.ResultItem)
+		ret.Title = r.Title != ""
+		ret.PublishUnit = r.PublishUnit != ""
+		ret.PublishTime = r.PublishTime != ""
+		ret.Content = r.Content != ""
+		ret.Attaches = len(r.AttachLinks) > 0
+	}
+	qu.Debug(verifyResult.Len())
+	ret.ListItems = (sc.MaxPages == 1 && verifyResult.Len() > 0) || (sc.MaxPages > 1 && verifyResult.Len() > 2)
+
+	//TODO:每次验证结果存库、内存?
+	return ret, nil
+}
+
+// VerifySpiderConfig 只验证列表标注
 //func (vm *VM) VerifySpiderConfig(sc *be.SpiderConfig) (*be.SpiderConfigVerifyResult, error) {
 //func (vm *VM) VerifySpiderConfig(sc *be.SpiderConfig) (*be.SpiderConfigVerifyResult, error) {
 //	qu.Debug("sc---", *sc)
 //	qu.Debug("sc---", *sc)
 //	verifyResult := list.New()
 //	verifyResult := list.New()
-//	ret := &be.SpiderConfigVerifyResult{true, true, true, true, true, true, true}
-//	_, baseCancelFn, _, _, ctx, incCancelFn := be.NewBrowser(false, false, "")    //列表页使用
-//	_, baseCancelFn2, _, _, ctx2, incCancelFn2 := be.NewBrowser(false, false, "") //详情页使用
+//	ret := &be.SpiderConfigVerifyResult{false, true, false, true, true, true, false}
+//	_, baseCancelFn, _, _, ctx, incCancelFn := be.NewBrowser(false, false, "") //列表页使用
 //	defer func() {
 //	defer func() {
-//		incCancelFn2()
-//		baseCancelFn2()
 //		incCancelFn()
 //		incCancelFn()
 //		baseCancelFn()
 //		baseCancelFn()
 //	}()
 //	}()
 //
 //
 //	listRunJs, contentRunJs := sc.ListJSCode, sc.ContentJSCode
 //	listRunJs, contentRunJs := sc.ListJSCode, sc.ContentJSCode
 //	//TODO 2. 执行JS代码,获取列表页信息
 //	//TODO 2. 执行JS代码,获取列表页信息
-//	if listRunJs == "" {
+//	if be.RegSpace.ReplaceAllString(listRunJs, "") == "" {
 //		listRunJs = renderJavascriptCoder(loadListItemsJS, sc)
 //		listRunJs = renderJavascriptCoder(loadListItemsJS, sc)
 //	}
 //	}
-//	if contentRunJs == "" {
+//	if be.RegSpace.ReplaceAllString(contentRunJs, "") == "" {
 //		contentRunJs = renderJavascriptCoder(loadContentJS, sc)
 //		contentRunJs = renderJavascriptCoder(loadContentJS, sc)
 //	}
 //	}
 //	qu.Debug("列表页JS:", listRunJs)
 //	qu.Debug("列表页JS:", listRunJs)
-//	qu.Debug("详情页JS:", contentRunJs)
 //	//TODO 3.打开列表,获取条目清单
 //	//TODO 3.打开列表,获取条目清单
 //	chromedp.Run(ctx, chromedp.Tasks{
 //	chromedp.Run(ctx, chromedp.Tasks{
 //		chromedp.Navigate(sc.Href),
 //		chromedp.Navigate(sc.Href),
@@ -56,37 +171,17 @@ import (
 //		//TODO 5.操作详情页
 //		//TODO 5.操作详情页
 //		qu.Debug("列表采集条数:", len(listResult))
 //		qu.Debug("列表采集条数:", len(listResult))
 //		for contentIndex, r := range listResult {
 //		for contentIndex, r := range listResult {
-//			qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条")
-//			if contentIndex > 1 { //每页只采集2条
+//			if contentIndex > 1 { //每页只校验2条
 //				break
 //				break
 //			}
 //			}
-//			//打开详情页
-//			err = chromedp.Run(ctx2, chromedp.Tasks{
-//				chromedp.Navigate(r.Href),
-//				chromedp.WaitReady("document.body", chromedp.ByJSPath),
-//				chromedp.Sleep(2000 * time.Millisecond),
-//				//chromedp.Sleep(time.Duration(sc.ContentDelayTime) * time.Millisecond),
-//			})
-//			if err != nil {
-//				qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条详情页打开异常")
-//				continue
-//			}
-//			//获取详情页内容
-//			err = chromedp.Run(ctx2, chromedp.Tasks{
-//				chromedp.Evaluate(contentRunJs, r),
-//			})
-//			if err != nil {
-//				qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条详情页内容获取失败")
-//				continue
-//			}
-//			if sc.AttachCss != "" {
-//				downloadAttaches(r, vm.attachesDir)
-//			}
+//			qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条")
 //			r.Site = sc.Site
 //			r.Site = sc.Site
 //			r.Channel = sc.Channel
 //			r.Channel = sc.Channel
+//			qu.Debug(r.Title, r.ListTitle)
 //			if r.Title == "" {
 //			if r.Title == "" {
 //				r.Title = r.ListTitle
 //				r.Title = r.ListTitle
 //			}
 //			}
+//			qu.Debug(r.PublishTime, r.ListPubTime)
 //			if r.PublishTime == "" {
 //			if r.PublishTime == "" {
 //				r.PublishTime = r.ListPubTime
 //				r.PublishTime = r.ListPubTime
 //			}
 //			}
@@ -101,8 +196,9 @@ import (
 //			if sc.MaxPages == 1 { //最大页为1,不校验翻页
 //			if sc.MaxPages == 1 { //最大页为1,不校验翻页
 //				ret.ListTrunPage = true
 //				ret.ListTrunPage = true
 //				break
 //				break
-//			} else if sc.MaxPages > 1 { //&& !ret.ListTrunPage {
+//			} else if sc.MaxPages > 1 { // && !ret.ListTrunPage {
 //				if err = trunPage(sc, sc.ListTurnDelayTime, ctx); err != nil { //翻页失败
 //				if err = trunPage(sc, sc.ListTurnDelayTime, ctx); err != nil { //翻页失败
+//					qu.Debug("翻页失败:", err)
 //					break T
 //					break T
 //				} else {
 //				} else {
 //					ret.ListTrunPage = true
 //					ret.ListTrunPage = true
@@ -113,119 +209,14 @@ import (
 //	//检查
 //	//检查
 //	for el := verifyResult.Front(); el != nil; el = el.Next() {
 //	for el := verifyResult.Front(); el != nil; el = el.Next() {
 //		r, _ := el.Value.(*be.ResultItem)
 //		r, _ := el.Value.(*be.ResultItem)
-//		if ret.Title {
-//			ret.Title = r.Title != ""
-//		}
-//		if ret.PublishUnit {
-//			ret.PublishUnit = r.PublishUnit != ""
-//		}
-//		if ret.PublishTime {
-//			ret.PublishTime = r.PublishTime != ""
-//		}
-//		if ret.Content {
-//			ret.Content = r.Content != ""
-//		}
-//		if ret.Attaches {
-//			ret.Attaches = len(r.AttachLinks) > 0
-//		}
+//		ret.Title = r.Title != ""
+//		qu.Debug("Check Title:", ret.Title, r.Title, r.ListTitle)
+//		ret.PublishTime = r.PublishTime != ""
+//		qu.Debug("Check PublishTime:", ret.PublishTime, r.PublishTime, r.ListPubTime)
 //	}
 //	}
-//	qu.Debug(verifyResult.Len())
 //	if ret.ListItems {
 //	if ret.ListItems {
 //		ret.ListItems = (sc.MaxPages == 1 && verifyResult.Len() > 0) || (sc.MaxPages > 1 && verifyResult.Len() > 2)
 //		ret.ListItems = (sc.MaxPages == 1 && verifyResult.Len() > 0) || (sc.MaxPages > 1 && verifyResult.Len() > 2)
 //	}
 //	}
 //
 //
-//	//TODO:每次验证结果存库、内存?
 //	return ret, nil
 //	return ret, nil
 //}
 //}
-
-// VerifySpiderConfig 只验证列表标注
-func (vm *VM) VerifySpiderConfig(sc *be.SpiderConfig) (*be.SpiderConfigVerifyResult, error) {
-	qu.Debug("sc---", *sc)
-	verifyResult := list.New()
-	ret := &be.SpiderConfigVerifyResult{false, true, false, true, true, true, false}
-	_, baseCancelFn, _, _, ctx, incCancelFn := be.NewBrowser(false, false, "") //列表页使用
-	defer func() {
-		incCancelFn()
-		baseCancelFn()
-	}()
-
-	listRunJs, contentRunJs := sc.ListJSCode, sc.ContentJSCode
-	//TODO 2. 执行JS代码,获取列表页信息
-	if be.RegSpace.ReplaceAllString(listRunJs, "") == "" {
-		listRunJs = renderJavascriptCoder(loadListItemsJS, sc)
-	}
-	if be.RegSpace.ReplaceAllString(contentRunJs, "") == "" {
-		contentRunJs = renderJavascriptCoder(loadContentJS, sc)
-	}
-	qu.Debug("列表页JS:", listRunJs)
-	//TODO 3.打开列表,获取条目清单
-	chromedp.Run(ctx, chromedp.Tasks{
-		chromedp.Navigate(sc.Href),
-		chromedp.WaitReady("document.body", chromedp.ByJSPath),
-		//chromedp.Sleep(1000 * time.Millisecond),
-		chromedp.Sleep(time.Duration(sc.ListDelayTime) * time.Millisecond),
-	})
-	no := 1
-T:
-	for j := 0; j < 2; j++ { //最多检查2页
-		qu.Debug("开始检查第" + fmt.Sprint(j+1) + "页...")
-		listResult := make(be.ResultItems, 0)
-		err := chromedp.Run(ctx, chromedp.Tasks{
-			chromedp.Evaluate(listRunJs, &listResult),
-		})
-		if err != nil {
-			qu.Debug("执行列表页JS代码失败", err.Error())
-			continue
-		}
-		//TODO 5.操作详情页
-		qu.Debug("列表采集条数:", len(listResult))
-		for contentIndex, r := range listResult {
-			if contentIndex > 1 { //每页只校验2条
-				break
-			}
-			qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条")
-			r.Site = sc.Site
-			r.Channel = sc.Channel
-			qu.Debug(r.Title, r.ListTitle)
-			if r.Title == "" {
-				r.Title = r.ListTitle
-			}
-			qu.Debug(r.PublishTime, r.ListPubTime)
-			if r.PublishTime == "" {
-				r.PublishTime = r.ListPubTime
-			}
-			r.No = no
-			no += 1
-			//结果放入缓存
-			verifyResult.PushBack(r)
-		}
-		qu.Debug("列表采集条数结果:", verifyResult.Len())
-		//TODO 6.翻页
-		if verifyResult.Len() > 0 {
-			if sc.MaxPages == 1 { //最大页为1,不校验翻页
-				ret.ListTrunPage = true
-				break
-			} else if sc.MaxPages > 1 { // && !ret.ListTrunPage {
-				if err = trunPage(sc, sc.ListTurnDelayTime, ctx); err != nil { //翻页失败
-					qu.Debug("翻页失败:", err)
-					break T
-				} else {
-					ret.ListTrunPage = true
-				}
-			}
-		}
-	}
-	//检查
-	for el := verifyResult.Front(); el != nil; el = el.Next() {
-		r, _ := el.Value.(*be.ResultItem)
-		ret.Title = r.Title != ""
-		qu.Debug("Check Title:", ret.Title, r.Title, r.ListTitle)
-		ret.PublishTime = r.PublishTime != ""
-		qu.Debug("Check PublishTime:", ret.PublishTime, r.PublishTime, r.ListPubTime)
-	}
-	if ret.ListItems {
-		ret.ListItems = (sc.MaxPages == 1 && verifyResult.Len() > 0) || (sc.MaxPages > 1 && verifyResult.Len() > 2)
-	}
-
-	return ret, nil
-}

+ 4 - 4
backend/vm/single.go

@@ -99,7 +99,7 @@ func (vm *VM) RunSpiderTmp(url string, maxPages int, listDealy, trunPageDelay, c
 	qu.Debug("6采集测试完成")
 	qu.Debug("6采集测试完成")
 }
 }
 
 
-// RunSpider
+// RunSpider 适用于测试1页数据
 func (vm *VM) RunSpider(url string, maxPages int, listDealy int64, contentDelay int64, headless bool, showImage bool, proxyServe string, exit chan bool, cssMark map[string]interface{}) {
 func (vm *VM) RunSpider(url string, maxPages int, listDealy int64, contentDelay int64, headless bool, showImage bool, proxyServe string, exit chan bool, cssMark map[string]interface{}) {
 	sc, err := be.NewSpiderConfig(cssMark)
 	sc, err := be.NewSpiderConfig(cssMark)
 	if err != nil {
 	if err != nil {
@@ -133,7 +133,7 @@ func (vm *VM) RunSpider(url string, maxPages int, listDealy int64, contentDelay
 	if be.RegSpace.ReplaceAllString(runJs, "") == "" {
 	if be.RegSpace.ReplaceAllString(runJs, "") == "" {
 		runJs = renderJavascriptCoder(loadListItemsJS, sc)
 		runJs = renderJavascriptCoder(loadListItemsJS, sc)
 	}
 	}
-	qu.Debug("execute list jscode", runJs)
+	qu.Debug("获取列表JS代码:", runJs)
 	err = chromedp.Run(ctx, chromedp.Tasks{
 	err = chromedp.Run(ctx, chromedp.Tasks{
 		chromedp.Evaluate(runJs, &listResult),
 		chromedp.Evaluate(runJs, &listResult),
 	})
 	})
@@ -152,14 +152,14 @@ func (vm *VM) RunSpider(url string, maxPages int, listDealy int64, contentDelay
 	}
 	}
 	currentResult := list.New()
 	currentResult := list.New()
 	be.DataResults[sc.Code] = currentResult
 	be.DataResults[sc.Code] = currentResult
-	qu.Debug("execute content js", runJs)
+	qu.Debug("详情页JS代码:", runJs)
 	no := 1
 	no := 1
 	for _, v := range listResult {
 	for _, v := range listResult {
 		select {
 		select {
 		case <-exit:
 		case <-exit:
 			return
 			return
 		default:
 		default:
-			qu.Debug(v.No, v.ListTitle, v.Href)
+			qu.Debug(v.No, v.Href, v.ListTitle, v.ListPubTime)
 			vm.dnf.Dispatch("debug_event", fmt.Sprintf("4. %d- 待 下载详情页 %s ", v.No, v.ListTitle))
 			vm.dnf.Dispatch("debug_event", fmt.Sprintf("4. %d- 待 下载详情页 %s ", v.No, v.ListTitle))
 			var result string = ""
 			var result string = ""
 			err = chromedp.Run(ctx, chromedp.Tasks{
 			err = chromedp.Run(ctx, chromedp.Tasks{

+ 8 - 4
backend/vm/worker.go

@@ -24,7 +24,8 @@ func (w *Worker) Destory() {
 // NewWorker
 // NewWorker
 func NewWorker(headless bool, showImage bool, proxyServe string, contentDelay int64, js string, vm *VM) *Worker {
 func NewWorker(headless bool, showImage bool, proxyServe string, contentDelay int64, js string, vm *VM) *Worker {
 	_, baseCancel, _, _, ctx, cancel := be.NewBrowser(headless, showImage, proxyServe)
 	_, baseCancel, _, _, ctx, cancel := be.NewBrowser(headless, showImage, proxyServe)
-	return &Worker{baseCancel: baseCancel,
+	return &Worker{
+		baseCancel:   baseCancel,
 		incCancel:    cancel,
 		incCancel:    cancel,
 		ctx:          ctx,
 		ctx:          ctx,
 		js:           js,
 		js:           js,
@@ -92,7 +93,8 @@ func (vm *VM) RunSpiderMulThreads(url string, maxPages int, listDealy int64, tru
 	if be.RegSpace.ReplaceAllString(runContentJs, "") == "" {
 	if be.RegSpace.ReplaceAllString(runContentJs, "") == "" {
 		runContentJs = renderJavascriptCoder(loadContentJS, sc)
 		runContentJs = renderJavascriptCoder(loadContentJS, sc)
 	}
 	}
-	qu.Debug("获取列表JS代码", runListJs)
+	qu.Debug("获取列表页JS代码:", runListJs)
+	qu.Debug("获取详情页JS代码:", runContentJs)
 	wts := make([]*Worker, threads)
 	wts := make([]*Worker, threads)
 	ch := make(chan *Worker, threads)
 	ch := make(chan *Worker, threads)
 	wg := new(sync.WaitGroup)
 	wg := new(sync.WaitGroup)
@@ -110,7 +112,7 @@ func (vm *VM) RunSpiderMulThreads(url string, maxPages int, listDealy int64, tru
 		}
 		}
 	}()
 	}()
 
 
-	no := 1
+	no := 0
 	//TODO 1.翻页操作,需要在外层打开列表页
 	//TODO 1.翻页操作,需要在外层打开列表页
 	chromedp.Run(ctx, chromedp.Tasks{
 	chromedp.Run(ctx, chromedp.Tasks{
 		chromedp.Navigate(sc.Href),
 		chromedp.Navigate(sc.Href),
@@ -136,6 +138,7 @@ func (vm *VM) RunSpiderMulThreads(url string, maxPages int, listDealy int64, tru
 		qu.Debug("3获取列表完成")
 		qu.Debug("3获取列表完成")
 
 
 		//TODO 3. 打开详情页 ,支持多线程
 		//TODO 3. 打开详情页 ,支持多线程
+		qu.Debug("开始下载"+fmt.Sprint(i+1)+"页详情数据,共", len(listResult), "条")
 		for _, v := range listResult {
 		for _, v := range listResult {
 			select {
 			select {
 			case <-exit:
 			case <-exit:
@@ -152,7 +155,8 @@ func (vm *VM) RunSpiderMulThreads(url string, maxPages int, listDealy int64, tru
 			}
 			}
 		}
 		}
 		wg.Wait()
 		wg.Wait()
-		vm.dnf.Dispatch("debug_event", "4 当前页采集完成,准备执行翻页逻辑//"+sc.ListNextPageCss)
+		vm.dnf.Dispatch("debug_event", "4 当前页采集完成,准备执行翻页逻辑")
+		//翻页
 		if err = trunPage(sc, trunPageDelay, ctx); err != nil {
 		if err = trunPage(sc, trunPageDelay, ctx); err != nil {
 			qu.Debug("翻页失败", err.Error())
 			qu.Debug("翻页失败", err.Error())
 			vm.dnf.Dispatch("debug_event", "6 翻页失败: "+err.Error())
 			vm.dnf.Dispatch("debug_event", "6 翻页失败: "+err.Error())

+ 7 - 6
bind4spider.go

@@ -15,12 +15,12 @@ func (a *App) DebugSpider(url string, proxyServe string, maxPages int, listDealy
 	exitCh = make(chan bool, 1)
 	exitCh = make(chan bool, 1)
 	qu.Debug(url, proxyServe, maxPages, listDealy, trunPageDelay, contentDelay, headless, showImage, threads)
 	qu.Debug(url, proxyServe, maxPages, listDealy, trunPageDelay, contentDelay, headless, showImage, threads)
 	qu.Debug("cssMark---", cssMark)
 	qu.Debug("cssMark---", cssMark)
-	vm.RunSpiderTmp(url, maxPages, listDealy, trunPageDelay, contentDelay, headless, showImage, proxyServe, exitCh, cssMark)
-	//if maxPages == 1 && threads == 1 {
-	//	vm.RunSpider(url, maxPages, listDealy, contentDelay, headless, showImage, proxyServe, exitCh, cssMark)
-	//} else { //多页下载强制使用多线程模式
-	//	vm.RunSpiderMulThreads(url, maxPages, listDealy, trunPageDelay, contentDelay, headless, showImage, proxyServe, threads, exitCh, cssMark)
-	//}
+	//vm.RunSpiderTmp(url, maxPages, listDealy, trunPageDelay, contentDelay, headless, showImage, proxyServe, exitCh, cssMark)
+	if maxPages == 1 && threads == 1 {
+		vm.RunSpider(url, maxPages, listDealy, contentDelay, headless, showImage, proxyServe, exitCh, cssMark)
+	} else { //多页下载强制使用多线程模式
+		vm.RunSpiderMulThreads(url, maxPages, listDealy, trunPageDelay, contentDelay, headless, showImage, proxyServe, threads, exitCh, cssMark)
+	}
 }
 }
 
 
 // VerifySpiderConfig 验证
 // VerifySpiderConfig 验证
@@ -79,6 +79,7 @@ func (a *App) ViewResultItemAll(code string) be.ResultItems {
 			ret = append(ret, v)
 			ret = append(ret, v)
 		}
 		}
 	}
 	}
+	qu.Debug(len(ret))
 	return ret
 	return ret
 }
 }
 
 

+ 2 - 3
frontend/src/components/spider/EditSpider.vue

@@ -129,13 +129,12 @@
                         </el-col>
                         </el-col>
                     </el-row>
                     </el-row>
                     <el-row>
                     <el-row>
-                        <el-col :span="24">
+                        <el-col :span="12">
                             <el-form-item label="详情页附件">
                             <el-form-item label="详情页附件">
                                 <el-input v-model="formData.attachCss" placeholder="span" :style="{ background: cssInputBg.attachCss.color }"></el-input>
                                 <el-input v-model="formData.attachCss" placeholder="span" :style="{ background: cssInputBg.attachCss.color }"></el-input>
                                 <div class="form-item-sub-line">{{ fastKeyDownMap.attachCss }}</div>
                                 <div class="form-item-sub-line">{{ fastKeyDownMap.attachCss }}</div>
                             </el-form-item>
                             </el-form-item>
                         </el-col>
                         </el-col>
-
                     </el-row>
                     </el-row>
                 </el-form>
                 </el-form>
             </el-tab-pane>
             </el-tab-pane>
@@ -189,7 +188,7 @@
                 <el-divider>
                 <el-divider>
                     手写附件下载/上传JS代码
                     手写附件下载/上传JS代码
                     <!-- <el-button type="primary" @click='editorHandle.ImportContentCode'>导入模板</el-button> -->
                     <!-- <el-button type="primary" @click='editorHandle.ImportContentCode'>导入模板</el-button> -->
-                    <el-button type="primary" @click='editorHandle.createImportContentCode'>生成样例</el-button>
+                    <el-button type="primary" @click='editorHandle.createImportContentCode'>生成JS代码</el-button>
                 </el-divider>
                 </el-divider>
                 <el-row><el-input v-model="formData.contentJs" class="codeEditor" :rows="6" type="textarea"
                 <el-row><el-input v-model="formData.contentJs" class="codeEditor" :rows="6" type="textarea"
                         placeholder="Please input" />
                         placeholder="Please input" />

+ 2 - 1
frontend/src/components/spider/RunSpider.vue

@@ -89,7 +89,7 @@
         <el-divider />
         <el-divider />
         <el-table :data="tableData" style="width: 100%" :height="tableHeight" @row-click="handleRowClick">
         <el-table :data="tableData" style="width: 100%" :height="tableHeight" @row-click="handleRowClick">
             <el-table-column prop="no" label="序号" width="90" />
             <el-table-column prop="no" label="序号" width="90" />
-            <el-table-column prop="title" label="标题" width="240" show-overflow-tooltip />
+            <el-table-column prop="listTitle" label="标题" width="240" show-overflow-tooltip />
             <el-table-column prop="href" label="链接" show-overflow-tooltip />
             <el-table-column prop="href" label="链接" show-overflow-tooltip />
             <el-table-column prop="listPublishTime" label="发布时间" show-overflow-tooltip />
             <el-table-column prop="listPublishTime" label="发布时间" show-overflow-tooltip />
             <el-table-column prop="contentShort" label="正文" show-overflow-tooltip />
             <el-table-column prop="contentShort" label="正文" show-overflow-tooltip />
@@ -204,6 +204,7 @@ const truncateString = (str, maxLength) => {
 const handleRefersh = () => {
 const handleRefersh = () => {
     ViewResultItemAll(formData.value.code).then(result => {
     ViewResultItemAll(formData.value.code).then(result => {
         //result = result.slice(-20);
         //result = result.slice(-20);
+      console.log("---------",result)
         result.forEach((v, i) => {
         result.forEach((v, i) => {
             v.contentShort = truncateString(v.content, 50)
             v.contentShort = truncateString(v.content, 50)
         })
         })

+ 64 - 5
frontend/src/components/spider/VerifySpider.vue

@@ -1,8 +1,9 @@
 <!--爬虫配置自动校验-->
 <!--爬虫配置自动校验-->
 <!-- 新增爬虫 -->
 <!-- 新增爬虫 -->
 <template>
 <template>
-    <el-dialog title="爬虫配置验证结果" v-model="dialogVisible" width="50%">
-        <el-descriptions title="验证结果" direction="vertical" :column="4" border>
+    <el-dialog title="爬虫配置验证结果" v-model="dialogVisible" width="70%">
+        <el-divider content-position="center">验证结果</el-divider>
+        <el-descriptions direction="vertical" :column="4" border>
             <el-descriptions-item :class-name="formData.title ? 'c-success' : 'c-danger'" label="标题">{{ formatText(formData.title) }}</el-descriptions-item>
             <el-descriptions-item :class-name="formData.title ? 'c-success' : 'c-danger'" label="标题">{{ formatText(formData.title) }}</el-descriptions-item>
             <el-descriptions-item :class-name="formData.publishUnit ? 'c-success' : 'c-danger'" label="发布单位">{{ formatText(formData.publishUnit) }}</el-descriptions-item>
             <el-descriptions-item :class-name="formData.publishUnit ? 'c-success' : 'c-danger'" label="发布单位">{{ formatText(formData.publishUnit) }}</el-descriptions-item>
             <el-descriptions-item :class-name="formData.publishTime ? 'c-success' : 'c-danger'" label="发布时间">{{ formatText(formData.publishTime) }}</el-descriptions-item>
             <el-descriptions-item :class-name="formData.publishTime ? 'c-success' : 'c-danger'" label="发布时间">{{ formatText(formData.publishTime) }}</el-descriptions-item>
@@ -11,10 +12,30 @@
             <el-descriptions-item :class-name="formData.listItems ? 'c-success' : 'c-danger'" label="列表信息条数">{{ formatText(formData.listItems) }}</el-descriptions-item>
             <el-descriptions-item :class-name="formData.listItems ? 'c-success' : 'c-danger'" label="列表信息条数">{{ formatText(formData.listItems) }}</el-descriptions-item>
             <el-descriptions-item :class-name="formData.listTrunPage ? 'c-success' : 'c-danger'" label="列表翻页">{{ formatText(formData.listTrunPage) }}</el-descriptions-item>
             <el-descriptions-item :class-name="formData.listTrunPage ? 'c-success' : 'c-danger'" label="列表翻页">{{ formatText(formData.listTrunPage) }}</el-descriptions-item>
         </el-descriptions>
         </el-descriptions>
+        <el-divider content-position="center">样例数据</el-divider>
+        <el-table :data="tableData" style="width: 100%" height="180" @row-click="handleRowClick">
+            <el-table-column prop="no" label="序号" width="60" />
+            <el-table-column prop="listTitle" label="标题" width="240" show-overflow-tooltip />
+            <el-table-column prop="href" label="链接" show-overflow-tooltip />
+            <el-table-column prop="listPublishTime" label="发布时间" show-overflow-tooltip />
+            <el-table-column prop="contentShort" label="正文" show-overflow-tooltip />
+        </el-table>
+<!--        <template #footer>-->
+<!--            <div class="dialog-footer">-->
+<!--                <el-button @click="dialogVisible = false" type="primary">关闭</el-button>-->
+<!--            </div>-->
+<!--        </template>-->
+        <ViewArticle ref="articleDialog" />
     </el-dialog>
     </el-dialog>
 </template>
 </template>
 <script setup>
 <script setup>
-import { ref, defineExpose } from 'vue';
+import { ref, watch, defineExpose } from 'vue';
+import { ViewResultItemAll} from "../../../wailsjs/go/main/App"
+import ViewArticle from "./ViewArticle.vue";
+const tableData = ref([])
+const articleDialog = ref(null)
+
+let spiderInfo = {}
 const formData = ref({
 const formData = ref({
     title: false,
     title: false,
     publishUnit: false,
     publishUnit: false,
@@ -24,17 +45,55 @@ const formData = ref({
     listItems: false,
     listItems: false,
     listTrunPage: false,
     listTrunPage: false,
 });
 });
+
 const dialogVisible = ref(false)
 const dialogVisible = ref(false)
 
 
+const setPageData = (show = false, info) => {
+  dialogVisible.value = show
+  if (info) {
+    formData.value = info.ret || {}
+    spiderInfo = info.row
+  }
+}
+const replaceAll = function (src, search, replacement) {
+  return src.split(search).join(replacement);
+};
+//行点击事件
+const handleRowClick = (row, column, event) => {
+  articleDialog.value.dialogVisible = true
+  row.content = replaceAll(row.content, '\n', '<br/>')
+  articleDialog.value.formData = row
+  articleDialog.value.scrollTop()
+}
+
 const formatText = (f = false) => {
 const formatText = (f = false) => {
     return f ? '通过' : '未通过'
     return f ? '通过' : '未通过'
 }
 }
 
 
+const truncateString = (str, maxLength) => {
+  return str.substring(0, maxLength) + "..";
+}
+
+const loadResultData = () => {
+  ViewResultItemAll(spiderInfo.code).then(result => {
+    result.forEach((v, i) => {
+      v.contentShort = truncateString(v.content, 50)
+    })
+    tableData.value = result
+  }).catch(err => {
+    console.log(err)
+  })
+}
+
+watch(dialogVisible, (v) => {
+  if (v) {
+    loadResultData()
+  }
+})
 
 
 //这里是重点
 //这里是重点
 defineExpose({
 defineExpose({
-    dialogVisible,
-    formData
+  setPageData
 })
 })
 </script>
 </script>
 <style lang="scss" scoped>
 <style lang="scss" scoped>

+ 4 - 2
frontend/src/views/CodeList.vue

@@ -641,8 +641,10 @@ const tableEvents = {
         const mark = getMarkWithRow(row)
         const mark = getMarkWithRow(row)
         VerifySpiderConfig(mark).then(r => {
         VerifySpiderConfig(mark).then(r => {
             if (r.err === 1 && r.ret) {
             if (r.err === 1 && r.ret) {
-                verifySpiderDialog.value.dialogVisible = true
-                verifySpiderDialog.value.formData = r.ret
+                verifySpiderDialog.value.setPageData(true, {
+                  ret: r.ret,
+                  row
+                })
             } else {
             } else {
                 return ElMessage({
                 return ElMessage({
                     message: r.msg || '验证异常',
                     message: r.msg || '验证异常',

+ 4 - 2
frontend/src/views/ReviewList.vue

@@ -647,8 +647,10 @@ const tableEvents = {
         const mark = getMarkWithRow(row)
         const mark = getMarkWithRow(row)
         VerifySpiderConfig(mark).then(r => {
         VerifySpiderConfig(mark).then(r => {
             if (r.err === 1 && r.ret) {
             if (r.err === 1 && r.ret) {
-                verifySpiderDialog.value.dialogVisible = true
-                verifySpiderDialog.value.formData = r.ret
+                verifySpiderDialog.value.setPageData(true, {
+                  ret: r.ret,
+                  row
+                })
             } else {
             } else {
                 return ElMessage({
                 return ElMessage({
                     message: r.msg || '验证异常',
                     message: r.msg || '验证异常',

+ 4 - 3
server.go

@@ -12,8 +12,9 @@ import (
 	"time"
 	"time"
 )
 )
 
 
-// const HREF = "http://127.0.0.1:8091/%s"
-const HREF = "http://visualize.spdata.jianyu360.com/%s"
+const HREF = "http://127.0.0.1:8091/%s"
+
+//const HREF = "http://visualize.spdata.jianyu360.com/%s"
 
 
 type Result struct {
 type Result struct {
 	Msg  string `json:"msg"`
 	Msg  string `json:"msg"`
@@ -100,7 +101,7 @@ func (a *App) ServerActionUpdateCodeState(param map[string]interface{}) *Result
 			} else if !vr.ListItems || !vr.Content || !vr.Title || !vr.PublishTime || !vr.ListTrunPage { //校验检验清单必通过项
 			} else if !vr.ListItems || !vr.Content || !vr.Title || !vr.PublishTime || !vr.ListTrunPage { //校验检验清单必通过项
 				r.Msg = "验证清单未通过!"
 				r.Msg = "验证清单未通过!"
 			} else {
 			} else {
-				//p["verify"] = vr
+				p["verify"] = vr
 				be.VerifyResults[code] = nil //清空验证结果
 				be.VerifyResults[code] = nil //清空验证结果
 			}
 			}
 		}
 		}