编辑页面调整

backend/types.go

@@ -59,7 +59,7 @@ type (
 		Channel     string        `json:"channel"`
 		Href        string        `json:"href"`
 		ListTitle   string        `json:"listTitle"`
-		ListPubTime string        `json:"listPubishTime"`
+		ListPubTime string        `json:"listPublishTime"`
 		Title       string        `json:"title"`
 		PublishUnit string        `json:"publishUnit"`
 		PublishTime string        `json:"publishTime"`

backend/vm/check.go

@@ -2,6 +2,7 @@ package vm
 
 import (
 	"container/list"
+	"fmt"
 	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	be "spider_creator/backend"
 	"time"
@@ -10,15 +11,133 @@ import (
 )
 
 // VerifySpiderConfig 验证爬虫配置，支持翻页，列表项数据只提取2条
+//func (vm *VM) VerifySpiderConfig(sc *be.SpiderConfig) (*be.SpiderConfigVerifyResult, error) {
+//	qu.Debug("sc---", *sc)
+//	verifyResult := list.New()
+//	ret := &be.SpiderConfigVerifyResult{true, true, true, true, true, true, true}
+//	_, baseCancelFn, _, _, ctx, incCancelFn := be.NewBrowser(false, false, "")    //列表页使用
+//	_, baseCancelFn2, _, _, ctx2, incCancelFn2 := be.NewBrowser(false, false, "") //详情页使用
+//	defer func() {
+//		incCancelFn2()
+//		baseCancelFn2()
+//		incCancelFn()
+//		baseCancelFn()
+//	}()
+//
+//	listRunJs, contentRunJs := sc.ListJSCode, sc.ContentJSCode
+//	//TODO 2. 执行JS代码，获取列表页信息
+//	if listRunJs == "" {
+//		listRunJs = renderJavascriptCoder(loadListItemsJS, sc)
+//	}
+//	if contentRunJs == "" {
+//		contentRunJs = renderJavascriptCoder(loadContentJS, sc)
+//	}
+//	qu.Debug("列表页JS:", listRunJs)
+//	qu.Debug("详情页JS:", contentRunJs)
+//	//TODO 3.打开列表，获取条目清单
+//	chromedp.Run(ctx, chromedp.Tasks{
+//		chromedp.Navigate(sc.Href),
+//		chromedp.WaitReady("document.body", chromedp.ByJSPath),
+//		//chromedp.Sleep(1000 * time.Millisecond),
+//		chromedp.Sleep(time.Duration(sc.ListDelayTime) * time.Millisecond),
+//	})
+//	no := 1
+//T:
+//	for j := 0; j < 2; j++ { //最多检查2页
+//		qu.Debug("开始检查第" + fmt.Sprint(j+1) + "页...")
+//		listResult := make(be.ResultItems, 0)
+//		err := chromedp.Run(ctx, chromedp.Tasks{
+//			chromedp.Evaluate(listRunJs, &listResult),
+//		})
+//		if err != nil {
+//			qu.Debug("执行列表页JS代码失败", err.Error())
+//			continue
+//		}
+//		//TODO 5.操作详情页
+//		qu.Debug("列表采集条数：", len(listResult))
+//		for contentIndex, r := range listResult {
+//			qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条")
+//			if contentIndex > 1 { //每页只采集2条
+//				break
+//			}
+//			//打开详情页
+//			err = chromedp.Run(ctx2, chromedp.Tasks{
+//				chromedp.Navigate(r.Href),
+//				chromedp.WaitReady("document.body", chromedp.ByJSPath),
+//				chromedp.Sleep(2000 * time.Millisecond),
+//				//chromedp.Sleep(time.Duration(sc.ContentDelayTime) * time.Millisecond),
+//			})
+//			if err != nil {
+//				qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条详情页打开异常")
+//				continue
+//			}
+//			//获取详情页内容
+//			err = chromedp.Run(ctx2, chromedp.Tasks{
+//				chromedp.Evaluate(contentRunJs, r),
+//			})
+//			if err != nil {
+//				qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条详情页内容获取失败")
+//				continue
+//			}
+//			if sc.AttachCss != "" {
+//				downloadAttaches(r, vm.attachesDir)
+//			}
+//			r.Site = sc.Site
+//			r.Channel = sc.Channel
+//			if r.Title == "" {
+//				r.Title = r.ListTitle
+//			}
+//			if r.PublishTime == "" {
+//				r.PublishTime = r.ListPubTime
+//			}
+//			r.No = no
+//			no += 1
+//			//结果放入缓存
+//			verifyResult.PushBack(r)
+//		}
+//		qu.Debug("列表采集条数结果：", verifyResult.Len())
+//		//TODO 6.翻页
+//		if err = trunPage(sc, 2000, ctx); err != nil {
+//			//if err = trunPage(sc, sc.ListTurnDelayTime, ctx); err != nil {
+//			ret.ListTrunPage = false
+//			break T
+//		}
+//	}
+//	//检查
+//	for el := verifyResult.Front(); el != nil; el = el.Next() {
+//		r, _ := el.Value.(*be.ResultItem)
+//		if ret.Title {
+//			ret.Title = r.Title != ""
+//		}
+//		if ret.PublishUnit {
+//			ret.PublishUnit = r.PublishUnit != ""
+//		}
+//		if ret.PublishTime {
+//			ret.PublishTime = r.PublishTime != ""
+//		}
+//		if ret.Content {
+//			ret.Content = r.Content != ""
+//		}
+//		if ret.Attaches {
+//			ret.Attaches = len(r.AttachLinks) > 0
+//		}
+//	}
+//	qu.Debug(verifyResult.Len())
+//	if ret.ListItems {
+//		ret.ListItems = verifyResult.Len() > 2
+//	}
+//
+//	//TODO:每次验证结果存库、内存？
+//	return ret, nil
+//}
+
+// VerifySpiderConfig 只验证列表标注
 func (vm *VM) VerifySpiderConfig(sc *be.SpiderConfig) (*be.SpiderConfigVerifyResult, error) {
 	qu.Debug("sc---", *sc)
 	verifyResult := list.New()
-	ret := &be.SpiderConfigVerifyResult{true, true, true, true, true, true, true}
-	_, baseCancelFn, _, _, ctx, incCancelFn := be.NewBrowser(false, false, "")    //列表页使用
-	_, baseCancelFn2, _, _, ctx2, incCancelFn2 := be.NewBrowser(false, false, "") //详情页使用
+	ret := &be.SpiderConfigVerifyResult{false, true, false, true, true, true, false}
+	_, baseCancelFn, _, _, ctx, incCancelFn := be.NewBrowser(false, false, "") //列表页使用
 	defer func() {
-		incCancelFn2()
-		baseCancelFn2()
 		incCancelFn()
 		baseCancelFn()
 	}()
@@ -31,6 +150,8 @@ func (vm *VM) VerifySpiderConfig(sc *be.SpiderConfig) (*be.SpiderConfigVerifyRes
 	if contentRunJs == "" {
 		contentRunJs = renderJavascriptCoder(loadContentJS, sc)
 	}
+	qu.Debug("列表页JS:", listRunJs)
+	qu.Debug("详情页JS:", contentRunJs)
 	//TODO 3.打开列表，获取条目清单
 	chromedp.Run(ctx, chromedp.Tasks{
 		chromedp.Navigate(sc.Href),
@@ -41,44 +162,29 @@ func (vm *VM) VerifySpiderConfig(sc *be.SpiderConfig) (*be.SpiderConfigVerifyRes
 	no := 1
 T:
 	for j := 0; j < 2; j++ { //最多检查2页
+		qu.Debug("开始检查第" + fmt.Sprint(j+1) + "页...")
 		listResult := make(be.ResultItems, 0)
 		err := chromedp.Run(ctx, chromedp.Tasks{
 			chromedp.Evaluate(listRunJs, &listResult),
 		})
 		if err != nil {
-			qu.Debug("执行JS代码失败", err.Error())
+			qu.Debug("执行列表页JS代码失败", err.Error())
 			continue
 		}
 		//TODO 5.操作详情页
+		qu.Debug("列表采集条数：", len(listResult))
 		for contentIndex, r := range listResult {
 			if contentIndex > 1 { //每页只采集2条
 				break
 			}
-			//打开详情页
-			err = chromedp.Run(ctx2, chromedp.Tasks{
-				chromedp.Navigate(r.Href),
-				chromedp.WaitReady("document.body", chromedp.ByJSPath),
-				//chromedp.Sleep(1000 * time.Millisecond),
-				chromedp.Sleep(time.Duration(sc.ContentDelayTime) * time.Millisecond),
-			})
-			if err != nil {
-				continue
-			}
-			//获取详情页内容
-			err = chromedp.Run(ctx2, chromedp.Tasks{
-				chromedp.Evaluate(contentRunJs, r),
-			})
-			if err != nil {
-				continue
-			}
-			if sc.AttachCss != "" {
-				downloadAttaches(r, vm.attachesDir)
-			}
+			qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条")
 			r.Site = sc.Site
 			r.Channel = sc.Channel
+			qu.Debug(r.Title, r.ListTitle)
 			if r.Title == "" {
 				r.Title = r.ListTitle
 			}
+			qu.Debug(r.PublishTime, r.ListPubTime)
 			if r.PublishTime == "" {
 				r.PublishTime = r.ListPubTime
 			}
@@ -87,38 +193,28 @@ T:
 			//结果放入缓存
 			verifyResult.PushBack(r)
 		}
-
+		qu.Debug("列表采集条数结果：", verifyResult.Len())
 		//TODO 6.翻页
-		//if err = trunPage(sc, 2000, ctx); err != nil {
-		if err = trunPage(sc, sc.ListTurnDelayTime, ctx); err != nil {
-			ret.ListTrunPage = false
-			break T
+		if len(listResult) > 0 && !ret.ListTrunPage {
+			if err = trunPage(sc, 2000, ctx); err != nil { //翻页失败
+				break T
+			} else {
+				ret.ListTrunPage = true
+			}
 		}
 	}
 	//检查
 	for el := verifyResult.Front(); el != nil; el = el.Next() {
 		r, _ := el.Value.(*be.ResultItem)
-		if ret.Title {
-			ret.Title = r.Title != ""
-		}
-		if ret.PublishUnit {
-			ret.PublishUnit = r.PublishUnit != ""
-		}
-		if ret.PublishTime {
-			ret.PublishTime = r.PublishTime != ""
-		}
-		if ret.Content {
-			ret.Content = r.Content != ""
-		}
-		if ret.Attaches {
-			ret.Attaches = len(r.AttachLinks) > 0
-		}
+		qu.Debug("Check Title:", ret.Title, r.Title, r.ListTitle)
+		ret.Title = r.Title != ""
+		qu.Debug("Check PublishTime:", ret.PublishTime, r.PublishTime, r.ListPubTime)
+		ret.PublishTime = r.PublishTime != ""
 	}
 	qu.Debug(verifyResult.Len())
 	if ret.ListItems {
 		ret.ListItems = verifyResult.Len() > 2
 	}
 
-	//TODO:每次验证结果存库、内存？
 	return ret, nil
 }

backend/vm/load_list_items.js

@@ -15,7 +15,7 @@ document.querySelectorAll("{{.ListItemCss}}").forEach((v, i) => {
     if ("{{.ListPubtimeCss}}" != "") {
         let pubtime = v.querySelector("{{.ListPubtimeCss}}")
         if (pubtime) {
-            item["listPubishTime"] = pubtime.innerText
+            item["listPublishTime"] = pubtime.innerText
         }
     }
     ret.push(item)

backend/vm/single.go

@@ -19,6 +19,85 @@ func NewVM(attachesDir string, dnf be.EventNotifyFace) *VM {
 	}
 }
 
+func (vm *VM) RunSpiderTmp(url string, maxPages int, listDealy, trunPageDelay, contentDelay int64, headless bool, showImage bool, proxyServe string, exit chan bool, cssMark map[string]interface{}) {
+	sc, err := be.NewSpiderConfig(cssMark)
+	if err != nil {
+		qu.Debug("标注信息传输失败！")
+		vm.dnf.Dispatch("debug_event", "标注信息传输失败！")
+		return
+	}
+	if url != "" {
+		sc.Href = url
+	}
+	_, baseCancel, _, _, ctx, cancel := be.NewBrowser(headless, showImage, proxyServe)
+	qu.Debug("1浏览器打开", *sc)
+	vm.dnf.Dispatch("debug_event", "1 浏览器打开")
+	defer func() {
+		cancel()
+		baseCancel()
+		qu.Debug("0浏览器已经销毁")
+		vm.dnf.Dispatch("debug_event", "0 浏览器已经销毁")
+		close(exit)
+	}()
+	chromedp.Run(ctx, chromedp.Tasks{
+		chromedp.Navigate(sc.Href),                                  //打开页面
+		chromedp.WaitReady("document.body", chromedp.ByJSPath),      //等待body加载完毕
+		chromedp.Sleep(time.Duration(listDealy) * time.Millisecond), //列表页等待
+	})
+	vm.dnf.Dispatch("debug_event", "2 页面已经打开")
+	qu.Debug("2页面打开")
+	var runJs string = sc.ListJSCode
+
+	//TODO 2. 执行JS代码，获取列表页信息
+	if runJs == "" {
+		runJs = renderJavascriptCoder(loadListItemsJS, sc)
+	}
+	qu.Debug("列表页执行JS:", runJs)
+	currentResult := list.New()
+	be.DataResults[sc.Code] = currentResult
+	no := 1
+	for j := 0; j < maxPages; j++ {
+		qu.Debug("开始检查第" + fmt.Sprint(j+1) + "页...")
+		listResult := make(be.ResultItems, 0)
+		err = chromedp.Run(ctx, chromedp.Tasks{
+			chromedp.Evaluate(runJs, &listResult),
+		})
+		if err != nil {
+			qu.Debug("执行JS代码失败", err.Error())
+			vm.dnf.Dispatch("debug_event", "2 第"+fmt.Sprint(j+1)+"页执行JS代码失败")
+			continue
+		}
+		qu.Debug("第"+fmt.Sprint(j+1)+"页列表采集条数：", len(listResult))
+		vm.dnf.Dispatch("debug_event", "3 第"+fmt.Sprint(j+1)+"获取列表完成")
+		for _, v := range listResult {
+			select {
+			case <-exit:
+				return
+			default:
+				v.Site = sc.Site
+				v.Channel = sc.Channel
+				v.Title = v.ListTitle
+				v.Content = "详见正文"
+				v.No = no
+				qu.Debug(v.No, v.ListTitle, v.Href)
+				no++
+				currentResult.PushBack(v)
+			}
+		}
+		vm.dnf.Dispatch("debug_event", "4 第"+fmt.Sprint(j+1)+"页采集完成，准备执行翻页")
+		if j < maxPages-1 {
+			if err = trunPage(sc, trunPageDelay, ctx); err != nil {
+				qu.Debug("翻页失败", err.Error())
+				vm.dnf.Dispatch("debug_event", "5 第"+fmt.Sprint(j+1)+"页翻页失败")
+				time.Sleep(3 * time.Second)
+				break
+			}
+		}
+	}
+	vm.dnf.Dispatch("debug_event", "6 采集测试完成")
+	qu.Debug("6采集测试完成")
+}
+
 // RunSpider
 func (vm *VM) RunSpider(url string, maxPages int, listDealy int64, contentDelay int64, headless bool, showImage bool, proxyServe string, exit chan bool, cssMark map[string]interface{}) {
 	sc, err := be.NewSpiderConfig(cssMark)
@@ -73,6 +152,7 @@ func (vm *VM) RunSpider(url string, maxPages int, listDealy int64, contentDelay
 	currentResult := list.New()
 	be.DataResults[sc.Code] = currentResult
 	qu.Debug("execute content js", runJs)
+	no := 1
 	for _, v := range listResult {
 		select {
 		case <-exit:
@@ -87,6 +167,8 @@ func (vm *VM) RunSpider(url string, maxPages int, listDealy int64, contentDelay
 				chromedp.Sleep(time.Duration(contentDelay) * time.Millisecond),
 				chromedp.Evaluate(runJs, v),
 			})
+			v.No = no
+			no++
 			v.Site = sc.Site
 			v.Channel = sc.Channel
 			if err != nil {

backend/vm/vm.go

@@ -132,7 +132,8 @@ func trunPage(sc *be.SpiderConfig, delay int64, ctx context.Context) error {
 	//TODO 1. 获取当前列表当前页的内容快照，以便与翻页后的结果对比
 	var result1, result2 string
 	var checkRunJs = fmt.Sprintf(`document.querySelector("%s").outerText`, sc.ListBodyCss)
-	qu.Debug("检查翻页是否成功，执行的JS", checkRunJs)
+	qu.Debug("获取当前页内容，执行的JS", checkRunJs)
+	//获取当前页内容
 	err := chromedp.Run(ctx, chromedp.Tasks{
 		chromedp.Evaluate(checkRunJs, &result1),
 	})
@@ -141,7 +142,8 @@ func trunPage(sc *be.SpiderConfig, delay int64, ctx context.Context) error {
 		return err
 	}
 	qu.Debug("第一页：", result1)
-	qu.Debug("runJs：", runJs, delay)
+	qu.Debug("执行翻页JS：", runJs, delay)
+	//执行翻页
 	if runJs != "" {
 		//可能就没有分页
 		err = chromedp.Run(ctx, chromedp.Tasks{
@@ -156,6 +158,7 @@ func trunPage(sc *be.SpiderConfig, delay int64, ctx context.Context) error {
 		return errors.New("trun page error ")
 	}
 	qu.Debug("--------------------------")
+	//获取翻页后内容
 	err = chromedp.Run(ctx, chromedp.Tasks{
 		chromedp.Evaluate(checkRunJs, &result2),
 	})

bind4spider.go

@@ -15,11 +15,12 @@ func (a *App) DebugSpider(url string, proxyServe string, maxPages int, listDealy
 	exitCh = make(chan bool, 1)
 	qu.Debug(url, proxyServe, maxPages, listDealy, trunPageDelay, contentDelay, headless, showImage, threads)
 	qu.Debug("cssMark---", cssMark)
-	if maxPages == 1 && threads == 1 {
-		vm.RunSpider(url, maxPages, listDealy, contentDelay, headless, showImage, proxyServe, exitCh, cssMark)
-	} else { //多页下载强制使用多线程模式
-		vm.RunSpiderMulThreads(url, maxPages, listDealy, trunPageDelay, contentDelay, headless, showImage, proxyServe, threads, exitCh, cssMark)
-	}
+	vm.RunSpiderTmp(url, maxPages, listDealy, trunPageDelay, contentDelay, headless, showImage, proxyServe, exitCh, cssMark)
+	//if maxPages == 1 && threads == 1 {
+	//	vm.RunSpider(url, maxPages, listDealy, contentDelay, headless, showImage, proxyServe, exitCh, cssMark)
+	//} else { //多页下载强制使用多线程模式
+	//	vm.RunSpiderMulThreads(url, maxPages, listDealy, trunPageDelay, contentDelay, headless, showImage, proxyServe, threads, exitCh, cssMark)
+	//}
 }
 
 // VerifySpiderConfig 验证

frontend/src/App.vue

@@ -56,6 +56,7 @@ import { computed, ref } from 'vue';
 import { useStore } from 'vuex';
 import { useRouter } from 'vue-router';
 import { KillAllChrome } from "../wailsjs/go/main/App"
+import {ElMessageBox} from "element-plus";
 
 const store = useStore()
 const router = useRouter()
@@ -81,7 +82,17 @@ const doLogout = () => {
   router.replace({ name: 'logout' })
 }
 const doKillChrome = () => {
-  KillAllChrome().then(r => { })
+  ElMessageBox.confirm('确定杀死所有chrome进程？', '提示',
+      {
+        customClass: 'j-confirm-message-box',
+        type: 'warning',
+        confirmButtonText: '确定',
+        cancelButtonText: '取消',
+        showCancelButton: false,
+      }
+  ).then(() => {
+    KillAllChrome()
+  })
 }
 
 </script>

frontend/src/components/spider/EditSpider.vue

@@ -31,7 +31,7 @@
         <el-form ref="form0" label-width="160px">
             <el-row>
                 <el-col :span="12">
-                    <el-form-item label="列表延迟时间(MS)">
+                    <el-form-item label="列表页延迟时间(MS)">
                         <el-input v-model="formData.listDelayTime" placeholder="500"></el-input>
                     </el-form-item>
                 </el-col>

frontend/src/components/spider/RunSpider.vue

@@ -91,6 +91,7 @@
             <el-table-column prop="no" label="序号" width="90" />
             <el-table-column prop="title" label="标题" width="240" show-overflow-tooltip />
             <el-table-column prop="href" label="链接" show-overflow-tooltip />
+            <el-table-column prop="listPublishTime" label="发布时间" show-overflow-tooltip />
             <el-table-column prop="contentShort" label="正文" show-overflow-tooltip />
         </el-table>
     </div>

frontend/src/store/index.js

@@ -23,18 +23,18 @@ export default createStore({
         menuConfig: {
             // 管理员菜单
             [USER_ROLE_ADMIN]: [
-                { title: '爬虫列表', icon: 'Guide', path: '/code/list' },
-                { title: '审核列表', icon: 'Setting', path: '/review/list' },
+                { title: '爬虫列表', icon: 'List', path: '/code/list' },
+                { title: '审核列表', icon: 'Checked', path: '/review/list' },
                 // { title: '系统设置', icon: 'Help', path: '/setting' },
             ],
             // 开发者菜单
             [USER_ROLE_DEVELOPER]: [
-                { title: '爬虫列表', icon: 'Guide', path: '/code/list' },
+                { title: '爬虫列表', icon: 'List', path: '/code/list' },
                 // { title: '系统设置', icon: 'Help', path: '/setting' },
             ],
             // 审核人员菜单
             [USER_ROLE_REVIEWER]: [
-                { title: '审核列表', icon: 'Setting', path: '/review/list' },
+                { title: '审核列表', icon: 'Checked', path: '/review/list' },
                 // { title: '系统设置', icon: 'Help', path: '/setting' },
             ],
         },

frontend/wailsjs/go/main/App.d.ts

@@ -24,6 +24,8 @@ export function ExportJsonFile(arg1:string,arg2:string):Promise<{[key: string]:
 
 export function ImportSpiderConfigByExcelFile(arg1:string):Promise<string>;
 
+export function KillAllChrome():Promise<string>;
+
 export function LoadAllJobs():Promise<backend.Jobs>;
 
 export function LoadJob(arg1:string):Promise<backend.Job>;

frontend/wailsjs/go/main/App.js

@@ -42,6 +42,10 @@ export function ImportSpiderConfigByExcelFile(arg1) {
   return window['go']['main']['App']['ImportSpiderConfigByExcelFile'](arg1);
 }
 
+export function KillAllChrome() {
+  return window['go']['main']['App']['KillAllChrome']();
+}
+
 export function LoadAllJobs() {
   return window['go']['main']['App']['LoadAllJobs']();
 }

server.go

@@ -14,6 +14,8 @@ import (
 
 const HREF = "http://127.0.0.1:8091/%s"
 
+//const HREF = "http://visualize.spdata.jianyu360.com/%s"
+
 type Result struct {
 	Msg  string `json:"msg"`
 	Err  int    `json:"err"`
@@ -84,8 +86,8 @@ func (a *App) ServerActionUpdateCodeState(param map[string]interface{}) *Result
 			code := qu.ObjToString(lua["code"])
 			if vr := be.VerifyResults[code]; vr == nil { //没有检验清单，不允许提交
 				data.Msg = "未验证，无法提交！"
-			} else if !vr.ListItems || !vr.Content || !vr.Title || !vr.PublishTime { //校验检验清单必通过项
-				//vr.ListTrunPage
+				//} else if !vr.ListItems || !vr.Content || !vr.Title || !vr.PublishTime { //校验检验清单必通过项
+			} else if !vr.ListItems || !vr.Content || !vr.Title || !vr.PublishTime || !vr.ListTrunPage { //校验检验清单必通过项
 				data.Msg = "验证清单未通过！"
 			} else {
 				lua["verify"] = vr

tpl/load_list_items.js

@@ -15,7 +15,7 @@ document.querySelectorAll("{{.ListItemCss}}").forEach((v, i) => {
     if ("{{.ListPubtimeCss}}" != "") {
         let pubtime = v.querySelector("{{.ListPubtimeCss}}")
         if (pubtime) {
-            item["listPubishTime"] = pubtime.innerText
+            item["listPublishTime"] = pubtime.innerText
         }
     }
     ret.push(item)