添加忽略请求

.gitignore

@@ -8,4 +8,6 @@ bin
 */bin
 .idea
 .db
-
+go.sum
+spider.dat
+spider_attaches/

backend/browser.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/chromedp/cdproto/fetch"
-	"github.com/chromedp/cdproto/network"
 	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"strings"
-	"time"
+
+	"github.com/chromedp/cdproto/cdp"
+	"github.com/chromedp/cdproto/network"
+
+	"github.com/chromedp/cdproto/fetch"
 
 	"github.com/chromedp/cdproto/page"
 
@@ -28,7 +30,7 @@ var (
 		"Mozilla/5.0(Macintosh;IntelMacOSX10.6;rv:2.0.1)Gecko/20100101Firefox/4.0.1",
 		"Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1",
 		"Mozilla/5.0(Macintosh;IntelMacOSX10_7_0)AppleWebKit/535.11(KHTML,likeGecko)Chrome/17.0.963.56Safari/535.11",
-		"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;SE2.XMetaSr1.0;SE2.XMetaSr1.0;.NETCLR2.0.50727;SE2.XMetaSr1.0)",
+		//"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;SE2.XMetaSr1.0;SE2.XMetaSr1.0;.NETCLR2.0.50727;SE2.XMetaSr1.0)",
 		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.70 Safari/537.36",
 		"Chrome 9 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
 		"Safari Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15",
@@ -42,20 +44,20 @@ var (
 		"Safari Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
 		"Chrome 8 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36",
 		"Chrome Mozilla/5.0 (X11; U; U; Linux x86_64; zh-my) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36 Puffin/8.3.1.41624AP",
-		"Opera 28 Mozilla/5.0 (Linux; BRAVIA 4K 2015 Build/LMY48E.S265) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36 OPR/28.0.1754.0",
+		//"Opera 28 Mozilla/5.0 (Linux; BRAVIA 4K 2015 Build/LMY48E.S265) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36 OPR/28.0.1754.0",
 		"Safari Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 HeyTapBrowser/40.7.29.1",
 		"Chrome 9 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.58 Safari/537.36 Edg/93.0.961.33",
 		"Chrome 9 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/15.0 Chrome/90.0.4430.210 Safari/537.36",
 		"Chrome 9 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36",
 		"Chrome Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
 		"Microsoft Edge Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134",
-		"Chrome 8 Mozilla/5.0 (Windows NT 10.0; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
-		"Chrome 8 Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
+		//"Chrome 8 Mozilla/5.0 (Windows NT 10.0; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
+		//"Chrome 8 Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
 		"Chrome 9 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36",
-		"Chrome 8 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36",
+		//"Chrome 8 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36",
 		"Chrome 9 Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
 		"Chrome Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
-		"Firefox 7 Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0",
+		//"Firefox 7 Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0",
 		"Chrome 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
 		"Internet Explorer 11 Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; LCJB; rv:11.0) like Gecko",
 		"Chrome 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36",
@@ -63,7 +65,7 @@ var (
 		"Chrome Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3870.400 QQBrowser/10.8.4405.400",
 		"Chrome 58 Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0",
 		"Firefox 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0",
-		"Chrome 8 Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
+		//"Chrome 8 Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
 		"Chrome 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Edg/94.0.992.38",
 	}
 )
@@ -126,26 +128,24 @@ func NewBrowser(headless bool, showImage bool, proxyServe bool, baseUrl string)
 	// 创建一个浏览器实例
 	incCtx, incCancelFn := chromedp.NewContext(allocCtx,
 		chromedp.WithLogf(nil))
-	//TODO 设置浏览器网络加载超时
-	// 监听网络请求并设置特定资源的超时
-	chromedp.ListenTarget(ctx, func(ev interface{}) {
-		switch ev := ev.(type) {
-		case *network.EventRequestWillBeSent:
-			// 设置超时时间
-			timeout := time.Duration(Cfg.BrowserLoadResourceTimeout) * time.Second
-			// TODO 这里要检查哪些资源进行超时中断监测，默认仅资源类请求
-			// 配置太麻烦，先全局，所有请求类型，这个chromedp的超时上下文不一个概念
-			// 设置一个定时器，当超时后取消请求
+	trie := NewTrie()
+	//TODO 这里默认构建通用的资源加载排除，最好是单个网站可以定制，
+	// 对于纯后端渲染网站，可以屏蔽所有资源加载，达到平台最高性能目的
+	trie.BatchInsert(Cfg.DisableLoadResource)
+	chromedp.ListenTarget(ctx, func(event interface{}) {
+		switch ev := event.(type) {
+		case *fetch.EventRequestPaused:
 			go func() {
-				select {
-				case <-time.After(timeout):
-					fid := fetch.RequestID(ev.RequestID)
-					fetch.FailRequest(fid, network.ErrorReasonTimedOut).Do(ctx)
+				c := chromedp.FromContext(ctx)
+				ctx := cdp.WithExecutor(ctx, c.Target)
+				if trie.HasKeyword(ev.Request.URL) {
+					fetch.FailRequest(ev.RequestID, network.ErrorReasonBlockedByClient).Do(ctx)
+				} else {
+					fetch.ContinueRequest(ev.RequestID).Do(ctx)
 				}
 			}()
 		}
 	})
-
 	//
 	chromedp.Run(ctx,
 		fetch.Enable(),

backend/config.go

@@ -0,0 +1,32 @@
+package backend
+
+import (
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+// 配置
+type Config struct {
+	IsOnly4MainSite     bool `yaml:"isOnly4MainSite"` //仅仅使用重点网站数据
+	DisableLoadResource string
+}
+
+var (
+	Cfg *Config = new(Config)
+)
+
+// 加载配置文件
+func LoadConfig(cf string) error {
+	fi, err := os.Open(cf)
+	if err != nil {
+		return err
+	}
+	defer fi.Close()
+
+	err = yaml.NewDecoder(fi).Decode(Cfg)
+	if err != nil {
+		return err
+	}
+	return nil
+}

backend/prefixtree.go

@@ -0,0 +1,112 @@
+package backend
+
+import (
+	"strings"
+	"unicode/utf8"
+)
+
+// TrieNode represents each node in the Trie
+type TrieNode struct {
+	children map[rune]*TrieNode
+	isEnd    bool
+}
+
+// NewTrieNode creates a new TrieNode
+func NewTrieNode() *TrieNode {
+	return &TrieNode{
+		children: make(map[rune]*TrieNode),
+		isEnd:    false,
+	}
+}
+
+// Trie represents the Trie data structure
+type Trie struct {
+	root *TrieNode
+}
+
+// NewTrie creates a new Trie
+func NewTrie() *Trie {
+	return &Trie{root: NewTrieNode()}
+}
+
+// Insert inserts a word into the Trie
+func (t *Trie) Insert(word string) {
+	node := t.root
+	for _, char := range word {
+		if _, ok := node.children[char]; !ok {
+			node.children[char] = NewTrieNode()
+		}
+		node = node.children[char]
+	}
+	node.isEnd = true
+}
+
+// SearchInText searches for words in the text that are present in the Trie
+func (t *Trie) SearchInText(text string) []string {
+	var foundWords []string
+	node := t.root
+	var wordBuilder []rune
+
+	for i, w := 0, 0; i < len(text); i += w {
+		runeValue, width := utf8.DecodeRuneInString(text[i:])
+		w = width
+
+		if runeValue == utf8.RuneError {
+			continue
+		}
+
+		if _, ok := node.children[runeValue]; !ok {
+			node = t.root
+			wordBuilder = wordBuilder[:0]
+			continue
+		}
+
+		node = node.children[runeValue]
+		wordBuilder = append(wordBuilder, runeValue)
+
+		if node.isEnd {
+			foundWords = append(foundWords, string(wordBuilder))
+			// Continue to search for other words starting from this node
+			node = t.root
+			wordBuilder = wordBuilder[:0]
+		}
+	}
+
+	return foundWords
+}
+
+// SearchInText searches for words in the text that are present in the Trie
+func (t *Trie) HasKeyword(text string) bool {
+	node := t.root
+	var wordBuilder []rune
+
+	for i, w := 0, 0; i < len(text); i += w {
+		runeValue, width := utf8.DecodeRuneInString(text[i:])
+		w = width
+		if runeValue == utf8.RuneError {
+			continue
+		}
+
+		if _, ok := node.children[runeValue]; !ok {
+			node = t.root
+			wordBuilder = wordBuilder[:0]
+			continue
+		}
+
+		node = node.children[runeValue]
+		wordBuilder = append(wordBuilder, runeValue)
+
+		if node.isEnd {
+			return true
+		}
+	}
+
+	return false
+}
+
+// BatchInsert
+func (t *Trie) BatchInsert(words string) {
+	for _, s := range strings.Split(words, ";") {
+		t.Insert(s)
+	}
+}

frontend/src/components/spider/CodeEditor.vue

@@ -1,5 +1,12 @@
 <template>
     <el-dialog title="编辑代码" :model-value="props.show" @update:model-value="updateModelValue" :close-on-click-modal="false" width="70%">
+        <el-button-group>
+            <el-tooltip v-for="item,index in TemplateJsCode.InitListPageJsCodes" :key="index" class="box-item" effect="dark" :content="item.tooltip"
+                        placement="top-start">
+              <el-button size="small" type="primary" @click='useInitPageJsCode(index)'>{{item.name}}</el-button>
+            </el-tooltip>
+          </el-button-group>
+        <div style="line-height: 6px;height: 6px;"></div>
         <div class="textarea-container">
             <el-input v-model="dialogInfo.text" class="textarea" autofocus :autosize="{ minRows: 12, maxRows: 13 }" type="textarea" placeholder="Please input"></el-input>
         </div>
@@ -11,6 +18,7 @@
 </template>
 <script setup>
 import { reactive } from 'vue';
+import { TemplateJsCode } from './jscodetpl.js'
 
 const emit = defineEmits(['update:show', 'save'])
 const props = defineProps({
@@ -20,6 +28,12 @@ const props = defineProps({
     }
 })
 
+//使用页面初始化模板
+const useInitPageJsCode=(index)=>{
+    let code =  TemplateJsCode.InitListPageJsCodes[index].code
+    dialogInfo.text = code
+}
+
 const dialogInfo = reactive({
     key: '',
     text: '',

frontend/src/components/spider/jscodetpl.js

@@ -446,7 +446,72 @@ let chineseCharacters = ret["content"]?ret["content"].match(regex):[];
 let chineseCharactersLen=chineseCharacters ? chineseCharacters.length : 0;
 if (chineseCharactersLen < 20 && ret["attachLinks"] && ret["attachLinks"].length>0) ret["content"] = '详情请访问原网页！'
 ret 
-`},
+`}, {
+            "name": "模版6",
+            "tooltip": "基础模板扩展，正文、附件区域，都需要找附件",
+            "code": `
+var ret = {}
+var tmp = null
+
+if ("{{.TitleCss}}" != "") {//标题
+tmp = document.querySelector("{{.TitleCss}}")
+if (tmp) ret["title"] = tmp.getAttribute("title") || tmp.innerText
+}
+if ("{{.PublishUnitCss}}" != "") {//采购单位
+tmp = document.querySelector("{{.PublishUnitCss}}")
+if (tmp) ret["publishUnit"] = tmp.getAttribute("title") || tmp.innerText
+}
+if ("{{.PublishTimeCss}}" != "") {//发布时间
+tmp = document.querySelector("{{.PublishTimeCss}}")
+if (tmp) ret["publishTime"] = tmp.getAttribute("title") || tmp.innerText
+}
+if ("{{.ContentCss}}" != "") {//正文内容
+tmp = document.querySelector("{{.ContentCss}}")
+if (tmp) {
+ret["content"] = tmp.innerText
+ret["contentHtml"] = tmp.innerHTML
+var patchContent = false
+//处理详情页中的大图，大图作为附件使用
+const images = tmp.querySelectorAll("img");
+images.forEach((img, i) => {
+if (img.width > 300) {
+patchContent = true
+const a = document.createElement("a");
+a.href = img.src;
+a.innerText = img.src;
+tmp.appendChild(a);
+}
+})
+}
+}
+if("{{.AttachCss}}"!=""){//附件
+//附件区域检查
+tmp = document.querySelectorAll("{{.AttachCss}} a")
+let attach=[]
+if(tmp){
+tmp.forEach((v,i)=>{
+    attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
+})
+}
+//正文区域附件检查
+tmp = document.querySelectorAll("{{.ContentCss}} a")
+if(tmp){
+tmp.forEach((v,i)=>{
+    attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
+})
+}
+//附件过滤
+attch = attach.filter((item)=>item.href && item.href.startsWith('http'))
+ret["attachLinks"]=attach
+}
+//检查中文字符个数，少于20，修正正文内容
+let regex = /[\\u4e00-\\u9fa5]/g;
+let chineseCharacters = ret["content"]?ret["content"].match(regex):[];
+let chineseCharactersLen=chineseCharacters ? chineseCharacters.length : 0;
+if (chineseCharactersLen < 20 && ret["attachLinks"] && ret["attachLinks"].length>0) ret["content"] = '详情请访问原网页！'
+ret 
+`,
+        },
     ],
 
     AttachJsCode: `
@@ -486,7 +551,7 @@ document.querySelectorAll("{{.ListNextPageCss}}").forEach(link=>{
             "tooltip": "选择器匹配多个 翻页链接/按钮，可根据检测是否包含子对象匹配",
             "code": `
 document.querySelectorAll("{{.ListNextPageCss}}").forEach(link=>{
-  if(link.querySelector("自对象CSS选择器，请修改"))link.click();
+  if(link.querySelector("子对象CSS选择器，请修改"))link.click();
 })
 ""
     `,
@@ -517,5 +582,41 @@ window.location=href
     `,
         },
     ],
-
+    //初始化代码集合
+    InitListPageJsCodes: [
+        {
+            "name": "模版1",
+            "tooltip": "页面跳转",
+            "code": `
+window.location=''
+''
+    `,
+        }, {
+            "name": "模版2",
+            "tooltip": "点击唯一元素",
+            "code": `
+var link=document.querySelector("CSS选择器");
+if(link)link.click();
+"" 
+    `,
+        }, {
+            "name": "模版3",
+            "tooltip": "点击需要文本过滤",
+            "code": `
+document.querySelectorAll("CSS选择器").forEach(link=>{
+  if(link.innerText==="下一页")link.click();
+})
+""
+    `,
+        }, {
+            "name": "模版4",
+            "tooltip": "点击需要子元素过滤",
+            "code": `
+document.querySelectorAll("CSS选择器").forEach(link=>{
+  if(link.querySelector("子对象CSS选择器，请修改"))link.click();
+})
+""
+    `,
+        },
+    ],
 }

frontend/src/data/filters.js

@@ -108,6 +108,10 @@ export const rollbackReasonList = [
     label: '栏目拆分',
     value: '栏目拆分',
   },
+{
+    label: '栏目无数据',
+    value: '栏目无数据',
+  },
   {
     label: '验证码反爬',
     value: '验证码反爬',

go.mod

@@ -11,9 +11,10 @@ require (
 	github.com/chromedp/chromedp v0.10.0
 	github.com/gabriel-vasile/mimetype v1.3.1
 	github.com/itcwc/go-zhipu v0.0.0-20240626065325-ffc8bf1cfaaa
-	github.com/pkg/errors v0.9.1
 	github.com/wailsapp/wails/v2 v2.9.1
 	github.com/xuri/excelize/v2 v2.8.1
+	github.com/yuin/gopher-lua v1.1.1
+	gopkg.in/yaml.v3 v3.0.1
 	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40
 )
 
@@ -44,6 +45,7 @@ require (
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/richardlehane/mscfb v1.0.4 // indirect
 	github.com/richardlehane/msoleps v1.0.3 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
@@ -65,3 +67,4 @@ require (
 )
 
 // replace github.com/wailsapp/wails/v2 v2.9.1 => /Users/taozhang/go/pkg/mod
+replace jygit.jydev.jianyu360.cn/data_processing/common_utils => ../common_utils

main.go

@@ -3,13 +3,13 @@ package main
 import (
 	"container/list"
 	"embed"
-	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	be "spider_creator/backend"
 	bdb "spider_creator/backend/db"
 	"spider_creator/backend/script"
 	bvm "spider_creator/backend/vm"
 	bws "spider_creator/backend/webservice"
-	"strconv"
+
+	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 
 	"github.com/wailsapp/wails/v2"
 	"github.com/wailsapp/wails/v2/pkg/options"
@@ -30,9 +30,9 @@ var (
 	glvm                 *script.GLVm
 	ws                   *bws.WebService
 	//重点网站和正式环境
-	isOnly4MainSite            string = "false"
-	BrowserLoadResourceTimeout        = "5"
-	serverAddress                     = "http://visualizeld.spdata.jianyu360.com/%s" //正式环境
+	isOnly4MainSite             string = "false"
+	browserDisableLoadResources        = "ws://;wss://;.tof;.woff;.ico;.mp4;.zip;.rar;.exe;"
+	serverAddress                      = "http://visualizeld.spdata.jianyu360.com/%s" //正式环境
 	//serverAddress = "http://127.0.0.1:8091/%s" //正式环境
 )
 
@@ -41,11 +41,12 @@ var (
 
 func init() {
 	//be.LoadConfig("./config.yaml")
+	be.Cfg.DisableLoadResource = browserDisableLoadResources
 	be.Cfg.IsOnly4MainSite = isOnly4MainSite == "true"
 	if be.Cfg.IsOnly4MainSite {
 		serverAddress = "http://visualize.spdata.jianyu360.com/%s" //重点网站
 	}
-	be.Cfg.BrowserLoadResourceTimeout, _ = strconv.ParseInt(BrowserLoadResourceTimeout, 10, 64)
+
 	qu.Debug("重点网站：", be.Cfg.IsOnly4MainSite, serverAddress)
 }