browser初始化

.gitignore

@@ -8,4 +8,6 @@ bin
 */bin
 .idea
 .db
-
+go.sum
+spider.dat
+spider_attaches/

backend/browser.go

@@ -4,13 +4,14 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/chromedp/cdproto/fetch"
+	"github.com/chromedp/cdproto/cdp"
 	"github.com/chromedp/cdproto/network"
 	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"strings"
-	"time"
+
+	"github.com/chromedp/cdproto/fetch"
 
 	"github.com/chromedp/cdproto/page"
 
@@ -28,7 +29,7 @@ var (
 		"Mozilla/5.0(Macintosh;IntelMacOSX10.6;rv:2.0.1)Gecko/20100101Firefox/4.0.1",
 		"Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1",
 		"Mozilla/5.0(Macintosh;IntelMacOSX10_7_0)AppleWebKit/535.11(KHTML,likeGecko)Chrome/17.0.963.56Safari/535.11",
-		"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;SE2.XMetaSr1.0;SE2.XMetaSr1.0;.NETCLR2.0.50727;SE2.XMetaSr1.0)",
+		//"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;SE2.XMetaSr1.0;SE2.XMetaSr1.0;.NETCLR2.0.50727;SE2.XMetaSr1.0)",
 		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.70 Safari/537.36",
 		"Chrome 9 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
 		"Safari Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15",
@@ -42,20 +43,20 @@ var (
 		"Safari Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
 		"Chrome 8 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36",
 		"Chrome Mozilla/5.0 (X11; U; U; Linux x86_64; zh-my) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36 Puffin/8.3.1.41624AP",
-		"Opera 28 Mozilla/5.0 (Linux; BRAVIA 4K 2015 Build/LMY48E.S265) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36 OPR/28.0.1754.0",
+		//"Opera 28 Mozilla/5.0 (Linux; BRAVIA 4K 2015 Build/LMY48E.S265) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36 OPR/28.0.1754.0",
 		"Safari Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 HeyTapBrowser/40.7.29.1",
 		"Chrome 9 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.58 Safari/537.36 Edg/93.0.961.33",
 		"Chrome 9 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/15.0 Chrome/90.0.4430.210 Safari/537.36",
 		"Chrome 9 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36",
 		"Chrome Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
 		"Microsoft Edge Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134",
-		"Chrome 8 Mozilla/5.0 (Windows NT 10.0; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
-		"Chrome 8 Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
+		//"Chrome 8 Mozilla/5.0 (Windows NT 10.0; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
+		//"Chrome 8 Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
 		"Chrome 9 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36",
-		"Chrome 8 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36",
+		//"Chrome 8 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36",
 		"Chrome 9 Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
 		"Chrome Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
-		"Firefox 7 Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0",
+		//"Firefox 7 Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0",
 		"Chrome 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
 		"Internet Explorer 11 Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; LCJB; rv:11.0) like Gecko",
 		"Chrome 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36",
@@ -63,7 +64,7 @@ var (
 		"Chrome Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3870.400 QQBrowser/10.8.4405.400",
 		"Chrome 58 Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0",
 		"Firefox 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0",
-		"Chrome 8 Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
+		//"Chrome 8 Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
 		"Chrome 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Edg/94.0.992.38",
 	}
 )
@@ -126,28 +127,26 @@ func NewBrowser(headless bool, showImage bool, proxyServe bool, baseUrl string)
 	// 创建一个浏览器实例
 	incCtx, incCancelFn := chromedp.NewContext(allocCtx,
 		chromedp.WithLogf(nil))
-	//TODO 设置浏览器网络加载超时
-	// 监听网络请求并设置特定资源的超时
-	chromedp.ListenTarget(ctx, func(ev interface{}) {
-		switch ev := ev.(type) {
-		case *network.EventRequestWillBeSent:
-			// 设置超时时间
-			timeout := time.Duration(Cfg.BrowserLoadResourceTimeout) * time.Second
-			// TODO 这里要检查哪些资源进行超时中断监测，默认仅资源类请求
-			// 配置太麻烦，先全局，所有请求类型，这个chromedp的超时上下文不一个概念
-			// 设置一个定时器，当超时后取消请求
+	trie := NewTrie()
+	//TODO 这里默认构建通用的资源加载排除，最好是单个网站可以定制，
+	// 对于纯后端渲染网站，可以屏蔽所有资源加载，达到平台最高性能目的
+	trie.BatchInsert(Cfg.DisableLoadResource)
+	chromedp.ListenTarget(incCtx, func(event interface{}) {
+		switch ev := event.(type) {
+		case *fetch.EventRequestPaused:
 			go func() {
-				select {
-				case <-time.After(timeout):
-					fid := fetch.RequestID(ev.RequestID)
-					fetch.FailRequest(fid, network.ErrorReasonTimedOut).Do(ctx)
+				c := chromedp.FromContext(incCtx)
+				_ctx := cdp.WithExecutor(incCtx, c.Target)
+				if trie.HasKeyword(ev.Request.URL) {
+					fetch.FailRequest(ev.RequestID, network.ErrorReasonBlockedByClient).Do(_ctx)
+				} else {
+					fetch.ContinueRequest(ev.RequestID).Do(_ctx)
 				}
 			}()
 		}
 	})
-
 	//
-	chromedp.Run(ctx,
+	chromedp.Run(incCtx,
 		fetch.Enable(),
 		chromedp.ActionFunc(func(cxt context.Context) error {
 			_, err := page.AddScriptToEvaluateOnNewDocument("Object.defineProperty(navigator, 'webdriver', { get: () => false, });").Do(cxt)

backend/config.go

@@ -8,14 +8,15 @@ import (
 
 // 配置
 type Config struct {
+	DisableLoadResource             string `yaml:"disableLoadResource"`
 	IsOnly4MainSite                 bool   `yaml:"isOnly4MainSite"`            //仅仅使用重点网站数据
 	BrowserLoadResourceTimeout      int64  `yaml:"browserLoadResourceTimeout"` //浏览器加载资源超时设定、
 	ServerCodeTimeOut               int64  `yaml:"timeout"`
 	ServerCodeAddress               string `yaml:"address"`
 	ServerCodeFreeAddressOcr        string `yaml:"freeaddressocr"`
 	ServerCodeFreeAddressArithmetic string `yaml:"freeaddressarithmetic"`
-	Username                        string `yaml:"username"`
-	Password                        string `yaml:"password"`
+	ServerCodeUsername              string `yaml:"username"`
+	ServerCodePassword              string `yaml:"password"`
 }
 
 var (

backend/config.yaml

@@ -9,10 +9,4 @@
 ## 默认浏览器检查
 #default-browser-check: false
 browserLoadResourceTimeout: 5
-#验证码解析
-timeout: 15
-address: "http://pycaptcha.spdata.jianyu360.com/v1/images/discern?pic_type="
-freeaddressocr: "http://pycaptcha.spdata.jianyu360.com/v1/images/verify"
-freeaddressarithmetic: "http://pycaptcha.spdata.jianyu360.com/v1/images/arithmetic"
-username: "jianyu001"
-password: "123qwe!A"
+disableLoadResource : "ws://;wss://;.tof;.woff;.ico;.mp4;.zip;.rar;.exe;"

backend/prefixtree.go

@@ -0,0 +1,112 @@
+package backend
+
+import (
+	"strings"
+	"unicode/utf8"
+)
+
+// TrieNode represents each node in the Trie
+type TrieNode struct {
+	children map[rune]*TrieNode
+	isEnd    bool
+}
+
+// NewTrieNode creates a new TrieNode
+func NewTrieNode() *TrieNode {
+	return &TrieNode{
+		children: make(map[rune]*TrieNode),
+		isEnd:    false,
+	}
+}
+
+// Trie represents the Trie data structure
+type Trie struct {
+	root *TrieNode
+}
+
+// NewTrie creates a new Trie
+func NewTrie() *Trie {
+	return &Trie{root: NewTrieNode()}
+}
+
+// Insert inserts a word into the Trie
+func (t *Trie) Insert(word string) {
+	node := t.root
+	for _, char := range word {
+		if _, ok := node.children[char]; !ok {
+			node.children[char] = NewTrieNode()
+		}
+		node = node.children[char]
+	}
+	node.isEnd = true
+}
+
+// SearchInText searches for words in the text that are present in the Trie
+func (t *Trie) SearchInText(text string) []string {
+	var foundWords []string
+	node := t.root
+	var wordBuilder []rune
+
+	for i, w := 0, 0; i < len(text); i += w {
+		runeValue, width := utf8.DecodeRuneInString(text[i:])
+		w = width
+
+		if runeValue == utf8.RuneError {
+			continue
+		}
+
+		if _, ok := node.children[runeValue]; !ok {
+			node = t.root
+			wordBuilder = wordBuilder[:0]
+			continue
+		}
+
+		node = node.children[runeValue]
+		wordBuilder = append(wordBuilder, runeValue)
+
+		if node.isEnd {
+			foundWords = append(foundWords, string(wordBuilder))
+			// Continue to search for other words starting from this node
+			node = t.root
+			wordBuilder = wordBuilder[:0]
+		}
+	}
+
+	return foundWords
+}
+
+// SearchInText searches for words in the text that are present in the Trie
+func (t *Trie) HasKeyword(text string) bool {
+	node := t.root
+	var wordBuilder []rune
+
+	for i, w := 0, 0; i < len(text); i += w {
+		runeValue, width := utf8.DecodeRuneInString(text[i:])
+		w = width
+		if runeValue == utf8.RuneError {
+			continue
+		}
+
+		if _, ok := node.children[runeValue]; !ok {
+			node = t.root
+			wordBuilder = wordBuilder[:0]
+			continue
+		}
+
+		node = node.children[runeValue]
+		wordBuilder = append(wordBuilder, runeValue)
+
+		if node.isEnd {
+			return true
+		}
+	}
+
+	return false
+}
+
+// BatchInsert
+func (t *Trie) BatchInsert(words string) {
+	for _, s := range strings.Split(words, ";") {
+		t.Insert(s)
+	}
+}

backend/script/script.go

@@ -617,8 +617,8 @@ func getCode(b []byte, stype string, free bool) (code string, err error) {
 	if !free {
 		data := map[string]string{
 			"grant_type":     "",
-			"username":       be.Cfg.Username,
-			"password":       be.Cfg.Password,
+			"username":       be.Cfg.ServerCodeUsername,
+			"password":       be.Cfg.ServerCodePassword,
 			"scope":          "",
 			"client_id":      "",
 			"client_secret ": "",

frontend/src/components/spider/EditSpider.vue

@@ -28,27 +28,27 @@
       </el-space>
     </div>
     <div class="space" />
-    <el-form ref="form0" label-width="160px">
+    <el-form ref="form0" label-width="115px">
       <el-row>
-        <el-col :span="12">
-          <el-form-item label="列表页延迟时间(MS)">
+        <el-col :span="5">
+          <el-form-item label="列表页延迟(MS)">
             <el-input v-model="formData.listDelayTime" placeholder="1000"></el-input>
           </el-form-item>
         </el-col>
-        <el-col :span="12">
-          <el-form-item label="列表翻页延迟时间(MS)">
+        <el-col :span="5">
+          <el-form-item label="翻页延迟(MS)">
             <el-input v-model="formData.listTurnDelayTime" placeholder="1000"></el-input>
           </el-form-item>
         </el-col>
       </el-row>
       <el-row>
-        <el-col :span="12">
-          <el-form-item label="详情页延迟时间(MS)">
+        <el-col :span="5">
+          <el-form-item label="详情页延迟(MS)">
             <el-input v-model="formData.contentDelayTime" placeholder="1000"></el-input>
           </el-form-item>
         </el-col>
-        <el-col :span="12">
-          <el-form-item label="采集最大页数">
+        <el-col :span="5">
+          <el-form-item label="采集最大页">
             <el-input v-model="formData.maxPages" :placeholder="defaultFormValue.maxPages + ''"></el-input>
           </el-form-item>
         </el-col>

frontend/src/components/spider/jscodetpl.js

@@ -184,8 +184,14 @@ if ("{{.ContentCss}}" != "") {//正文内容
   }
 }
 if("{{.AttachCss}}"!=""){//附件
-	tmp = document.querySelectorAll("{{.AttachCss}} a")
 	let attach=[]
+	//先处理正文附件
+	if("{{.AttachCss}}"!="{{.ContentCss}}"){
+		document.querySelectorAll("{{.ContentCss}} a").forEach(v=>{
+			attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
+		})
+	}
+	tmp = document.querySelectorAll("{{.AttachCss}} a")
 	if(tmp){
 		tmp.forEach((v,i)=>{
 			attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
@@ -240,8 +246,14 @@ if ("{{.ContentCss}}" != "") {//正文内容
     }
 }
 if("{{.AttachCss}}"!=""){//附件
-	tmp = document.querySelectorAll("{{.AttachCss}} a")  
 	let attach=[]
+	//先处理正文附件
+	if("{{.AttachCss}}"!="{{.ContentCss}}"){
+		document.querySelectorAll("{{.ContentCss}} a").forEach(v=>{
+			attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
+		})
+	}
+	tmp = document.querySelectorAll("{{.AttachCss}} a")  
 	if(tmp){
 		tmp.forEach((v,i)=>{
 			attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
@@ -301,8 +313,14 @@ ret
         }
     }
     if("{{.AttachCss}}"!=""){//附件
+		let attach=[]
+		//先处理正文附件
+		if("{{.AttachCss}}"!="{{.ContentCss}}"){
+			document.querySelectorAll("{{.ContentCss}} a").forEach(v=>{
+				attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
+			})
+		}
         tmp = document.querySelectorAll("{{.AttachCss}} a")
-        let attach=[]
         if(tmp){
             tmp.forEach((v,i)=>{
                 attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
@@ -370,8 +388,14 @@ if ("{{.ContentCss}}" != "") {//正文内容
   }
 }
 if("{{.AttachCss}}"!=""){//附件
-	tmp = document.querySelectorAll("{{.AttachCss}} a")
 	let attach=[]
+	//先处理正文附件
+	if("{{.AttachCss}}"!="{{.ContentCss}}"){
+		document.querySelectorAll("{{.ContentCss}} a").forEach(v=>{
+			attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
+		})
+	}
+	tmp = document.querySelectorAll("{{.AttachCss}} a")
 	if(tmp){
 		tmp.forEach((v,i)=>{
 			attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
@@ -431,8 +455,14 @@ if ("{{.ContentCss}}" != "") {//正文内容
     }
 }
 if("{{.AttachCss}}"!=""){//附件
+	let attach=[]
+	//先处理正文附件
+	if("{{.AttachCss}}"!="{{.ContentCss}}"){
+		document.querySelectorAll("{{.ContentCss}} a").forEach(v=>{
+			attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
+		})
+	}
     tmp = document.querySelectorAll("{{.AttachCss}} a")
-    let attach=[]
     if(tmp){
         tmp.forEach((v,i)=>{
             attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
@@ -446,72 +476,7 @@ let chineseCharacters = ret["content"]?ret["content"].match(regex):[];
 let chineseCharactersLen=chineseCharacters ? chineseCharacters.length : 0;
 if (chineseCharactersLen < 20 && ret["attachLinks"] && ret["attachLinks"].length>0) ret["content"] = '详情请访问原网页！'
 ret 
-`}, {
-            "name": "模版6",
-            "tooltip": "基础模板扩展，正文、附件区域，都需要找附件",
-            "code": `
-var ret = {}
-var tmp = null
-
-if ("{{.TitleCss}}" != "") {//标题
-    tmp = document.querySelector("{{.TitleCss}}")
-    if (tmp) ret["title"] = tmp.getAttribute("title") || tmp.innerText
-}
-if ("{{.PublishUnitCss}}" != "") {//采购单位
-    tmp = document.querySelector("{{.PublishUnitCss}}")
-    if (tmp) ret["publishUnit"] = tmp.getAttribute("title") || tmp.innerText
-}
-if ("{{.PublishTimeCss}}" != "") {//发布时间
-    tmp = document.querySelector("{{.PublishTimeCss}}")
-    if (tmp) ret["publishTime"] = tmp.getAttribute("title") || tmp.innerText
-}
-if ("{{.ContentCss}}" != "") {//正文内容
-    tmp = document.querySelector("{{.ContentCss}}")
-    if (tmp) {
-        ret["content"] = tmp.innerText
-        ret["contentHtml"] = tmp.innerHTML
-        var patchContent = false
-        //处理详情页中的大图，大图作为附件使用
-        const images = tmp.querySelectorAll("img");
-        images.forEach((img, i) => {
-            if (img.width > 300) {
-                patchContent = true
-                const a = document.createElement("a");
-                a.href = img.src;
-                a.innerText = img.src;
-                tmp.appendChild(a);
-            }
-        })
-    }
-}
-if("{{.AttachCss}}"!=""){//附件
-    //附件区域检查
-    tmp = document.querySelectorAll("{{.AttachCss}} a")
-    let attach=[]
-    if(tmp){
-    tmp.forEach((v,i)=>{
-        attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
-    })
-}
-//正文区域附件检查
-tmp = document.querySelectorAll("{{.ContentCss}} a")
-if(tmp){
-    tmp.forEach((v,i)=>{
-        attach.push({title:v.getAttribute("title")||v.innerText,href:v.href})
-    })
-}
-//附件过滤
-attch = attach.filter((item)=>item.href && item.href.startsWith('http'))
-ret["attachLinks"]=attach
-}
-//检查中文字符个数，少于20，修正正文内容
-let regex = /[\\u4e00-\\u9fa5]/g;
-let chineseCharacters = ret["content"]?ret["content"].match(regex):[];
-let chineseCharactersLen=chineseCharacters ? chineseCharacters.length : 0;
-if (chineseCharactersLen < 20 && ret["attachLinks"] && ret["attachLinks"].length>0) ret["content"] = '详情请访问原网页！'
-ret 
-`,
-        },
+`},
     ],
 
     AttachJsCode: `

frontend/src/data/filters.js

@@ -108,6 +108,10 @@ export const rollbackReasonList = [
     label: '栏目拆分',
     value: '栏目拆分',
   },
+{
+    label: '栏目无数据',
+    value: '栏目无数据',
+  },
   {
     label: '验证码反爬',
     value: '验证码反爬',

main.go

@@ -3,15 +3,17 @@ package main
 import (
 	"container/list"
 	"embed"
-	"github.com/wailsapp/wails/v2"
-	"github.com/wailsapp/wails/v2/pkg/options"
-	"github.com/wailsapp/wails/v2/pkg/options/assetserver"
-	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+
 	be "spider_creator/backend"
 	bdb "spider_creator/backend/db"
 	"spider_creator/backend/script"
 	bvm "spider_creator/backend/vm"
 	bws "spider_creator/backend/webservice"
+
+	"github.com/wailsapp/wails/v2"
+	"github.com/wailsapp/wails/v2/pkg/options"
+	"github.com/wailsapp/wails/v2/pkg/options/assetserver"
+	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 )
 
 var (
@@ -26,20 +28,59 @@ var (
 	vm                   *bvm.VM
 	glvm                 *script.GLVm
 	ws                   *bws.WebService
-	isOnly4MainSite      = "false"
-	serverAddress        = "http://visualizeld.spdata.jianyu360.com/%s" //正式环境
-	//serverAddress = "http://127.0.0.1:8091/%s" //正式环境
+
+	browserLoadResourceTimeout int64 = 5
+
+	//重点网站和正式环境
+	isOnly4MainSite             = "false"
+	browserDisableLoadResources = "ws://;wss://;.tof;.woff;.ico;.mp4;.zip;.rar;.exe;"
+	//serverAddress               = "http://visualizeld.spdata.jianyu360.com/%s" //正式环境
+	serverAddress = "http://127.0.0.1:8091/%s" //正式环境
+
+	//验证码服务
+	serverCodeTimeOut               int64 = 15
+	serverCodeAddress                     = "http://pycaptcha.spdata.jianyu360.com/v1/images/discern?pic_type="
+	serverCodeFreeAddressOcr              = "http://pycaptcha.spdata.jianyu360.com/v1/images/verify"
+	serverCodeFreeAddressArithmetic       = "http://pycaptcha.spdata.jianyu360.com/v1/images/arithmetic"
+	serverCodeUsername                    = "jianyu001"
+	serverCodePassword                    = "123qwe!A"
 )
 
 //build
 // wails build -ldflags="-X 'main.isOnly4MainSite=false'" -o="剑鱼可视化爬虫开发工具_正式.exe"
 
 func init() {
-	be.LoadConfig("backend/config.yaml")
+	//be.LoadConfig("./config.yaml")
+	//从配置文件外，编译时设定浏览器排除加载资源。比配置文件优先级低
+	if browserDisableLoadResources != "" {
+		be.Cfg.DisableLoadResource = browserDisableLoadResources
+	}
+	if browserLoadResourceTimeout > 0 {
+		be.Cfg.BrowserLoadResourceTimeout = browserLoadResourceTimeout
+	}
+	//验证码
+	if serverCodeTimeOut > 0 {
+		be.Cfg.ServerCodeTimeOut = serverCodeTimeOut
+	}
+	if serverCodeFreeAddressOcr != "" {
+		be.Cfg.ServerCodeFreeAddressOcr = serverCodeFreeAddressOcr
+	}
+	if serverCodeFreeAddressArithmetic != "" {
+		be.Cfg.ServerCodeFreeAddressArithmetic = serverCodeFreeAddressArithmetic
+	}
+	if serverCodeUsername != "" {
+		be.Cfg.ServerCodeUsername = serverCodeUsername
+	}
+	if serverCodePassword != "" {
+		be.Cfg.ServerCodePassword = serverCodePassword
+	}
+
+	//客户端版本
 	be.Cfg.IsOnly4MainSite = isOnly4MainSite == "true"
 	if be.Cfg.IsOnly4MainSite {
 		serverAddress = "http://visualize.spdata.jianyu360.com/%s" //重点网站
 	}
+
 	qu.Debug("重点网站：", be.Cfg.IsOnly4MainSite, serverAddress)
 }