后台方法目录优化

ai.go

@@ -1,68 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-
-	//"log"
-	"strings"
-
-	zhipu "github.com/itcwc/go-zhipu/model_api"
-)
-
-const (
-	MODEL_NAME = "glm-4-flash" //"glm-4-air"
-)
-
-var (
-	expireAtTime = int64(1719803252) // token 过期时间
-	apiKey       = "5343038d9d934536456f281f8487866a.YUmO7HK9xNb990j9"
-)
-
-// UpdateResultDateStr
-func UpdateResultDateStr(rs ResultItems) (err error) {
-	tmp := make([]string, len(rs))
-	for i, v := range rs {
-		tmp[i] = v.ListPubTime
-	}
-
-	prompt := fmt.Sprintf(`根据我提供的内容，识别每行文本中的日期，按照YYYY-MM-DD形式输出,如：2024-01-01；找不到日期数据，输出NULL。不要联网，不要解释，不要说明，直接输出结果。
-				---------------------------
-				%s				
-				`, strings.Join(tmp, "\n"))
-	mssage := zhipu.PostParams{
-		Model: MODEL_NAME,
-		Messages: []zhipu.Message{
-			{
-				Role:    "user", // 消息的角色信息 详见文档
-				Content: prompt, // 消息内容
-			},
-		},
-	}
-	postResponse, err := zhipu.BeCommonModel(expireAtTime, mssage, apiKey)
-	if err != nil {
-		return err
-	}
-
-	//解析数据
-	choices, _ := postResponse["choices"].([]interface{})
-	obj, _ := choices[0].(map[string]interface{})
-	message, _ := obj["message"].(map[string]interface{})
-	value, _ := message["content"].(string)
-	// log.Println("提示语", prompt)
-	// log.Println("AI调用结果", value)
-	results := strings.Split(value, "\n")
-	if len(results) < len(rs) {
-		err = errors.New("调用大模型失败")
-		return
-	}
-	//更新
-	for i, v := range rs {
-		if results[i] != "NULL" {
-			v.ListPubTime = results[i]
-		} else {
-			v.ListPubTime = ""
-		}
-	}
-	return nil
-}

ai_test.go

@@ -1,29 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestAI(t *testing.T) {
-	tmp := ResultItems{
-		{ListPubTime: "2024年5月1日"},
-		{ListPubTime: "2024/8/1"},
-		{ListPubTime: "中文干扰文本"},
-		{ListPubTime: "中文干扰文本 2024-7-12"},
-	}
-	callAIState := false
-	for i := 0; i < 5; i++ {
-		err := UpdateResultDateStr(tmp)
-		if err == nil {
-			callAIState = true
-			break
-		}
-		fmt.Println("ai调用失败，再次尝试", err.Error())
-	}
-	if callAIState {
-		for _, v := range tmp {
-			fmt.Println(v.ListPubTime)
-		}
-	}
-}

app.go

@@ -1,17 +1,33 @@
 package main
 
 import (
-	"context"
 	"fmt"
+	"golang.org/x/net/context"
 	"log"
-
-	"github.com/wailsapp/wails/v2/pkg/runtime"
+	"os"
+	rt "runtime"
+	bdb "spidercreator/backend/db"
+	bvm "spidercreator/backend/vm"
+	bws "spidercreator/backend/webservice"
 )
 
-var (
-	db     *SpiderDb
-	exitCh chan bool
-)
+func getDownloadDir() (string, error) {
+	var downloadDir string
+	var err error
+
+	switch rt.GOOS {
+	case "windows":
+		downloadDir = os.Getenv("USERPROFILE") + "\\Downloads"
+	case "darwin":
+		downloadDir = os.Getenv("HOME") + "/Downloads"
+	case "linux":
+		downloadDir = os.Getenv("HOME") + "/Downloads"
+	default:
+		err = fmt.Errorf("unsupported operating system")
+	}
+
+	return downloadDir, err
+}
 
 // App struct
 type App struct {
@@ -27,7 +43,25 @@ func NewApp() *App {
 // so we can call the runtime methods
 func (a *App) startup(ctx context.Context) {
 	a.ctx = ctx
-	db = NewSpiderDb("./data.db")
+	if rt.GOOS == "darwin" {
+		baseDir = os.Getenv("HOME") + "/Downloads"
+	} else {
+		baseDir = "."
+	}
+	//创建附件下载目录
+	attachesDir = baseDir + "/spdier_attaches"
+	if _, err := os.Stat(attachesDir); err != nil {
+		os.MkdirAll(attachesDir, 0777)
+	}
+	var dbfile = baseDir + "/spider.dat"
+	log.Println("db file:", dbfile)
+
+	db = bdb.NewSpiderDb(dbfile, a)
+	bdb.Db = db
+	vm = bvm.NewVM(attachesDir, a)
+	ws = bws.NewWebService(db, a, currentSpiderConfig)
+	//
+	go ws.RunHttpServe()
 }
 
 // destory
@@ -35,118 +69,145 @@ func (a *App) destory(ctx context.Context) {
 	db.Close()
 }
 
-// Greet returns a greeting for the given name
-func (a *App) Greet(name string) string {
-	return fmt.Sprintf("Hello %s, It's show time!", name)
-}
+//var (
+//	//db     *SpiderDb
+//	exitCh chan bool
+//)
 
-// LoadSpiderConfigAll，带分页
-func (a *App) LoadSpiderConfigAll(pageSize, pageNo int) []*SpiderConfig {
-	return db.LoadAll()
-}
+// App struct
+//type App struct {
+//	ctx context.Context
+//}
 
-// LoadSpiderConfigAll，带分页
-func (a *App) SaveOrUpdateSpiderConfig(sc *SpiderConfig) string {
-	db.SaveOrUpdate(sc)
-	return "ok"
-}
+// NewApp creates a new App application struct
+//func NewApp() *App {
+//	return &App{}
+//}
+
+// startup is called when the app starts. The context is saved
+// so we can call the runtime methods
+//func (a *App) startup(ctx context.Context) {
+//	a.ctx = ctx
+//	db = NewSpiderDb("./data.db")
+//}
+
+// destory
+//func (a *App) destory(ctx context.Context) {
+//	db.Close()
+//}
+
+// Greet returns a greeting for the given name
+//func (a *App) Greet(name string) string {
+//	return fmt.Sprintf("Hello %s, It's show time!", name)
+//}
+
+//// LoadSpiderConfigAll，带分页
+//func (a *App) LoadSpiderConfigAll(pageSize, pageNo int) []*SpiderConfig {
+//	return db.LoadAll()
+//}
+//
+//// LoadSpiderConfigAll，带分页
+//func (a *App) SaveOrUpdateSpiderConfig(sc *SpiderConfig) string {
+//	db.SaveOrUpdate(sc)
+//	return "ok"
+//}
 
 // SwitchSpiderConfig
-func (a *App) SwitchSpiderConfig(code string) string {
-	log.Println("切换当前默认爬虫配置：", code)
-	db.Switch(code)
-	return "ok"
-}
+//func (a *App) SwitchSpiderConfig(code string) string {
+//	log.Println("切换当前默认爬虫配置：", code)
+//	db.Switch(code)
+//	return "ok"
+//}
 
 // SwitchSpiderConfig
-func (a *App) ViewCurrentSpiderConfig() *SpiderConfig {
-	return currentSpiderConfig
-}
+//func (a *App) ViewCurrentSpiderConfig() *SpiderConfig {
+//	return currentSpiderConfig
+//}
 
 // SwitchSpiderConfig
-func (a *App) DeleteSpiderConfig(code string) string {
-	db.Delete(code)
-	return "ok"
-}
+//func (a *App) DeleteSpiderConfig(code string) string {
+//	db.Delete(code)
+//	return "ok"
+//}
 
 // 推送消息
-func (a *App) pushMessage(event string, data interface{}) {
-	runtime.EventsEmit(a.ctx, event, data)
-}
+//func (a *App) pushMessage(event string, data interface{}) {
+//	runtime.EventsEmit(a.ctx, event, data)
+//}
 
 // 调试爬虫
-func (a *App) DebugSpider(url string, listDealy int64, contentDelay int64, headless bool, showImage bool, proxyServe string) {
-	exitCh = make(chan bool, 1)
-	RunSpider(url, listDealy, contentDelay, headless, showImage, proxyServe, exitCh)
-}
+//func (a *App) DebugSpider(url string, listDealy int64, contentDelay int64, headless bool, showImage bool, proxyServe string) {
+//	exitCh = make(chan bool, 1)
+//	RunSpider(url, listDealy, contentDelay, headless, showImage, proxyServe, exitCh)
+//}
 
 // 停止调试
-func (a *App) StopDebugSpider() string {
-	defer func() {
-		if err := recover(); err != nil {
-			log.Println(err)
-		}
-	}()
-	exitCh <- true
-	return "ok"
-}
+//func (a *App) StopDebugSpider() string {
+//	defer func() {
+//		if err := recover(); err != nil {
+//			log.Println(err)
+//		}
+//	}()
+//	exitCh <- true
+//	return "ok"
+//}
 
 // 查看所有结果
-func (a *App) ViewResultItemAll() ResultItems {
-	return currentResult
-}
-
-// ExportEpubFile
-func (a *App) ExportEpubFile(filepath string) string {
-	ExportEpubFile(filepath)
-	return "ok"
-}
+//func (a *App) ViewResultItemAll() ResultItems {
+//	return currentResult
+//}
+//
+//// ExportEpubFile
+//func (a *App) ExportEpubFile(filepath string) string {
+//	ExportEpubFile(filepath)
+//	return "ok"
+//}
 
 // SelectSaveFilePath
-func (a *App) SelectSaveFilePath() string {
-	path, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{Filters: []runtime.FileFilter{
-		{Pattern: "*.epub", DisplayName: "epub file *.epub"},
-		{Pattern: "*.xlsx", DisplayName: "excel file *.xlsx"},
-		{Pattern: "*.json", DisplayName: "json file *.json"},
-	}})
-	if err != nil {
-		log.Println(err.Error())
-		return ""
-	}
-	return path
-}
+//func (a *App) SelectSaveFilePath() string {
+//	path, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{Filters: []runtime.FileFilter{
+//		{Pattern: "*.epub", DisplayName: "epub file *.epub"},
+//		{Pattern: "*.xlsx", DisplayName: "excel file *.xlsx"},
+//		{Pattern: "*.json", DisplayName: "json file *.json"},
+//	}})
+//	if err != nil {
+//		log.Println(err.Error())
+//		return ""
+//	}
+//	return path
+//}
 
 // SelectOpenFilePath
-func (a *App) SelectOpenFilePath() string {
-	path, err := runtime.OpenFileDialog(a.ctx, runtime.OpenDialogOptions{Filters: []runtime.FileFilter{
-		{Pattern: "*.xlsx", DisplayName: "excel file *.xlsx"},
-	}})
-	if err != nil {
-		log.Println(err.Error())
-		return ""
-	}
-	return path
-}
+//func (a *App) SelectOpenFilePath() string {
+//	path, err := runtime.OpenFileDialog(a.ctx, runtime.OpenDialogOptions{Filters: []runtime.FileFilter{
+//		{Pattern: "*.xlsx", DisplayName: "excel file *.xlsx"},
+//	}})
+//	if err != nil {
+//		log.Println(err.Error())
+//		return ""
+//	}
+//	return path
+//}
 
 // ImportSpiderConfigByExcelFile 通过excel文件导入爬虫配置
-func (a *App) ImportSpiderConfigByExcelFile(filepath string) string {
-	db.BatchImport(filepath)
-	return "ok"
-}
+//func (a *App) ImportSpiderConfigByExcelFile(filepath string) string {
+//	db.BatchImport(filepath)
+//	return "ok"
+//}
 
 // 获取login状态
-func (a *App) GetLoginState() bool {
-	return loginState
-}
-
-func (a *App) PutLoginState(state bool) string {
-	loginState = state
-	return "ok"
-}
+//func (a *App) GetLoginState() bool {
+//	return loginState
+//}
+//
+//func (a *App) PutLoginState(state bool) string {
+//	loginState = state
+//	return "ok"
+//}
 
 // CountYestodayArts
-func (a *App) CountYestodayArts(url string, listDealy int64,
-	trunPageDelay int64, headless bool, showImage bool) {
-	exitCh = make(chan bool, 1)
-	CountYestodayArts(url, listDealy, trunPageDelay, headless, showImage, exitCh)
-}
+//func (a *App) CountYestodayArts(url string, listDealy int64,
+//	trunPageDelay int64, headless bool, showImage bool) {
+//	exitCh = make(chan bool, 1)
+//	CountYestodayArts(url, listDealy, trunPageDelay, headless, showImage, exitCh)
+//}

browser.go

@@ -1,116 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"math/rand"
-
-	"github.com/chromedp/cdproto/page"
-
-	"github.com/chromedp/chromedp"
-)
-
-var (
-	useragent = []string{
-		"Chrome: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36",
-		"Firefox: Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
-		"Safari: Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_5 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0 Mobile/15D60 Safari/604.1",
-		"MacOSX: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14",
-		"Mozilla/5.0(Macintosh;U;IntelMacOSX10_6_8;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50",
-		"Mozilla/5.0(Windows;U;WindowsNT6.1;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50",
-		"Mozilla/5.0(Macintosh;IntelMacOSX10.6;rv:2.0.1)Gecko/20100101Firefox/4.0.1",
-		"Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1",
-		"Mozilla/5.0(Macintosh;IntelMacOSX10_7_0)AppleWebKit/535.11(KHTML,likeGecko)Chrome/17.0.963.56Safari/535.11",
-		"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;SE2.XMetaSr1.0;SE2.XMetaSr1.0;.NETCLR2.0.50727;SE2.XMetaSr1.0)",
-		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.70 Safari/537.36",
-		"Chrome 9 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
-		"Safari Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15",
-		"Safari Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15",
-		"Safari 11 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15 QQBrowserLite/1.3.0",
-		"Chrome 9 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
-		"Chrome 59 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
-		"Chrome 9 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36",
-		"Safari 11 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/604.3.5 (KHTML, like Gecko) Version/11.0.1 Safari/604.3.5",
-		"Firefox 9 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:93.0) Gecko/20100101 Firefox/93.0",
-		"Safari Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
-		"Chrome 8 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36",
-		"Chrome Mozilla/5.0 (X11; U; U; Linux x86_64; zh-my) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36 Puffin/8.3.1.41624AP",
-		"Opera 28 Mozilla/5.0 (Linux; BRAVIA 4K 2015 Build/LMY48E.S265) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36 OPR/28.0.1754.0",
-		"Safari Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 HeyTapBrowser/40.7.29.1",
-		"Chrome 9 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.58 Safari/537.36 Edg/93.0.961.33",
-		"Chrome 9 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/15.0 Chrome/90.0.4430.210 Safari/537.36",
-		"Chrome 9 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36",
-		"Chrome Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
-		"Microsoft Edge Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134",
-		"Chrome 8 Mozilla/5.0 (Windows NT 10.0; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
-		"Chrome 8 Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
-		"Chrome 9 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36",
-		"Chrome 8 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36",
-		"Chrome 9 Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-		"Chrome Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
-		"Firefox 7 Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0",
-		"Chrome 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
-		"Internet Explorer 11 Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; LCJB; rv:11.0) like Gecko",
-		"Chrome 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36",
-		"Firefox 36  Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0",
-		"Chrome Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3870.400 QQBrowser/10.8.4405.400",
-		"Chrome 58 Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0",
-		"Firefox 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0",
-		"Chrome 8 Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
-		"Chrome 9 Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Edg/94.0.992.38",
-	}
-)
-
-func NewBrowser(headless bool, showImage bool, proxyServe string) (
-	context.Context, context.CancelFunc,
-	context.Context, context.CancelFunc,
-	context.Context, context.CancelFunc,
-) {
-	ctx, cancelFn := chromedp.NewContext(context.Background())
-
-	chromeOptions := append(chromedp.DefaultExecAllocatorOptions[:],
-		chromedp.NoDefaultBrowserCheck,                                  //不检查默认浏览器
-		chromedp.Flag("enable-automation", false),                       // 防止监测webdriver
-		chromedp.Flag("disable-blink-features", "AutomationControlled"), //禁用 blink 特征
-		chromedp.Flag("force-dev-mode-highlighting", true),
-		chromedp.Flag("disable-extensions", false), //是否禁用扩展
-		chromedp.Flag("headless", headless),
-		chromedp.Flag("user-agent", useragent[rand.Intn(20)]), //搞到底还是要在这里设置useragent
-		chromedp.Flag("disable-keep-alive", true),
-		chromedp.Flag("disable-gpu", true),
-		chromedp.Flag("no-sandbox", true),
-		chromedp.Flag("disable-dev-shm-usage", false),
-		chromedp.Flag("default-browser-check", false),
-		chromedp.Flag("ignore-certificate-errors", true), //忽略错误
-		chromedp.Flag("disable-web-security", true),      //禁用网络安全标志
-		chromedp.Flag("mute-audio", false),
-		chromedp.Flag("accept-language", `zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-TW;q=0.6`),
-	)
-	if proxyServe != "" {
-		chromeOptions = append(chromeOptions,
-			chromedp.ProxyServer(fmt.Sprintf("socks5://%s", proxyServe)),
-		)
-	}
-	if showImage {
-		chromeOptions = append(chromeOptions,
-			chromedp.Flag("blink-settings", "imagesEnabled=true"),
-		)
-	} else {
-		chromeOptions = append(chromeOptions,
-			chromedp.Flag("blink-settings", "imagesEnabled=false"),
-		)
-	}
-
-	allocCtx, allocCancelFn := chromedp.NewExecAllocator(ctx, chromeOptions...)
-	// 创建一个浏览器实例
-	incCtx, incCancelFn := chromedp.NewContext(allocCtx,
-		chromedp.WithLogf(nil))
-	//
-	chromedp.Run(ctx,
-		chromedp.ActionFunc(func(cxt context.Context) error {
-			_, err := page.AddScriptToEvaluateOnNewDocument("Object.defineProperty(navigator, 'webdriver', { get: () => false, });").Do(cxt)
-			return err
-		}),
-	)
-	return ctx, cancelFn, allocCtx, allocCancelFn, incCtx, incCancelFn
-}

cert.pem

@@ -1,18 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIC+TCCAeGgAwIBAgIQWG6URMK1Ue9VcGjsy8zypjANBgkqhkiG9w0BAQsFADAS
-MRAwDgYDVQQKEwdBY21lIENvMB4XDTI0MDkwNTIxNTM1MloXDTI1MDkwNTIxNTM1
-MlowEjEQMA4GA1UEChMHQWNtZSBDbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
-AQoCggEBAK6HcRaSQiUXu1poHgToSSyrJZoC9ry4R6m+4DRMMHjh9tV0ZD0Nikdo
-HVcgneeaIgs+3ZQJTbG7NP2IHVTX26nAIpM4TlkDEXtx+uJnNH5h0V/vtwVk0lE7
-Hv3cwOxzDbVYuIJO23EAII3Dh/BEhT1tL50xtaS6hOUDeYcVv7BRqdcpNMaNVpC3
-156N9lCzYVlEL8/W/km8M3QQa6bpOo6iyHj68VLVjsd8hQG19XctpXnz/RPfZONS
-YqM04fGddRdNtcy+TUBC/qdYxtSJwI9nPNx+K9DynyBVwQ1ppRaPOl9mPAwyxzFp
-CAkr+m3adFWNaDVsWIrQaZNWzh8DWh8CAwEAAaNLMEkwDgYDVR0PAQH/BAQDAgWg
-MBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwFAYDVR0RBA0wC4IJ
-bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQCLAfeh30waA5o5za1CUNJheAr0
-m/mwmVxTdWdxLdwOZTcGdNHnpdXKfwoPVt1ZaSbBKxj9+1haw2FFXjl01pczOuMK
-HSmDOqeJFCtovJVoHazXw+5o35fA5iJukENUmGhE3w3YI7gBaxwIRD/3dO17IkmN
-JJtqmYpFp6WsTYV50v083a8hEqX2FGF4b74l6MbVV/Q1XHBc60HL2UWIwop/V/3s
-KP0zqnSzC/eh7qfFFGFEZ0Xlumt2Pc7+96+0QflZOFrOQIIIskFo4jMqdblB3EV+
-4tx1shZdMnHT6fX6kmKaG6663qgRaMaMIW1DiWdsP+5mh7wqDyPiTmkbvaHh
------END CERTIFICATE-----

db.go

@@ -1,213 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"sort"
-
-	"log"
-
-	"github.com/boltdb/bolt"
-	"github.com/xuri/excelize/v2"
-)
-
-type (
-	//SpiderDB 爬虫库，这里模拟真实数据库
-	SpiderDb struct {
-		db *bolt.DB
-	}
-)
-
-var (
-	loginState          bool = false
-	currentSpiderConfig *SpiderConfig
-)
-
-// NewSpiderDb
-func NewSpiderDb(dbfile string) *SpiderDb {
-	db, err := bolt.Open(dbfile, 0600, nil)
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte("myBucket"))
-		return err
-	})
-	if err != nil {
-		log.Fatal(err)
-	}
-	return &SpiderDb{
-		db,
-	}
-}
-
-// Close
-func (s *SpiderDb) Close() {
-	s.db.Close()
-}
-
-// CopyAttribute
-func CopyAttribute(dst *string, value1, value2 string) {
-	if value1 != "" {
-		*dst = value1
-	} else if value2 != "" {
-		*dst = value2
-	}
-}
-
-// MergeSpiderConfig 合并
-func MergeSpiderConfig(src1, src2 *SpiderConfig) *SpiderConfig {
-	nsc := new(SpiderConfig)
-	CopyAttribute(&nsc.Code, src2.Code, src1.Code)
-	CopyAttribute(&nsc.Site, src2.Site, src1.Site)
-	CopyAttribute(&nsc.Channel, src2.Channel, src1.Channel)
-	CopyAttribute(&nsc.Url, src2.Url, src1.Url)
-	CopyAttribute(&nsc.Author, src2.Author, src1.Author)
-	CopyAttribute(&nsc.ListItemCss, src2.ListItemCss, src1.ListItemCss)
-	CopyAttribute(&nsc.ListLinkCss, src2.ListLinkCss, src1.ListLinkCss)
-	CopyAttribute(&nsc.ListPubtimeCss, src2.ListPubtimeCss, src1.ListPubtimeCss)
-	CopyAttribute(&nsc.ListNextPageCss, src2.ListNextPageCss, src1.ListNextPageCss)
-	CopyAttribute(&nsc.TitleCss, src2.TitleCss, src1.TitleCss)
-	CopyAttribute(&nsc.PublishTimeCss, src2.PublishTimeCss, src1.PublishTimeCss)
-	CopyAttribute(&nsc.PublishUnitCss, src2.PublishUnitCss, src1.PublishUnitCss)
-	CopyAttribute(&nsc.ContentCss, src2.ContentCss, src1.ContentCss)
-	CopyAttribute(&nsc.AttachCss, src2.AttachCss, src1.AttachCss)
-	CopyAttribute(&nsc.ListJSCode, src2.ListJSCode, src1.ListJSCode)
-	CopyAttribute(&nsc.ContentJSCode, src2.ContentJSCode, src1.ContentJSCode)
-	CopyAttribute(&nsc.AttachJSCode, src2.AttachJSCode, src1.AttachJSCode)
-	return nsc
-}
-
-// Load
-func (s *SpiderDb) Load(code string) *SpiderConfig {
-	var req *SpiderConfig = new(SpiderConfig)
-	err := s.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte("myBucket"))
-		value := bucket.Get([]byte(code))
-		if value != nil && len(value) > 0 {
-			_ = json.Unmarshal(value, req)
-		}
-		return nil
-	})
-	if err != nil {
-		log.Fatal(err)
-	}
-	return req
-}
-
-// SaveOrUpdate
-func (s *SpiderDb) SaveOrUpdate(sc *SpiderConfig) {
-	//加载原始数据
-	var sc1 *SpiderConfig = new(SpiderConfig)
-	var sc2 *SpiderConfig
-	err := s.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte("myBucket"))
-		value := bucket.Get([]byte(sc.Code))
-		if value != nil && len(value) > 0 {
-			_ = json.Unmarshal(value, sc1)
-		}
-		return nil
-	})
-	if err != nil {
-		log.Println(err.Error())
-		return
-	}
-	//更新
-	if sc1 != nil {
-		sc2 = MergeSpiderConfig(sc1, sc)
-		value, _ := json.Marshal(sc2)
-		err = s.db.Update(func(tx *bolt.Tx) error {
-			bucket := tx.Bucket([]byte("myBucket"))
-			err := bucket.Put([]byte(sc.Code), value)
-			return err
-		})
-		if err != nil {
-			log.Println(err.Error())
-			return
-		}
-	}
-}
-
-// LoadAll,默认按照代码排序
-func (s *SpiderDb) LoadAll() SpiderConfiges {
-	ret := make(SpiderConfiges, 0)
-	// 开始读取事务
-	err := s.db.View(func(tx *bolt.Tx) error {
-		// 遍历数据库中的所有桶
-		bucket := tx.Bucket([]byte("myBucket"))
-		// 遍历桶中的所有键/值对
-		return bucket.ForEach(func(k, v []byte) error {
-			var sf *SpiderConfig = new(SpiderConfig)
-			json.Unmarshal(v, sf)
-			if sf != nil {
-				ret = append(ret, sf)
-			}
-			return nil
-		})
-	})
-	sort.Sort(ret)
-	if err != nil {
-		log.Println(err.Error())
-	}
-	return ret
-}
-
-// 切换当前默认爬虫配置
-func (s *SpiderDb) Switch(code string) {
-	if sc := s.Load(code); sc != nil {
-		currentSpiderConfig = sc
-	}
-}
-
-// Delete
-func (s *SpiderDb) Delete(code string) {
-	err := s.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte("myBucket"))
-		err := bucket.Delete([]byte(code))
-		return err
-	})
-	if err != nil {
-		log.Println(err.Error())
-		return
-	}
-}
-
-// 批量导入
-func (s *SpiderDb) BatchImport(filepath string) error {
-	f, err := excelize.OpenFile(filepath)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	for _, sheetName := range f.GetSheetList() {
-		// 获取工作表的所有行
-		rows, err := f.GetRows(sheetName)
-		if err != nil {
-			continue
-		}
-		//
-		for index, row := range rows {
-			if index == 0 || len(row) < 5 || row[0] == "" || row[3] == "" {
-				continue
-			}
-			sc := &SpiderConfig{
-				Code:    row[0],
-				Site:    row[1],
-				Channel: row[2],
-				Url:     row[3],
-				Author:  row[4],
-			}
-			value, _ := json.Marshal(sc)
-			err = s.db.Update(func(tx *bolt.Tx) error {
-				bucket := tx.Bucket([]byte("myBucket"))
-				err := bucket.Put([]byte(sc.Code), value)
-				return err
-			})
-			if err != nil {
-				continue
-			}
-		}
-
-	}
-
-	return nil
-}

frontend/src/store/modules/rulesList.js

@@ -30,6 +30,7 @@ export default {
     // 获取爬虫列表
     async getCodeList({ commit }, payload) {
       const { pageSize, pageNum } = payload
+      console.log(payload)
       const r = await ServerActionCodeList({
         modifyuser: payload.modifyuser, // 维护人
         state: payload.state, // 爬虫状态
@@ -38,6 +39,7 @@ export default {
         start: (pageNum - 1) * pageSize,
         limit: pageSize
       })
+      console.log(r)
       return r
     },
     // 爬虫认领

frontend/wailsjs/go/main/App.d.ts

@@ -1,30 +1,40 @@
 // Cynhyrchwyd y ffeil hon yn awtomatig. PEIDIWCH Â MODIWL
 // This file is automatically generated. DO NOT EDIT
-import {main} from '../models';
+import {backend} from '../models';
 
 export function CountYestodayArts(arg1:string,arg2:number,arg3:number,arg4:boolean,arg5:boolean):Promise<void>;
 
-export function DebugSpider(arg1:string,arg2:number,arg3:number,arg4:boolean,arg5:boolean,arg6:string):Promise<void>;
+export function DebugSpider(arg1:string,arg2:number,arg3:number,arg4:number,arg5:number,arg6:boolean,arg7:boolean,arg8:string,arg9:number):Promise<void>;
+
+export function DeleteJob(arg1:string):Promise<string>;
 
 export function DeleteSpiderConfig(arg1:string):Promise<string>;
 
-export function ExportEpubFile(arg1:string):Promise<string>;
+export function Dispatch(arg1:string,arg2:any):Promise<void>;
+
+export function ExportEpubFile(arg1:string,arg2:string):Promise<string>;
 
-export function GetLoginState():Promise<boolean>;
+export function ExportExcelFile(arg1:string):Promise<string>;
 
-export function Greet(arg1:string):Promise<string>;
+export function ExportJobResult(arg1:string,arg2:string):Promise<string>;
 
 export function ImportSpiderConfigByExcelFile(arg1:string):Promise<string>;
 
-export function LoadSpiderConfigAll(arg1:number,arg2:number):Promise<Array<main.SpiderConfig>>;
+export function LoadAllJobs():Promise<backend.Jobs>;
+
+export function LoadJob(arg1:string):Promise<backend.Job>;
 
-export function PutLoginState(arg1:boolean):Promise<string>;
+export function LoadSpiderConfigAll(arg1:number,arg2:number):Promise<Array<backend.SpiderConfig>>;
 
-export function SaveOrUpdateSpiderConfig(arg1:main.SpiderConfig):Promise<string>;
+export function RunJob(arg1:string):Promise<string>;
+
+export function SaveJob(arg1:backend.Job):Promise<string>;
+
+export function SaveOrUpdateSpiderConfig(arg1:backend.SpiderConfig):Promise<string>;
 
 export function SelectOpenFilePath():Promise<string>;
 
-export function SelectSaveFilePath():Promise<string>;
+export function SelectSaveFilePath(arg1:string,arg2:string):Promise<string>;
 
 export function ServerActionCheckLogin():Promise<{[key: string]: any}>;
 
@@ -34,7 +44,7 @@ export function ServerActionCodeList(arg1:{[key: string]: any}):Promise<{[key: s
 
 export function ServerActionGetModifyUsers():Promise<{[key: string]: any}>;
 
-export function ServerActionUpdateCode(arg1:{[key: string]: any}):Promise<{[key: string]: any}>;
+export function ServerActionUpdateCode(arg1:Array<{[key: string]: any}>):Promise<{[key: string]: any}>;
 
 export function ServerActionUserLogin(arg1:{[key: string]: any}):Promise<{[key: string]: any}>;
 
@@ -42,8 +52,10 @@ export function ServerActionUserLogout():Promise<{[key: string]: any}>;
 
 export function StopDebugSpider():Promise<string>;
 
+export function StopJob(arg1:string):Promise<string>;
+
 export function SwitchSpiderConfig(arg1:string):Promise<string>;
 
-export function ViewCurrentSpiderConfig():Promise<main.SpiderConfig>;
+export function ViewCurrentSpiderConfig():Promise<backend.SpiderConfig>;
 
-export function ViewResultItemAll():Promise<main.ResultItems>;
+export function ViewResultItemAll():Promise<backend.ResultItems>;

frontend/wailsjs/go/main/App.js

@@ -6,36 +6,56 @@ export function CountYestodayArts(arg1, arg2, arg3, arg4, arg5) {
   return window['go']['main']['App']['CountYestodayArts'](arg1, arg2, arg3, arg4, arg5);
 }
 
-export function DebugSpider(arg1, arg2, arg3, arg4, arg5, arg6) {
-  return window['go']['main']['App']['DebugSpider'](arg1, arg2, arg3, arg4, arg5, arg6);
+export function DebugSpider(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9) {
+  return window['go']['main']['App']['DebugSpider'](arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9);
+}
+
+export function DeleteJob(arg1) {
+  return window['go']['main']['App']['DeleteJob'](arg1);
 }
 
 export function DeleteSpiderConfig(arg1) {
   return window['go']['main']['App']['DeleteSpiderConfig'](arg1);
 }
 
-export function ExportEpubFile(arg1) {
-  return window['go']['main']['App']['ExportEpubFile'](arg1);
+export function Dispatch(arg1, arg2) {
+  return window['go']['main']['App']['Dispatch'](arg1, arg2);
+}
+
+export function ExportEpubFile(arg1, arg2) {
+  return window['go']['main']['App']['ExportEpubFile'](arg1, arg2);
 }
 
-export function GetLoginState() {
-  return window['go']['main']['App']['GetLoginState']();
+export function ExportExcelFile(arg1) {
+  return window['go']['main']['App']['ExportExcelFile'](arg1);
 }
 
-export function Greet(arg1) {
-  return window['go']['main']['App']['Greet'](arg1);
+export function ExportJobResult(arg1, arg2) {
+  return window['go']['main']['App']['ExportJobResult'](arg1, arg2);
 }
 
 export function ImportSpiderConfigByExcelFile(arg1) {
   return window['go']['main']['App']['ImportSpiderConfigByExcelFile'](arg1);
 }
 
+export function LoadAllJobs() {
+  return window['go']['main']['App']['LoadAllJobs']();
+}
+
+export function LoadJob(arg1) {
+  return window['go']['main']['App']['LoadJob'](arg1);
+}
+
 export function LoadSpiderConfigAll(arg1, arg2) {
   return window['go']['main']['App']['LoadSpiderConfigAll'](arg1, arg2);
 }
 
-export function PutLoginState(arg1) {
-  return window['go']['main']['App']['PutLoginState'](arg1);
+export function RunJob(arg1) {
+  return window['go']['main']['App']['RunJob'](arg1);
+}
+
+export function SaveJob(arg1) {
+  return window['go']['main']['App']['SaveJob'](arg1);
 }
 
 export function SaveOrUpdateSpiderConfig(arg1) {
@@ -46,8 +66,8 @@ export function SelectOpenFilePath() {
   return window['go']['main']['App']['SelectOpenFilePath']();
 }
 
-export function SelectSaveFilePath() {
-  return window['go']['main']['App']['SelectSaveFilePath']();
+export function SelectSaveFilePath(arg1, arg2) {
+  return window['go']['main']['App']['SelectSaveFilePath'](arg1, arg2);
 }
 
 export function ServerActionCheckLogin() {
@@ -82,6 +102,10 @@ export function StopDebugSpider() {
   return window['go']['main']['App']['StopDebugSpider']();
 }
 
+export function StopJob(arg1) {
+  return window['go']['main']['App']['StopJob'](arg1);
+}
+
 export function SwitchSpiderConfig(arg1) {
   return window['go']['main']['App']['SwitchSpiderConfig'](arg1);
 }

frontend/wailsjs/go/models.ts

@@ -1,8 +1,98 @@
-export namespace main {
+export namespace backend {
 	
+	export class JobItem {
+	    code: string;
+	    site: string;
+	    channel: string;
+	    url: string;
+	    proxyServe: string;
+	    maxPages: number;
+	    threads: number;
+	    listDelay: number;
+	    trunPageDelay: number;
+	    contentDelay: number;
+	    needDownloadAttaches: boolean;
+	
+	    static createFrom(source: any = {}) {
+	        return new JobItem(source);
+	    }
+	
+	    constructor(source: any = {}) {
+	        if ('string' === typeof source) source = JSON.parse(source);
+	        this.code = source["code"];
+	        this.site = source["site"];
+	        this.channel = source["channel"];
+	        this.url = source["url"];
+	        this.proxyServe = source["proxyServe"];
+	        this.maxPages = source["maxPages"];
+	        this.threads = source["threads"];
+	        this.listDelay = source["listDelay"];
+	        this.trunPageDelay = source["trunPageDelay"];
+	        this.contentDelay = source["contentDelay"];
+	        this.needDownloadAttaches = source["needDownloadAttaches"];
+	    }
+	}
+	export class Job {
+	    code: string;
+	    name: string;
+	    items: JobItem[];
+	    proxyServe: string;
+	    maxPages: number;
+	    threads: number;
+	    listDelay: number;
+	    trunPageDelay: number;
+	    contentDelay: number;
+	    state: number;
+	    stateType: string;
+	    progress: number;
+	    needDownloadAttaches: boolean;
+	
+	    static createFrom(source: any = {}) {
+	        return new Job(source);
+	    }
+	
+	    constructor(source: any = {}) {
+	        if ('string' === typeof source) source = JSON.parse(source);
+	        this.code = source["code"];
+	        this.name = source["name"];
+	        this.items = this.convertValues(source["items"], JobItem);
+	        this.proxyServe = source["proxyServe"];
+	        this.maxPages = source["maxPages"];
+	        this.threads = source["threads"];
+	        this.listDelay = source["listDelay"];
+	        this.trunPageDelay = source["trunPageDelay"];
+	        this.contentDelay = source["contentDelay"];
+	        this.state = source["state"];
+	        this.stateType = source["stateType"];
+	        this.progress = source["progress"];
+	        this.needDownloadAttaches = source["needDownloadAttaches"];
+	    }
+	
+		convertValues(a: any, classs: any, asMap: boolean = false): any {
+		    if (!a) {
+		        return a;
+		    }
+		    if (a.slice && a.map) {
+		        return (a as any[]).map(elem => this.convertValues(elem, classs));
+		    } else if ("object" === typeof a) {
+		        if (asMap) {
+		            for (const key of Object.keys(a)) {
+		                a[key] = new classs(a[key]);
+		            }
+		            return a;
+		        }
+		        return new classs(a);
+		    }
+		    return a;
+		}
+	}
 	export class AttachLink {
 	    title: string;
 	    href: string;
+	    fileName: string;
+	    fileType: string;
+	    fileSize: string;
+	    filePath: string;
 	
 	    static createFrom(source: any = {}) {
 	        return new AttachLink(source);
@@ -12,10 +102,16 @@ export namespace main {
 	        if ('string' === typeof source) source = JSON.parse(source);
 	        this.title = source["title"];
 	        this.href = source["href"];
+	        this.fileName = source["fileName"];
+	        this.fileType = source["fileType"];
+	        this.fileSize = source["fileSize"];
+	        this.filePath = source["filePath"];
 	    }
 	}
 	export class ResultItem {
 	    no: number;
+	    site: string;
+	    channel: string;
 	    href: string;
 	    listTitle: string;
 	    listPubishTime: string;
@@ -34,6 +130,8 @@ export namespace main {
 	    constructor(source: any = {}) {
 	        if ('string' === typeof source) source = JSON.parse(source);
 	        this.no = source["no"];
+	        this.site = source["site"];
+	        this.channel = source["channel"];
 	        this.href = source["href"];
 	        this.listTitle = source["listTitle"];
 	        this.listPubishTime = source["listPubishTime"];
@@ -70,6 +168,7 @@ export namespace main {
 	    author: string;
 	    url: string;
 	    code: string;
+	    listBodyCss: string;
 	    listItemCss: string;
 	    listLinkCss: string;
 	    listPublishTimeCss: string;
@@ -82,6 +181,7 @@ export namespace main {
 	    listJs: string;
 	    contentJs: string;
 	    attachJs: string;
+	    listTrunPageJs: string;
 	
 	    static createFrom(source: any = {}) {
 	        return new SpiderConfig(source);
@@ -94,6 +194,7 @@ export namespace main {
 	        this.author = source["author"];
 	        this.url = source["url"];
 	        this.code = source["code"];
+	        this.listBodyCss = source["listBodyCss"];
 	        this.listItemCss = source["listItemCss"];
 	        this.listLinkCss = source["listLinkCss"];
 	        this.listPublishTimeCss = source["listPublishTimeCss"];
@@ -106,6 +207,7 @@ export namespace main {
 	        this.listJs = source["listJs"];
 	        this.contentJs = source["contentJs"];
 	        this.attachJs = source["attachJs"];
+	        this.listTrunPageJs = source["listTrunPageJs"];
 	    }
 	}
 

key.pem

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCuh3EWkkIlF7ta
-aB4E6EksqyWaAva8uEepvuA0TDB44fbVdGQ9DYpHaB1XIJ3nmiILPt2UCU2xuzT9
-iB1U19upwCKTOE5ZAxF7cfriZzR+YdFf77cFZNJROx793MDscw21WLiCTttxACCN
-w4fwRIU9bS+dMbWkuoTlA3mHFb+wUanXKTTGjVaQt9eejfZQs2FZRC/P1v5JvDN0
-EGum6TqOosh4+vFS1Y7HfIUBtfV3LaV58/0T32TjUmKjNOHxnXUXTbXMvk1AQv6n
-WMbUicCPZzzcfivQ8p8gVcENaaUWjzpfZjwMMscxaQgJK/pt2nRVjWg1bFiK0GmT
-Vs4fA1ofAgMBAAECggEACguJmr74R6JCCkYL1ER6UbPYCjE5eksw9Lgjt17bO1nm
-FwsH6euplcqMRcN+0yGv6+3GWwreCei4eA8pgQSbg/2m/8ox2DWw/+Xjhrxh7RQ8
-NMVbR1gyMrKwafQWtoU4uMNOe1GGl85mEUK7xDxtXse2AdomlkCV/YhhqkC6M6+n
-OpfYe9SEIeOwJjeiXlLg8tsqWDO9jdgjS7ZwnHqHeYyMorNCErUfpkgxpDEP8Dgt
-IpPnssiENr0bIpR43YO7KNE0WqhwUWIAVLXt9lMTfDayB80mezgeCLKQPP64eWeh
-evcfpzFsma90VTkHojQBbWpN/75QRrDEYzu/zwg4IQKBgQDMnqAUDS/BCXpalj/R
-vcE+WeoUmKEK+TqAbW9d3HahVusH8/eSwrDWUk7zafV3bHNFVW6Fer47vs0BIS1D
-orUygIc6GsDg/77crU5dio4N2F/Pmg+tLJHq31qWQAEZZFVGoaIvRYD2qCabUZPx
-dVIPK3ghaAzSE05v8v6JVYfUbQKBgQDaWohgRWDYJcsPOkVqez4y5Y3gWUZkI968
-zyFwaMZ8tSMbg6m7AJaUZFtz+dePDsrX8n/iGuEpMXZSmSG3sSPwS41k5poiBBdt
-Qy9r/Irp31SlA4FXy9Yq1SpXFSXvUZMSPdxOC03tGfOPa30Z2rH0rX6l81u5KhEa
-vwMXQxnZOwKBgEa5XSMRG7xhBkVhQVXBfJWMhnfv+VnNowbYzHFozigd3sa08JFt
-canicR95NDq+5WjFipngPvhvjnQhf3+tMWvvOM5AiQI740BrNnbmeQsYCqW63khA
-635/DNR58udP4pmzLFeiclzO6ektXTFMF7zejXsed6/0tFvFZW0afwRRAoGBAIU3
-ltyld2BoLmsr8g31Aw2qX9TworGV8N7gwFYElpSfLrwqp/MfeL8wO1uWopz1OWxm
-1v7rx1OKidX690dLG9IPRkS5LHB0bpaK1vPbMCVfzBSg/tjB0/ht9VcL4AkSi9gl
-RbOX0gNGQgLOYZTUiJ3u+8Xjo6Jkt+rJfulCVxLhAoGAZ1M20dYPI0edvYvdUOyL
-xDwmdlkOO2DU4+od7hhV9vdvvshbRC88m3YgjcGx59oFie++ynwf8c+LHi4Qx2Tw
-J/NzmWoOlC4Q2FljTGrNmbxAtVzkXOfpKoL9+YETZzHJEBLYdmETLfXLQK4OaXQz
-AxverbaEA3UNOAQPte5r7pI=
------END PRIVATE KEY-----

main.go

@@ -1,7 +1,12 @@
 package main
 
 import (
+	"container/list"
 	"embed"
+	be "spidercreator/backend"
+	bdb "spidercreator/backend/db"
+	bvm "spidercreator/backend/vm"
+	bws "spidercreator/backend/webservice"
 
 	"github.com/wailsapp/wails/v2"
 	"github.com/wailsapp/wails/v2/pkg/options"
@@ -9,15 +14,18 @@ import (
 )
 
 var (
-	//go:embed all:frontend/dist
-	assets embed.FS
-	app    *App
+	assets               embed.FS
+	app                  *App
+	db                   *bdb.SpiderDb
+	exitCh               chan bool
+	baseDir, attachesDir string           = ".", ""
+	currentSpiderConfig  *be.SpiderConfig = new(be.SpiderConfig)
+	currentResults                        = list.New() //b.ResultItems = make(b.ResultItems, 0)
+	vm                   *bvm.VM
+	ws                   *bws.WebService
 )
 
 func main() {
-	//
-	go runHttpServe()
-
 	// Create an instance of the app structure
 	app = NewApp()
 

service.go

@@ -1,117 +0,0 @@
-// 对外服务
-package main
-
-import (
-	"crypto/tls"
-	_ "embed"
-	"encoding/json"
-	"fmt"
-	"log"
-	"net/http"
-
-	"github.com/wailsapp/wails/v2/pkg/runtime"
-)
-
-const (
-	LISTEN_ADDR = ":8080"
-)
-
-type (
-	SpiderConfigItem struct {
-		Key string `json:"key"`
-		Css string `json:"css"`
-	}
-)
-
-var (
-	//go:embed cert.pem
-	certBytes []byte
-	//go:embed key.pem
-	keyBytes []byte
-)
-
-func runHttpServe() {
-	// 设置HTTP服务器
-	mux := http.NewServeMux()
-	// 解析证书
-	cert, err := tls.X509KeyPair(certBytes, keyBytes)
-	if err != nil {
-		log.Println(err.Error())
-		return
-	}
-	// 创建一个TLS配置
-	tlsConfig := &tls.Config{
-		// 可以在这里添加其他TLS配置
-		Certificates:       []tls.Certificate{cert},
-		ServerName:         "localhost",
-		InsecureSkipVerify: true,
-	}
-	server := &http.Server{
-		Addr:      LISTEN_ADDR,
-		Handler:   mux,
-		TLSConfig: tlsConfig,
-	}
-	//这里注册HTTP服务
-	mux.HandleFunc("/save", SaveSpiderConfig)
-	mux.HandleFunc("/load", LoadSpiderConfig)
-	//
-	log.Println("Starting HTTPS server on ", LISTEN_ADDR)
-	err = server.ListenAndServeTLS("", "")
-	if err != nil {
-		log.Println("Failed to start server:  ", err.Error())
-		return
-	}
-}
-
-// LoadCurrentSpiderConfig,json处理
-func SaveSpiderConfig(w http.ResponseWriter, r *http.Request) {
-	log.Println("保存设置")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
-	w.Header().Set("Content-Type", "application/json")
-	var req = new(SpiderConfigItem)
-	err := json.NewDecoder(r.Body).Decode(req)
-	if err != nil {
-		log.Println("序列化失败")
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-	log.Println("CSS", req.Key, req.Css)
-	//TODO 业务操作
-	switch req.Key {
-	case "listItemCss":
-		currentSpiderConfig.ListItemCss = req.Css
-	case "listLinkCss":
-		currentSpiderConfig.ListLinkCss = req.Css
-	case "listPublishTimeCss":
-		currentSpiderConfig.ListPubtimeCss = req.Css
-	case "listNextPageCss":
-		currentSpiderConfig.ListNextPageCss = req.Css
-	case "titleCss":
-		currentSpiderConfig.TitleCss = req.Css
-	case "publishUnitCss":
-		currentSpiderConfig.PublishUnitCss = req.Css
-	case "publishTimeCss":
-		currentSpiderConfig.PublishTimeCss = req.Css
-	case "contentCss":
-		currentSpiderConfig.ContentCss = req.Css
-	case "attachCss":
-		currentSpiderConfig.AttachCss = req.Css
-	}
-	fmt.Fprint(w, "{'code':200}")
-	db.SaveOrUpdate(currentSpiderConfig)
-	//TODO 通知开发工具端，CSS选择器有变动
-	runtime.EventsEmit(app.ctx, "spiderConfigChange", currentSpiderConfig)
-}
-
-// LoadCurrentSpiderConfig,加载，返回当前配置项
-func LoadSpiderConfig(w http.ResponseWriter, r *http.Request) {
-	log.Println("加载当前配置项")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
-	w.Header().Set("Content-Type", "application/json")
-	err := json.NewEncoder(w).Encode(currentSpiderConfig)
-	if err != nil {
-		log.Println("反向序列化失败")
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-}

types.go

@@ -1,56 +0,0 @@
-package main
-
-type (
-	//爬虫配置信息
-	SpiderConfig struct {
-		Site            string `json:"site"`
-		Channel         string `json:"channel"`
-		Author          string `json:"author"`
-		Url             string `json:"url"`
-		Code            string `json:"code"`
-		ListItemCss     string `json:"listItemCss"`
-		ListLinkCss     string `json:"listLinkCss"`
-		ListPubtimeCss  string `json:"listPublishTimeCss"`
-		ListNextPageCss string `json:"listNextPageCss"`
-		TitleCss        string `json:"titleCss"`
-		PublishUnitCss  string `json:"publishUnitCss"`
-		PublishTimeCss  string `json:"publishTimeCss"`
-		ContentCss      string `json:"contentCss"`
-		AttachCss       string `json:"attachCss"`
-		ListJSCode      string `json:"listJs"`
-		ContentJSCode   string `json:"contentJs"`
-		AttachJSCode    string `json:"attachJs"`
-	}
-	//附件链接
-	AttachLink struct {
-		Title string `json:"title"`
-		Href  string `json:"href"`
-	}
-	//爬取结果信息
-	ResultItem struct {
-		No          int           `json:"no"`
-		Href        string        `json:"href"`
-		ListTitle   string        `json:"listTitle"`
-		ListPubTime string        `json:"listPubishTime"`
-		Title       string        `json:"title"`
-		PublishUnit string        `json:"publishUnit"`
-		PublishTime string        `json:"publishTime"`
-		Content     string        `json:"content"`
-		ContentHtml string        `json:"contentHtml"`
-		AttachLinks []*AttachLink `json:"attachLinks"` //存放附件的标题，链接
-		AttachJson  string        `json:"attachJson"`  //存放附件的OSS元信息
-	}
-
-	ResultItems    []*ResultItem
-	SpiderConfiges []*SpiderConfig
-)
-
-func (sc SpiderConfiges) Len() int {
-	return len(sc)
-}
-func (sc SpiderConfiges) Swap(i, j int) {
-	sc[i], sc[j] = sc[j], sc[i]
-}
-func (sc SpiderConfiges) Less(i, j int) bool {
-	return sc[i].Code > sc[j].Code
-}

vm.go

@@ -1,218 +0,0 @@
-package main
-
-import (
-	"bytes"
-	_ "embed"
-	"fmt"
-	"log"
-	"os"
-	"strconv"
-	"strings"
-	"text/template"
-	"time"
-
-	"github.com/bmaupin/go-epub"
-	"github.com/chromedp/chromedp"
-)
-
-const (
-	MAX_TRUN_PAGE = 1000
-)
-
-var (
-	//go:embed tpl/load_list_items.js
-	loadListItemsJS string
-	//go:embed tpl/load_content.js
-	loadContentJS string
-
-	currentResult = make(ResultItems, 0)
-)
-
-// renderJavascriptCoder
-func renderJavascriptCoder(tpl string, sc *SpiderConfig) string {
-	t, err := template.New("").Parse(tpl)
-	if err != nil {
-		log.Println("创建JS代码模板失败", err.Error())
-		return ""
-	}
-	buf := new(bytes.Buffer)
-	err = t.Execute(buf, sc)
-	if err != nil {
-		log.Println("执行JS代码模板失败", err.Error())
-		return ""
-	}
-	return buf.String()
-}
-
-// RunSpider
-func RunSpider(url string, listDealy int64, contentDelay int64, headless bool, showImage bool, proxyServe string, exit chan bool) {
-	sc := MergeSpiderConfig(currentSpiderConfig, &SpiderConfig{Url: url})
-	_, baseCancel, _, _, ctx, cancel := NewBrowser(headless, showImage, proxyServe)
-	log.Println("1浏览器打开")
-	app.pushMessage("debug_event", "1 浏览器打开")
-	defer func() {
-		cancel()
-		baseCancel()
-		log.Println("0浏览器已经销毁")
-		app.pushMessage("debug_event", "0 浏览器已经销毁")
-		close(exit)
-	}()
-	currentResult = make(ResultItems, 0, 0)
-	chromedp.Run(ctx, chromedp.Tasks{
-		chromedp.Navigate(sc.Url),
-		chromedp.WaitReady("document.body", chromedp.ByJSPath),
-		chromedp.Sleep(time.Duration(listDealy) * time.Millisecond),
-	})
-	app.pushMessage("debug_event", "2 页面已经打开")
-	log.Println("2页面打开")
-	listResult := make(ResultItems, 0)
-	//TODO 2. 执行JS代码，获取列表页信息
-	runJs := renderJavascriptCoder(loadListItemsJS, sc)
-	err := chromedp.Run(ctx, chromedp.Tasks{
-		chromedp.Evaluate(runJs, &listResult),
-	})
-	if err != nil {
-		log.Println("执行JS代码失败", err.Error())
-		app.pushMessage("debug_event", "2 执行JS代码失败")
-		return
-	}
-	app.pushMessage("debug_event", "3 获取列表完成")
-	log.Println("3获取列表完成")
-
-	//TODO 3. 打开详情页 ,最多打开10条
-	runJs = renderJavascriptCoder(loadContentJS, sc)
-	for _, v := range listResult {
-		select {
-		case <-exit:
-			return
-		default:
-			app.pushMessage("debug_event", fmt.Sprintf("4. %d- 待 下载详情页 %s ", v.No, v.Title))
-			var result string = ""
-			err = chromedp.Run(ctx, chromedp.Tasks{
-				chromedp.Navigate(v.Href),
-				chromedp.WaitReady(`document.body`, chromedp.ByJSPath),
-				chromedp.Sleep(time.Duration(contentDelay) * time.Millisecond),
-				chromedp.Evaluate(runJs, v),
-			})
-			if err != nil {
-				log.Println("执行JS代码失败", err.Error())
-			}
-			//关闭当前TAB页
-			chromedp.Run(ctx, chromedp.Tasks{
-				chromedp.Evaluate(`var ret="";window.close();ret`, &result),
-			})
-			app.pushMessage("debug_event", fmt.Sprintf("4. %d- 下载详情页 %s 完成", v.No, v.Title))
-			currentResult = append(currentResult, v)
-		}
-	}
-	app.pushMessage("debug_event", "5 采集测试完成")
-	log.Println("5采集测试完成")
-}
-
-// ExportEpubFile 导出epub文件
-func ExportEpubFile(filepath string) {
-	output := epub.NewEpub("")
-	output.SetTitle(currentSpiderConfig.Site)
-	output.SetAuthor("unknow")
-	for i, art := range currentResult {
-		body := "<h2>" + art.Title + "</h2><p>" + strings.Join(strings.Split(art.Content, "\n"), "</p><p>") + "</p>"
-		output.AddSection(body, art.Title, fmt.Sprintf("%06d.xhtml", i+1), "")
-	}
-	fo, err := os.Create(filepath)
-	if err != nil {
-		app.pushMessage("debug_event", err.Error())
-	}
-	output.WriteTo(fo)
-	fo.Close()
-}
-
-// CountYestodayArts 统计昨日信息发布量
-func CountYestodayArts(url string, listDealy int64, trunPageDelay int64,
-	headless bool, showImage bool, exit chan bool) (count int) {
-	sc := MergeSpiderConfig(currentSpiderConfig, &SpiderConfig{Url: url})
-	_, baseCancel, _, _, ctx, cancel := NewBrowser(headless, showImage, "")
-	log.Println("1浏览器打开")
-	app.pushMessage("debug_event", "1 浏览器打开")
-	defer func() {
-		cancel()
-		baseCancel()
-		log.Println("0浏览器已经销毁")
-		app.pushMessage("debug_event", "0 浏览器已经销毁")
-		app.pushMessage("debug_event", fmt.Sprintf("99 昨日信息发布量:%d ", count))
-		close(exit)
-	}()
-
-	//时间比较
-	now := time.Now()
-	yesterday := now.AddDate(0, 0, -1) // 获取昨天的日期
-	startOfYesterday := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, now.Location())
-	endOfYesterday := startOfYesterday.AddDate(0, 0, 1).Add(-time.Nanosecond)
-
-	//TODO 1.
-	chromedp.Run(ctx, chromedp.Tasks{
-		chromedp.Navigate(sc.Url),
-		chromedp.WaitReady("document.body", chromedp.ByJSPath),
-		chromedp.Sleep(time.Duration(listDealy) * time.Millisecond),
-	})
-	app.pushMessage("debug_event", "2 页面已经打开")
-	log.Println("2页面打开")
-	//TODO 2. 执行JS代码，获取列表页信息
-	runJs := renderJavascriptCoder(loadListItemsJS, sc)
-	tmp := map[string]bool{}
-	//最多翻页1000页
-	for i := 0; i < MAX_TRUN_PAGE; i++ {
-		select {
-		case <-exit:
-			return
-		default:
-			app.pushMessage("debug_event", "3 执行列表页JS")
-			listResult := make(ResultItems, 0)
-			err := chromedp.Run(ctx, chromedp.Tasks{
-				chromedp.Evaluate(runJs, &listResult),
-			})
-			if err != nil {
-				log.Println("执行JS代码失败", err.Error())
-				app.pushMessage("debug_event", "3 执行JS代码失败")
-				return
-			}
-			//TODO 人工智能转换采集到的日期
-			callAIState := false
-			for j := 0; j < 5; j++ {
-				app.pushMessage("debug_event", "3 执行AI提取列表发布时间"+strconv.Itoa(j+1))
-				err := UpdateResultDateStr(listResult)
-				if err == nil {
-					callAIState = true
-					break
-				}
-			}
-			if !callAIState {
-				app.pushMessage("debug_event", "3 多轮次调用AI均未得到合理结果")
-				return
-			}
-			//TODO 日期统计
-			for _, r := range listResult {
-				day, err := time.Parse("2006-01-02", r.ListPubTime)
-				if err != nil {
-					continue
-				}
-				if _, ok := tmp[r.Href]; ok { //去重
-					continue
-				}
-				if day.After(startOfYesterday) && day.Before(endOfYesterday) {
-					count += 1
-				} else if day.Before(startOfYesterday) {
-					return
-				}
-			}
-			app.pushMessage("debug_event", fmt.Sprintf("4 当前观测昨日信息发布量:%d ", count))
-			//TODO 翻页
-			//fmt.Println("下一页CSS选择器", currentSpiderConfig.ListNextPageCss)
-			chromedp.Run(ctx, chromedp.Tasks{
-				chromedp.Click(fmt.Sprintf(`document.querySelector("%s")`, currentSpiderConfig.ListNextPageCss),
-					chromedp.ByJSPath),
-				chromedp.Sleep(time.Duration(trunPageDelay) * time.Millisecond),
-			})
-		}
-	}
-	return
-}