导出功能页面功能修改

backend/types.go

@@ -54,10 +54,9 @@ type (
 	}
 	//爬取结果信息
 	ResultItem struct {
-		No      int    `json:"no"` //编号
-		Site    string `json:"site"`
-		Channel string `json:"channel"`
-
+		No          int           `json:"no"` //编号
+		Site        string        `json:"site"`
+		Channel     string        `json:"channel"`
 		Href        string        `json:"href"`
 		ListTitle   string        `json:"listTitle"`
 		ListPubTime string        `json:"listPubishTime"`

backend/vm/single.go

@@ -87,6 +87,8 @@ func (vm *VM) RunSpider(url string, maxPages int, listDealy int64, contentDelay
 				chromedp.Sleep(time.Duration(contentDelay) * time.Millisecond),
 				chromedp.Evaluate(runJs, v),
 			})
+			v.Site = sc.Site
+			v.Channel = sc.Channel
 			if err != nil {
 				qu.Debug("执行JS代码失败", err.Error())
 			}

backend/vm/worker.go

@@ -145,6 +145,8 @@ func (vm *VM) RunSpiderMulThreads(url string, maxPages int, listDealy int64, tru
 				wg.Add(1)
 				no += 1
 				v.No = no
+				v.Site = sc.Site
+				v.Channel = sc.Channel
 				currentResult.PushBack(v)
 				go w.Run(v, ch, wg)
 			}

bind4comm.go

@@ -2,10 +2,25 @@
 package main
 
 import (
+	"container/list"
+	"encoding/json"
+	"fmt"
+	"github.com/bmaupin/go-epub"
 	"github.com/wailsapp/wails/v2/pkg/runtime"
+	"github.com/xuri/excelize/v2"
 	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"os"
+	be "spider_creator/backend"
+	"strconv"
+	"strings"
 )
 
+var FileType = map[string]runtime.FileFilter{
+	"epub": runtime.FileFilter{Pattern: "*.epub", DisplayName: "epub file *.epub"},
+	"xlsx": runtime.FileFilter{Pattern: "*.xlsx", DisplayName: "excel file *.xlsx"},
+	"json": runtime.FileFilter{Pattern: "*.json", DisplayName: "json file *.json"},
+}
+
 // Greet returns a greeting for the given name
 func (a *App) Dispatch(event string, data interface{}) error {
 	runtime.EventsEmit(a.ctx, event, data)
@@ -13,12 +28,10 @@ func (a *App) Dispatch(event string, data interface{}) error {
 }
 
 // SelectSaveFilePath
-func (a *App) SelectSaveFilePath(defaultDirectory, defaultFileName string) string {
-	qu.Debug("导出文件位置：", defaultDirectory, defaultFileName)
+func (a *App) SelectSaveFilePath(defaultDirectory, defaultFileName, defaulFileType string) string {
+	qu.Debug("导出文件位置：", defaultDirectory, defaultFileName, defaulFileType)
 	path, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{Filters: []runtime.FileFilter{
-		{Pattern: "*.epub", DisplayName: "epub file *.epub"},
-		{Pattern: "*.xlsx", DisplayName: "excel file *.xlsx"},
-		{Pattern: "*.json", DisplayName: "json file *.json"},
+		FileType[defaulFileType],
 	},
 		DefaultFilename:  defaultFileName,
 		DefaultDirectory: defaultDirectory,
@@ -41,3 +54,104 @@ func (a *App) SelectOpenFilePath() string {
 	}
 	return path
 }
+
+// RunExportExcelFile 数据集导出到excel文件中
+func (a *App) RunExportExcelFile(filepath, code string, currentResult *list.List) error {
+	qu.Debug("filepath---", filepath)
+	f := excelize.NewFile()
+	defer f.Close()
+	f.SetCellStr("Sheet1", "A1", "站点")
+	f.SetCellStr("Sheet1", "B1", "栏目")
+	f.SetCellStr("Sheet1", "C1", "爬虫")
+	//写入数据
+	f.SetCellStr("Sheet1", "D1", "标题")
+	f.SetCellStr("Sheet1", "E1", "链接")
+	f.SetCellStr("Sheet1", "F1", "发布单位")
+	f.SetCellStr("Sheet1", "G1", "发布时间")
+	f.SetCellStr("Sheet1", "H1", "正文")
+	f.SetCellStr("Sheet1", "I1", "附件")
+	i := 0
+	for el := currentResult.Front(); el != nil; el = el.Next() {
+		r, _ := el.Value.(*be.ResultItem)
+		//写入站点信息
+		iStr := strconv.Itoa(i + 2)
+		f.SetCellStr("Sheet1", "A"+iStr, r.Site)
+		f.SetCellStr("Sheet1", "B"+iStr, r.Channel)
+		f.SetCellStr("Sheet1", "C"+iStr, code)
+		//写入数据
+		f.SetCellStr("Sheet1", "D"+iStr, r.Title)
+		f.SetCellStr("Sheet1", "E"+iStr, r.Href)
+		f.SetCellStr("Sheet1", "F"+iStr, r.PublishUnit)
+		f.SetCellStr("Sheet1", "G"+iStr, r.ListPubTime)
+		f.SetCellStr("Sheet1", "H"+iStr, r.Content)
+		f.SetCellStr("Sheet1", "I"+iStr, "")
+		if len(r.AttachLinks) > 0 {
+			bs, err := json.Marshal(r.AttachLinks)
+			if err == nil {
+				f.SetCellStr("Sheet1", "I"+iStr, string(bs))
+			}
+		}
+		i += 1
+	}
+	err := f.SaveAs(filepath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (a *App) RunExportJsonFile(filepath, code string, currentResult *list.List) error {
+	qu.Debug("filepath---", filepath)
+	var result []map[string]interface{}
+	for el := currentResult.Front(); el != nil; el = el.Next() {
+		r, _ := el.Value.(*be.ResultItem)
+		rmap := map[string]interface{}{
+			"site":        r.Site,
+			"channel":     r.Channel,
+			"code":        code,
+			"title":       r.Title,
+			"href":        r.Href,
+			"publishdept": r.PublishUnit,
+			"publishtime": r.ListPubTime,
+			"detail":      r.Content,
+			"attachLinks": r.AttachLinks,
+		}
+		result = append(result, rmap)
+	}
+	jsonData, err := json.MarshalIndent(result, "", "	")
+	if err != nil {
+		return err
+	}
+	fo, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer fo.Close()
+	if _, err := fo.Write(jsonData); err != nil {
+		return fmt.Errorf("failed to write data to file: %w", err)
+	}
+	return nil
+}
+
+// RunExportEpubFile 导出epub文件
+func (a *App) RunExportEpubFile(bookname, filepath string, currentResult *list.List) error {
+	qu.Debug("filepath---", filepath)
+	output := epub.NewEpub(bookname)
+	output.SetTitle(bookname)
+	output.SetDescription(bookname)
+	output.SetAuthor("unknow")
+	i := 1
+	for el := currentResult.Front(); el != nil; el = el.Next() {
+		art, _ := el.Value.(*be.ResultItem)
+		body := "<h2>" + art.Title + "</h2><p>" + strings.Join(strings.Split(art.Content, "\n"), "</p><p>") + "</p>"
+		output.AddSection(body, art.Title, fmt.Sprintf("%06d.xhtml", i+1), "")
+		i += 1
+	}
+	fo, err := os.Create(filepath)
+	if err != nil {
+		a.Dispatch("debug_event", err.Error())
+	}
+	output.WriteTo(fo)
+	fo.Close()
+	return nil
+}

bind4spider.go

@@ -83,7 +83,6 @@ func (a *App) ViewResultItemAll(code string) be.ResultItems {
 
 // ServerActionCurrentOpenTab 记录当前编辑code的cssmark，高亮回显使用
 func (a *App) ServerActionCurrentOpenTab(cssMark map[string]interface{}) {
-	qu.Debug("----------------")
 	sc, err := be.NewSpiderConfig(cssMark)
 	if err != nil {
 		qu.Debug("标注信息传输失败！")
@@ -91,11 +90,62 @@ func (a *App) ServerActionCurrentOpenTab(cssMark map[string]interface{}) {
 	bew.SetCurrentTabCssMark(sc)
 }
 
-// ExportEpubFile
-func (a *App) ExportEpubFile(bookname, filepath string) string {
+// ExportEpubFile epub导出
+func (a *App) ExportEpubFile(bookname, filepath, code string) map[string]interface{} {
+	sc := be.DataResults[code]
+	qu.Debug("结果导出：", bookname, filepath, code)
 	qu.Debug("EPUB 文件存储：", bookname, filepath)
-	db.ExportEpubFile(bookname, filepath, currentResults)
-	return "ok"
+	var msg string
+	var errType int
+	if sc != nil {
+		if err := a.RunExportEpubFile(bookname, filepath, sc); err != nil {
+			msg = err.Error()
+		} else {
+			msg = "导出成功"
+			errType = 1
+		}
+	} else {
+		msg = "导出失败，无法获取结果"
+	}
+	return map[string]interface{}{"err": errType, "msg": msg}
+}
+
+// ExportJsonFile json导出
+func (a *App) ExportJsonFile(filepath, code string) map[string]interface{} {
+	sc := be.DataResults[code]
+	qu.Debug("结果导出：", filepath, code)
+	var msg string
+	var errType int
+	if sc != nil {
+		if err := a.RunExportJsonFile(filepath, code, sc); err != nil {
+			msg = err.Error()
+		} else {
+			msg = "导出成功"
+			errType = 1
+		}
+	} else {
+		msg = "导出失败，无法获取结果"
+	}
+	return map[string]interface{}{"err": errType, "msg": msg}
+}
+
+// ExportExcelFile excel导出
+func (a *App) ExportExcelFile(filepath, code string) map[string]interface{} {
+	sc := be.DataResults[code]
+	qu.Debug("结果导出：", filepath, code)
+	var msg string
+	var errType int
+	if sc != nil {
+		if err := a.RunExportExcelFile(filepath, code, sc); err != nil {
+			msg = err.Error()
+		} else {
+			msg = "导出成功"
+			errType = 1
+		}
+	} else {
+		msg = "导出失败，无法获取结果"
+	}
+	return map[string]interface{}{"err": errType, "msg": msg}
 }
 
 // ImportSpiderConfigByExcelFile 通过excel文件导入爬虫配置
@@ -110,15 +160,6 @@ func (a *App) CountYestodayArts(url string, listDealy int64, trunPageDelay int64
 	vm.CountYestodayArts(url, listDealy, trunPageDelay, headless, showImage, exitCh, currentSpiderConfig)
 }
 
-// ExportExcelFile
-func (a *App) ExportExcelFile(filepath string) string {
-	if err := db.ExportExcelFile(filepath, currentSpiderConfig.Site, currentSpiderConfig.Channel, currentResults); err == nil {
-		return "ok"
-	} else {
-		return err.Error()
-	}
-}
-
 // LoadAllJobs 加载我的所有作业
 func (a *App) LoadAllJobs() be.Jobs {
 	rs, err := bdb.LoadEntities[be.Job]("jobs")

frontend/package.json.md5

@@ -1 +1 @@
-979198f11d1a5ca231cc33af9dda5cb9
+bfa495e8ffb597bbe5812c2b3730caa3

frontend/src/components/spider/RunSpider.vue

@@ -77,7 +77,7 @@
                     <template #dropdown>
                         <el-dropdown-menu>
                             <el-dropdown-item @click="handleExportEpub">导出EPUB格式文件</el-dropdown-item>
-                            <el-dropdown-item>导出JSON格式文件</el-dropdown-item>
+                            <el-dropdown-item @click="handleExportJson">导出JSON格式文件</el-dropdown-item>
                             <el-dropdown-item @click="handleExportExcel">导出Excel格式文件</el-dropdown-item>
                             <!-- <el-dropdown-item>补录/上推至平台</el-dropdown-item> -->
                         </el-dropdown-menu>
@@ -101,8 +101,8 @@ import { ref, computed } from 'vue';
 import { ElMessage, ElMessageBox } from 'element-plus'
 import ViewArticle from "./ViewArticle.vue"
 
-import { ViewCurrentSpiderConfig, DebugSpider, StopDebugSpider } from "../../../wailsjs/go/main/App"
-import { ViewResultItemAll, SelectSaveFilePath, ExportEpubFile, CountYestodayArts } from "../../../wailsjs/go/main/App"
+import {ViewCurrentSpiderConfig, DebugSpider, StopDebugSpider, ExportJsonFile} from "../../../wailsjs/go/main/App"
+import { ViewResultItemAll, SelectSaveFilePath, ExportEpubFile, ExportExcelFile, CountYestodayArts } from "../../../wailsjs/go/main/App"
 import { EventsOn, EventsOff } from "../../../wailsjs/runtime"
 
 let originData = {}
@@ -214,12 +214,20 @@ const handleExportEpub = () => {
         console.log("无效的文件存储路径", save2file)
         return
       }
-      ExportEpubFile(value, save2file).then(d => {
-        ElMessage({
-          message: `导出epub文件${save2file}完成！`,
-          showClose: true,
-          duration: 3000,
-        });
+      ExportEpubFile(value, save2file, formData.value.code).then(d => {
+        if (d.err === 1) {
+          ElMessage({
+            message: d.msg || `导出epub文件${save2file}完成！`,
+            type: 'success',
+            duration: 3000,
+          });
+        } else {
+          ElMessage({
+            message: d.msg || `导出epub文件${save2file}失败！`,
+            type: 'error',
+            duration: 3000,
+          });
+        }
       })
     })
   })
@@ -231,16 +239,25 @@ const handleExportJson = () => {
     cancelButtonText: '取消',
   }).then(({ value }) => {
     SelectSaveFilePath("", value,"json").then(save2file => {
+      console.log("json",save2file)
       if (save2file == "") {
         console.log("无效的文件存储路径", save2file)
         return
       }
-      ExportEpubFile(value, save2file).then(d => {
-        ElMessage({
-          message: `导出json文件${save2file}完成！`,
-          showClose: true,
-          duration: 3000,
-        });
+      ExportJsonFile(save2file, formData.value.code).then(d => {
+        if (d.err === 1) {
+          ElMessage({
+            message: d.msg || `导出excel文件${save2file}完成！`,
+            type: 'success',
+            duration: 3000,
+          });
+        } else {
+          ElMessage({
+            message: d.msg || `导出excel文件${save2file}失败！`,
+            type: 'error',
+            duration: 3000,
+          });
+        }
       })
     })
   })
@@ -252,11 +269,12 @@ const handleExportExcel = () => {
     cancelButtonText: '取消',
   }).then(({ value }) => {
     SelectSaveFilePath("", value,"xlsx").then(save2file => {
+      console.log("excel",save2file)
       if (save2file == "") {
         console.log("无效的文件存储路径", save2file)
         return
       }
-      ExportExcelFile(save2file, formData.value.site, formData.value.site, formData.value.code,).then(d => {
+      ExportExcelFile(save2file, formData.value.code).then(d => {
         if (d.err === 1) {
             ElMessage({
                 message: d.msg || `导出excel文件${save2file}完成！`,

frontend/src/views/CodeList.vue

@@ -479,7 +479,6 @@ const dialogEvents = {
                 })
             }
         } catch (error) {
-            console.log(error)
             ElMessage({
                 message: '保存失败',
                 type: 'error',
@@ -573,7 +572,6 @@ const tableEvents = {
         }
         const mark = getMarkWithRow(row)
         VerifySpiderConfig(mark).then(r => {
-            console.log("验证爬虫结果", r)
             if (r.err === 1 && r.ret) {
                 verifySpiderDialog.value.dialogVisible = true
                 verifySpiderDialog.value.formData = r.ret

frontend/src/views/Home.vue

@@ -112,7 +112,6 @@ const multipleSelection = ref([]);
 
 const dialogEvents = {
     editSpiderConfigSaveEvent: function (data) {
-        console.log("change data:", data)
         SaveOrUpdateSpiderConfig(data).then(result => {
             ElMessage({
                 message: `成功更新爬虫 ${data.site} /${data.channel}/${data.code}`,

frontend/wailsjs/go/main/App.d.ts

@@ -1,6 +1,7 @@
 // Cynhyrchwyd y ffeil hon yn awtomatig. PEIDIWCH Â MODIWL
 // This file is automatically generated. DO NOT EDIT
 import {backend} from '../models';
+import {list} from '../models';
 import {main} from '../models';
 
 export function CountYestodayArts(arg1:string,arg2:number,arg3:number,arg4:boolean,arg5:boolean):Promise<void>;
@@ -13,12 +14,14 @@ export function DeleteSpiderConfig(arg1:string):Promise<string>;
 
 export function Dispatch(arg1:string,arg2:any):Promise<void>;
 
-export function ExportEpubFile(arg1:string,arg2:string):Promise<string>;
+export function ExportEpubFile(arg1:string,arg2:string,arg3:string):Promise<{[key: string]: any}>;
 
-export function ExportExcelFile(arg1:string):Promise<string>;
+export function ExportExcelFile(arg1:string,arg2:string):Promise<{[key: string]: any}>;
 
 export function ExportJobResult(arg1:string,arg2:string):Promise<string>;
 
+export function ExportJsonFile(arg1:string,arg2:string):Promise<{[key: string]: any}>;
+
 export function ImportSpiderConfigByExcelFile(arg1:string):Promise<string>;
 
 export function LoadAllJobs():Promise<backend.Jobs>;
@@ -27,6 +30,12 @@ export function LoadJob(arg1:string):Promise<backend.Job>;
 
 export function LoadSpiderConfigAll(arg1:number,arg2:number):Promise<Array<backend.SpiderConfig>>;
 
+export function RunExportEpubFile(arg1:string,arg2:string,arg3:list.List):Promise<void>;
+
+export function RunExportExcelFile(arg1:string,arg2:string,arg3:list.List):Promise<void>;
+
+export function RunExportJsonFile(arg1:string,arg2:string,arg3:list.List):Promise<void>;
+
 export function RunJob(arg1:string):Promise<string>;
 
 export function SaveJob(arg1:backend.Job):Promise<string>;
@@ -35,7 +44,7 @@ export function SaveOrUpdateSpiderConfig(arg1:backend.SpiderConfig):Promise<stri
 
 export function SelectOpenFilePath():Promise<string>;
 
-export function SelectSaveFilePath(arg1:string,arg2:string):Promise<string>;
+export function SelectSaveFilePath(arg1:string,arg2:string,arg3:string):Promise<string>;
 
 export function ServerActionCheckLogin():Promise<{[key: string]: any}>;
 
@@ -43,7 +52,7 @@ export function ServerActionClaimCodes():Promise<main.Result>;
 
 export function ServerActionCodeList(arg1:{[key: string]: any}):Promise<main.Result>;
 
-export function ServerActionCurrentOpenTab(arg1:{[key: string]: any}):Promise<main.Result>;
+export function ServerActionCurrentOpenTab(arg1:{[key: string]: any}):Promise<void>;
 
 export function ServerActionGetModifyUsers():Promise<main.Result>;
 
@@ -61,7 +70,7 @@ export function StopJob(arg1:string):Promise<string>;
 
 export function SwitchSpiderConfig(arg1:string):Promise<string>;
 
-export function VerifySpiderConfig(arg1:{[key: string]: any}):Promise<backend.SpiderConfigVerifyResult>;
+export function VerifySpiderConfig(arg1:{[key: string]: any}):Promise<{[key: string]: any}>;
 
 export function ViewCurrentSpiderConfig():Promise<backend.SpiderConfig>;
 

frontend/wailsjs/go/main/App.js

@@ -22,18 +22,22 @@ export function Dispatch(arg1, arg2) {
   return window['go']['main']['App']['Dispatch'](arg1, arg2);
 }
 
-export function ExportEpubFile(arg1, arg2) {
-  return window['go']['main']['App']['ExportEpubFile'](arg1, arg2);
+export function ExportEpubFile(arg1, arg2, arg3) {
+  return window['go']['main']['App']['ExportEpubFile'](arg1, arg2, arg3);
 }
 
-export function ExportExcelFile(arg1) {
-  return window['go']['main']['App']['ExportExcelFile'](arg1);
+export function ExportExcelFile(arg1, arg2) {
+  return window['go']['main']['App']['ExportExcelFile'](arg1, arg2);
 }
 
 export function ExportJobResult(arg1, arg2) {
   return window['go']['main']['App']['ExportJobResult'](arg1, arg2);
 }
 
+export function ExportJsonFile(arg1, arg2) {
+  return window['go']['main']['App']['ExportJsonFile'](arg1, arg2);
+}
+
 export function ImportSpiderConfigByExcelFile(arg1) {
   return window['go']['main']['App']['ImportSpiderConfigByExcelFile'](arg1);
 }
@@ -50,6 +54,18 @@ export function LoadSpiderConfigAll(arg1, arg2) {
   return window['go']['main']['App']['LoadSpiderConfigAll'](arg1, arg2);
 }
 
+export function RunExportEpubFile(arg1, arg2, arg3) {
+  return window['go']['main']['App']['RunExportEpubFile'](arg1, arg2, arg3);
+}
+
+export function RunExportExcelFile(arg1, arg2, arg3) {
+  return window['go']['main']['App']['RunExportExcelFile'](arg1, arg2, arg3);
+}
+
+export function RunExportJsonFile(arg1, arg2, arg3) {
+  return window['go']['main']['App']['RunExportJsonFile'](arg1, arg2, arg3);
+}
+
 export function RunJob(arg1) {
   return window['go']['main']['App']['RunJob'](arg1);
 }
@@ -66,8 +82,8 @@ export function SelectOpenFilePath() {
   return window['go']['main']['App']['SelectOpenFilePath']();
 }
 
-export function SelectSaveFilePath(arg1, arg2) {
-  return window['go']['main']['App']['SelectSaveFilePath'](arg1, arg2);
+export function SelectSaveFilePath(arg1, arg2, arg3) {
+  return window['go']['main']['App']['SelectSaveFilePath'](arg1, arg2, arg3);
 }
 
 export function ServerActionCheckLogin() {

server.go

@@ -100,6 +100,7 @@ func (a *App) ServerActionUpdateCodeState(param map[string]interface{}) *Result
 	}
 	if data.Msg == "" {
 		param["user"] = User
+		qu.Debug("param---", param)
 		//formatUser(param)
 		getResult(param, data, "updatecodestate")
 	}