功能修改

app.go

@@ -55,9 +55,14 @@ func (a *App) startup(ctx context.Context) {
 		os.MkdirAll(attachesDir, 0777)
 	}
 	//创建千里马数据目录
-	qlmDir = baseDir + "/qlm"
-	if _, err := os.Stat(qlmDir); err != nil {
-		os.MkdirAll(qlmDir, 0777)
+	//脚本目录
+	qlmScriptDir := baseDir + "/special/script"
+	qlmLogsDir := baseDir + "/special/logs"
+	if _, err := os.Stat(qlmScriptDir); err != nil {
+		os.MkdirAll(qlmScriptDir, 0777)
+	}
+	if _, err := os.Stat(qlmLogsDir); err != nil {
+		os.MkdirAll(qlmLogsDir, 0777)
 	}
 	var dbfile = baseDir + "/spider.dat"
 	qu.Debug("db file:", dbfile)
@@ -65,7 +70,7 @@ func (a *App) startup(ctx context.Context) {
 	db = bdb.NewSpiderDb(dbfile, a)
 	bdb.Db = db
 	vm = bvm.NewVM(attachesDir, a)
-	glvm = script.NewGLVM(qlmDir, a)
+	glvm = script.NewGLVM(qlmScriptDir, qlmLogsDir, a)
 	ws = bws.NewWebService(db, a, currentSpiderConfig)
 	//
 	go ws.RunHttpServe()

backend/script/script.go

@@ -33,8 +33,8 @@ const (
 	execute_return_type_list   = 1
 	execute_return_type_table  = 2
 
-	qlm_list_lua   = "/script/qlm_list.lua"
-	qlm_detail_lua = "/script/qlm_detail.lua"
+	qlm_list_lua   = "/qlm_list.lua"
+	qlm_detail_lua = "/qlm_detail.lua"
 )
 
 var (
@@ -43,7 +43,9 @@ var (
 )
 
 type GLVm struct {
-	AttachesDir   string
+	ScriptDir     string
+	LogsDir       string
+	LogsFile      *os.File
 	Dnf           backend.EventNotifyFace
 	Headless      bool
 	ShowImage     bool
@@ -59,9 +61,10 @@ type GLBrowser struct {
 	CancelFn context.CancelFunc
 }
 
-func NewGLVM(attachesDir string, dnf be.EventNotifyFace) *GLVm {
+func NewGLVM(scriptDir, logsDir string, dnf be.EventNotifyFace) *GLVm {
 	return &GLVm{
-		AttachesDir:  attachesDir,
+		ScriptDir:    scriptDir,
+		LogsDir:      logsDir,
 		Dnf:          dnf,
 		DataSaveOver: make(chan bool, 1),
 	}
@@ -71,9 +74,9 @@ func NewGLVM(attachesDir string, dnf be.EventNotifyFace) *GLVm {
 func (glvm *GLVm) LoadScript(page string) string {
 	var path string
 	if page == "list" {
-		path = glvm.AttachesDir + qlm_list_lua
+		path = glvm.ScriptDir + qlm_list_lua
 	} else if page == "detail" {
-		path = glvm.AttachesDir + qlm_detail_lua
+		path = glvm.ScriptDir + qlm_detail_lua
 	}
 
 	bs, err := os.ReadFile(path)
@@ -85,9 +88,20 @@ func (glvm *GLVm) LoadScript(page string) string {
 
 // RunScript 执行lua代码
 func (glvm *GLVm) RunScript(script, recordId string) error {
-	defer Catch()
+	defer qu.Catch()
 	var s *lua.LState = lua.NewState()
 	defer s.Close()
+	//日志文件
+	now := time.Now()
+	path := glvm.LogsDir + fmt.Sprintf("/%s.log", qu.FormatDate(&now, qu.Date_Short_Layout))
+	qu.Debug("log path：", path)
+	file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		qu.Debug("日志创建失败：", err)
+		return err
+	}
+	glvm.LogsFile = file
+	defer glvm.LogsFile.Close()
 	//方法绑定
 	glvm.ResetBrowser()  //先创建浏览器对象
 	glvm.BindLuaState(s) //绑定虚拟机函数
@@ -97,6 +111,7 @@ func (glvm *GLVm) RunScript(script, recordId string) error {
 			b.CancelFn()
 			b.Ctx = nil
 			b.CancelFn = nil
+			b = nil
 		}
 	}()
 
@@ -137,29 +152,26 @@ func (glvm *GLVm) ResetBrowser() {
 }
 
 // BindLuaState 绑定虚拟机函数
-func (glvm *GLVm) BindLuaState(state *lua.LState) {
-	state.SetGlobal("browser_reset", state.NewFunction(func(l *lua.LState) int {
+func (glvm *GLVm) BindLuaState(s *lua.LState) {
+	s.SetGlobal("browser_reset", s.NewFunction(func(l *lua.LState) int {
 		glvm.ResetBrowser()
 		return 0
 	}))
-	//
-	state.SetGlobal("browser_save", state.NewFunction(func(l *lua.LState) int {
-		//spiderCode := l.ToString(-5)
-		//siteName := l.ToString(-4)
-		//siteChannelName := l.ToString(-3)
-		//siteChannelUrl := l.ToString(-2)
-		/*table := l.ToTable(-1)
-		data := TableToMap(table)*/
-		//vm.S.Save(spiderCode, siteName, siteChannelName, siteChannelUrl, data)
+	s.SetGlobal("browser_savelog", s.NewFunction(func(l *lua.LState) int {
+		text := l.ToString(-1)
+		qu.Debug("log：", text)
+		now := time.Now()
+		glvm.LogsFile.Write([]byte(fmt.Sprintf("%s%s%s%s", qu.FormatDate(&now, qu.Date_Full_Layout), "---", text, "\n")))
 		return 0
 	}))
 }
 
 func (glvm *GLVm) CloseTabs() {
-	if b := glvm.B; b != nil {
-		b.CancelFn()
-		b.Ctx = nil
-		b.CancelFn = nil
+	if glvm.B != nil && glvm.B.CancelFn != nil {
+		glvm.B.CancelFn()
+		glvm.B.Ctx = nil
+		glvm.B.CancelFn = nil
+		glvm.B = nil
 	}
 }
 
@@ -588,7 +600,7 @@ func (b *GLBrowser) BindLuaState(s *lua.LState, recordId string) {
 	}))
 	//保存数据
 	s.SetGlobal("browser_savedata", s.NewFunction(func(l *lua.LState) int {
-		fmt.Println("---browser_upsertdata---")
+		//fmt.Println("---browser_savedata---")
 		page := l.ToString(-2)
 		data := l.ToTable(-1)
 		result := TableToMap(data)

backend/script/util.go

@@ -196,15 +196,20 @@ func formatTime(year, month, day, hour, minute, second string) string {
 	now := time.Now()
 	if year == "" {
 		year = fmt.Sprint(now.Year())
-	} else if month == "" {
+	}
+	if month == "" {
 		month = fmt.Sprint(now.Month())
-	} else if day == "" {
+	}
+	if day == "" {
 		day = fmt.Sprint(now.Day())
-	} else if hour == "" {
+	}
+	if hour == "" {
 		hour = fmt.Sprint(now.Hour())
-	} else if minute == "" {
+	}
+	if minute == "" {
 		minute = fmt.Sprint(now.Minute())
-	} else if second == "" {
+	}
+	if second == "" {
 		second = fmt.Sprint(now.Second())
 	}
 	return fmt.Sprintf("%4d-%02d-%02d %02d:%02d:%02d",

backend/webservice/webservice.go

@@ -18,8 +18,8 @@ import (
 
 const (
 	LISTEN_ADDR       = ":8080"
-	AI_GEN_LIST_CSS   = "http://39.106.73.135:5005/listpage"
-	AI_GEN_DETAIL_CSS = "http://39.106.73.135:5005/detailpage"
+	AI_GEN_LIST_CSS   = "http://182.92.66.70:5005/listpage"
+	AI_GEN_DETAIL_CSS = "http://182.92.66.70:5005/detailpage"
 )
 
 type (

frontend/src/views/CollectionList.vue

@@ -28,7 +28,7 @@
                         <div class="highlight-main">{{scope.row.stateText}}</div>
                     </template>
                 </el-table-column>
-                <el-table-column label="功能" width="250" align="center">
+                <el-table-column label="功能" width="210" align="center">
                     <template #default="scope">
                         <el-tooltip content="列表页采集" placement="top">
                             <el-button size="small" :class="{ active: scope.row._action_clicked_list_collect }" @click="tableEvents.handleListCollect(scope.$index, scope.row)">
@@ -50,22 +50,27 @@
                                 <el-icon><Position /></el-icon>
                             </el-button>
                         </el-tooltip>
+<!--                        <el-tooltip content="清除记录" placement="top">-->
+<!--                          <el-button size="small" :class="{ active: scope.row._action_clicked_remove_history }" @click="tableEvents.handleRemoveHistory(scope.$index, scope.row)">-->
+<!--                            <el-icon><Delete /></el-icon>-->
+<!--                          </el-button>-->
+<!--                        </el-tooltip>-->
+                    </template>
+                </el-table-column>
+                <el-table-column label="操作" width="180" align="center">
+                    <template #default="scope">
+<!--                        <el-dropdown>-->
+<!--                            <el-button size="small" :class="{ active: scope.row._action_clicked_stop_download}" @click="tableEvents.stopDownloadData(scope.$index, scope.row)">-->
+<!--                              中断-->
+<!--                            </el-button>-->
+<!--                        </el-dropdown>-->
                         <el-tooltip content="清除记录" placement="top">
                           <el-button size="small" :class="{ active: scope.row._action_clicked_remove_history }" @click="tableEvents.handleRemoveHistory(scope.$index, scope.row)">
-                            <el-icon><Delete /></el-icon>
+                            清除记录
                           </el-button>
                         </el-tooltip>
-                    </template>
-                </el-table-column>
-                <el-table-column label="操作" width="120" align="center">
-                    <template #default="scope">
-                        <el-dropdown>
-                            <el-button size="small" :class="{ active: scope.row._action_clicked_stop_download}" @click="tableEvents.stopDownloadData(scope.$index, scope.row)">
-                              中断
-                            </el-button>
-                        </el-dropdown>
                         <el-dropdown>
-                            <el-button size="small">导出</el-button>
+                            <el-button size="small">导出数据</el-button>
                             <template #dropdown>
                                 <el-dropdown-menu>
                                     <el-dropdown-item @click="handleExportJson(scope.row)">导出JSON格式文件</el-dropdown-item>
@@ -122,7 +127,7 @@ import { ref, computed, reactive, watch } from 'vue'
 import { useStore } from 'vuex';
 import { ElMessage, ElMessageBox } from 'element-plus'
 import { ServerActionQlmAddRecord,QlmStopDownloadData, QlmListDataDownload, QlmDetailDataDownload, ServerActionQlmRemoveRepeat, ServerActionQlmPushData, ServerActionQlmClearData } from "../../wailsjs/go/main/App"
-import { SelectSaveFilePath, QlmRunExportJsonFile, QlmRunExportExcelFile } from "../../wailsjs/go/main/App"
+import { SelectSaveFilePath, QlmExportJsonFile, QlmExportExcelFile } from "../../wailsjs/go/main/App"
 
 import Breadcrumb from "../components/Breadcrumb.vue"
 import { USER_ROLE_ADMIN, USER_ROLE_DEVELOPER, USER_ROLE_REVIEWER } from '../data/user'
@@ -246,7 +251,6 @@ async function getTableList() {
         })
         const { data, err, msg } = r
         if (data) {
-            console.log(data)
             listState.total = data.total || 0
             if (Array.isArray(data.list)) {
                 const sList = data.list.map(t => {
@@ -572,7 +576,6 @@ const tableEvents = {
           const payload = {
             recordid: row._id,
           }
-          console.log("payload",payload)
           QlmStopDownloadData(payload).then(r => {
             if (r.err === 1) {
               ElMessage({
@@ -632,7 +635,7 @@ const handleExportJson = (row) => {
         console.log("无效的文件存储路径", save2file)
         return
       }
-      QlmRunExportJsonFile(save2file, row._id).then(d => {
+      QlmExportJsonFile(save2file, row._id).then(d => {
         if (d.err === 1) {
           ElMessage({
             message: d.msg || `导出excel文件${save2file}完成！`,
@@ -661,7 +664,7 @@ const handleExportExcel = (row) => {
         console.log("无效的文件存储路径", save2file)
         return
       }
-      QlmRunExportExcelFile(save2file, row._id).then(d => {
+      QlmExportExcelFile(save2file, row._id).then(d => {
         if (d.err === 1) {
             ElMessage({
                 message: d.msg || `导出excel文件${save2file}完成！`,

frontend/src/views/ReviewList.vue

@@ -900,7 +900,6 @@ const tableEvents = {
     },
     // 管理员打回
     adminReject(_, row) {
-      console.log(row)
       onlyClickHighlight(row, '_action_clicked_reject_code')
       ElMessageBox.prompt('请输入打回原因', '打回', {
         confirmButtonText: '确认打回',

frontend/wailsjs/go/main/App.d.ts

@@ -34,11 +34,11 @@ export function LoadSpiderConfigAll(arg1:number,arg2:number):Promise<Array<backe
 
 export function QlmDetailDataDownload(arg1:{[key: string]: any},arg2:{[key: string]: any}):Promise<main.Result>;
 
-export function QlmListDataDownload(arg1:{[key: string]: any},arg2:{[key: string]: any}):Promise<main.Result>;
+export function QlmExportExcelFile(arg1:string,arg2:string):Promise<{[key: string]: any}>;
 
-export function QlmRunExportExcelFile(arg1:string,arg2:string):Promise<void>;
+export function QlmExportJsonFile(arg1:string,arg2:string):Promise<{[key: string]: any}>;
 
-export function QlmRunExportJsonFile(arg1:string,arg2:string):Promise<void>;
+export function QlmListDataDownload(arg1:{[key: string]: any},arg2:{[key: string]: any}):Promise<main.Result>;
 
 export function QlmStopDownloadData(arg1:{[key: string]: any}):Promise<main.Result>;
 

frontend/wailsjs/go/main/App.js

@@ -62,16 +62,16 @@ export function QlmDetailDataDownload(arg1, arg2) {
   return window['go']['main']['App']['QlmDetailDataDownload'](arg1, arg2);
 }
 
-export function QlmListDataDownload(arg1, arg2) {
-  return window['go']['main']['App']['QlmListDataDownload'](arg1, arg2);
+export function QlmExportExcelFile(arg1, arg2) {
+  return window['go']['main']['App']['QlmExportExcelFile'](arg1, arg2);
 }
 
-export function QlmRunExportExcelFile(arg1, arg2) {
-  return window['go']['main']['App']['QlmRunExportExcelFile'](arg1, arg2);
+export function QlmExportJsonFile(arg1, arg2) {
+  return window['go']['main']['App']['QlmExportJsonFile'](arg1, arg2);
 }
 
-export function QlmRunExportJsonFile(arg1, arg2) {
-  return window['go']['main']['App']['QlmRunExportJsonFile'](arg1, arg2);
+export function QlmListDataDownload(arg1, arg2) {
+  return window['go']['main']['App']['QlmListDataDownload'](arg1, arg2);
 }
 
 export function QlmStopDownloadData(arg1) {

main.go

@@ -32,8 +32,8 @@ var (
 	//重点网站和正式环境
 	isOnly4MainSite            string = "false"
 	BrowserLoadResourceTimeout        = "5"
-	//serverAddress          = "http://visualizeld.spdata.jianyu360.com/%s" //正式环境
-	serverAddress = "http://127.0.0.1:8091/%s" //正式环境
+	serverAddress                     = "http://visualizeld.spdata.jianyu360.com/%s" //正式环境
+	//serverAddress = "http://127.0.0.1:8091/%s" //正式环境
 )
 
 //build

qianlima.go

@@ -81,6 +81,7 @@ func (a *App) QlmDetailDataDownload(param map[string]interface{}, record map[str
 
 // DownloadData 执行脚本下载数据
 func DownloadData(record map[string]interface{}, scriptText, page string) {
+	defer qu.Catch()
 	glvm.ScriptRunning = true
 	defer func() {
 		glvm.ScriptRunning = false
@@ -90,22 +91,52 @@ func DownloadData(record map[string]interface{}, scriptText, page string) {
 	glvm.ShowImage, _ = record["showImage"].(bool)
 	recordId := qu.ObjToString(record["recordid"])
 	//执行脚本
-	glvm.RunScript(scriptText, recordId)
-	for len(script.DataCache) > 0 {
-		qu.Debug("当前待保存数据量：", len(script.DataCache))
-		time.Sleep(time.Second * 1)
+	state := 0
+	err := glvm.RunScript(scriptText, recordId)
+	if err == nil {
+		for len(script.DataCache) > 0 {
+			qu.Debug("当前待保存数据量：", len(script.DataCache))
+			time.Sleep(time.Second * 1)
+		}
 	}
-	state := 2
-	if page == "detail" {
+	if page == "list" {
+		state = 2
+	} else if page == "detail" {
 		state = 5
 	}
 	r := &Result{}
 	getResult(map[string]interface{}{"param": map[string]interface{}{"recordid": recordId, "state": state}}, r, "qlm/updateRecord")
 }
 
-// QlmRunExportExcelFile 导出excel
-func (a *App) QlmRunExportExcelFile(filepath, recordId string) error {
+// QlmExportExcelFile 导出excel
+func (a *App) QlmExportExcelFile(filepath, recordId string) map[string]interface{} {
+	qu.Debug("filepath---", filepath)
+	var msg string
+	var errType int
+	if err := runExportExcelFile(filepath, recordId); err != nil {
+		msg = err.Error()
+	} else {
+		msg = "导出成功"
+		errType = 1
+	}
+	return map[string]interface{}{"err": errType, "msg": msg}
+}
+
+// QlmExportJsonFile 导出json
+func (a *App) QlmExportJsonFile(filepath, recordId string) map[string]interface{} {
 	qu.Debug("filepath---", filepath)
+	var msg string
+	var errType int
+	if err := runQlmExportJsonFile(filepath, recordId); err != nil {
+		msg = err.Error()
+	} else {
+		msg = "导出成功"
+		errType = 1
+	}
+	return map[string]interface{}{"err": errType, "msg": msg}
+}
+
+func runExportExcelFile(filepath, recordId string) error {
 	f := excelize.NewFile()
 	defer f.Close()
 	f.SetCellStr("Sheet1", "A1", "ID")
@@ -125,9 +156,7 @@ func (a *App) QlmRunExportExcelFile(filepath, recordId string) error {
 	return nil
 }
 
-// QlmRunExportJsonFile 导出json
-func (a *App) QlmRunExportJsonFile(filepath, recordId string) error {
-	qu.Debug("filepath---", filepath)
+func runQlmExportJsonFile(filepath, recordId string) error {
 	var result []map[string]interface{}
 	getData(nil, recordId, "json", "export", &result)
 	jsonData, err := json.MarshalIndent(result, "", "	")
@@ -144,7 +173,6 @@ func (a *App) QlmRunExportJsonFile(filepath, recordId string) error {
 	}
 	return nil
 }
-
 func getData(file *excelize.File, recordId, exportStype, from string, result *[]map[string]interface{}) {
 	// 将数据编码为JSON格式
 	param := map[string]interface{}{
@@ -208,7 +236,7 @@ func getData(file *excelize.File, recordId, exportStype, from string, result *[]
 						file.SetCellStr("Sheet1", "D"+indexStr, qu.FormatDateByInt64(&publishtime, qu.Date_Full_Layout))
 					}
 				}
-				repeatText := ""
+				repeatText := "未判重"
 				if repeat := tmp["rp"]; repeat != nil {
 					if repeatTmp, ok := repeat.(bool); ok && repeatTmp {
 						repeatText = "重复"

server.go

@@ -175,7 +175,6 @@ func (a *App) ServerActionQlmClearData(param map[string]interface{}) *Result {
 	r := &Result{}
 	//前期校验
 	if User != nil {
-		qu.Debug("param---", param)
 		getResult(map[string]interface{}{"param": param}, r, "qlm/clearData")
 	} else {
 		r.Msg = "用户登录异常，请重新登录！"