列表操作功能新增

app.go

@@ -65,7 +65,7 @@ func (a *App) startup(ctx context.Context) {
 	db = bdb.NewSpiderDb(dbfile, a)
 	bdb.Db = db
 	vm = bvm.NewVM(attachesDir, a)
-	qlmVm = script.NewGLVM(qlmDir, a)
+	glvm = script.NewGLVM(qlmDir, a)
 	ws = bws.NewWebService(db, a, currentSpiderConfig)
 	//
 	go ws.RunHttpServe()

backend/script/script.go

@@ -45,9 +45,11 @@ type GLVm struct {
 	ProxyServer bool
 	ProxyAddr   string
 	B           *GLBrowser
-	//WsAddr    string
-	//RunMode   int
-	//S         Storage
+}
+
+type GLTask struct {
+	glvm     *GLVm
+	recordid string //记录id
 }
 
 type GLBrowser struct {
@@ -115,32 +117,32 @@ func (glvm *GLVm) RunScript(script string) error {
 }
 
 // ResetBrowser 重置浏览器
-func (vm *GLVm) ResetBrowser() {
-	if vm.B != nil && vm.B.CancelFn != nil && vm.B.BaseCancelFn != nil {
-		vm.B.CancelFn()
-		vm.B.BaseCancelFn()
-		vm.B.Ctx = nil
-		vm.B.CancelFn = nil
-		vm.B.BaseCancelFn = nil
+func (glvm *GLVm) ResetBrowser() {
+	if glvm.B != nil && glvm.B.CancelFn != nil && glvm.B.BaseCancelFn != nil {
+		glvm.B.CancelFn()
+		glvm.B.BaseCancelFn()
+		glvm.B.Ctx = nil
+		glvm.B.CancelFn = nil
+		glvm.B.BaseCancelFn = nil
 	}
-	_, baseCancelFn, _, _, ctx, incCancelFn := backend.NewBrowser(vm.Headless, vm.ShowImage, vm.ProxyServer, "https://")
+	_, baseCancelFn, _, _, ctx, incCancelFn := backend.NewBrowser(glvm.Headless, glvm.ShowImage, glvm.ProxyServer, "https://")
 	b := &GLBrowser{
 		BaseCancelFn: baseCancelFn,
 		Ctx:          ctx,
 		CancelFn:     incCancelFn,
 	}
 
-	if vm.B == nil {
-		vm.B = b
+	if glvm.B == nil {
+		glvm.B = b
 	} else {
-		vm.B.Ctx, vm.B.CancelFn = b.Ctx, b.CancelFn
+		glvm.B.Ctx, glvm.B.CancelFn = b.Ctx, b.CancelFn
 	}
 }
 
 // BindLuaState 绑定虚拟机函数
-func (vm *GLVm) BindLuaState(state *lua.LState) {
+func (glvm *GLVm) BindLuaState(state *lua.LState) {
 	state.SetGlobal("browser_reset", state.NewFunction(func(l *lua.LState) int {
-		vm.ResetBrowser()
+		glvm.ResetBrowser()
 		return 0
 	}))
 	//
@@ -183,7 +185,7 @@ func (b *GLBrowser) findTabContext(tabTitle, tabUrl string, timeoutInt64 int64)
 	return nil, errors.New("can't find tab")
 }
 
-// CloseTabs关闭页面
+// CloseTabs 关闭页面
 func (b *GLBrowser) CloseTabs(tabTitle, tabUrl string, timeoutInt64 int64) (err error) {
 	if timeoutInt64 == 0 {
 		timeoutInt64 = 5
@@ -571,4 +573,12 @@ func (b *GLBrowser) BindLuaState(s *lua.LState) {
 		}
 		return 1
 	}))
+	//保存
+	s.SetGlobal("browser_savedata", s.NewFunction(func(l *lua.LState) int {
+		//fmt.Println("---browser_upsertdata---")
+		//param := l.ToTable(-1)
+		//upset := TableToMap(param)
+
+		return 1
+	}))
 }

frontend/src/components/spider/jscodetpl.js

@@ -32,6 +32,29 @@ ret
     `,
         }, {
             "name": "模版2",
+            "tooltip": "循环A链接标题在链接内，条目CSS选择器作为标题使用",
+            "code": `
+var ret = []
+document.querySelectorAll("{{.ListItemCss}}").forEach((v, i) => {
+    let item = {}
+    var href = v.href
+    if (!href.startsWith("http")) href = window.location.origin + "/" + href
+    let titleDom = v.querySelector("{{.ListLinkCss}}")
+    let title = titleDom.getAttribute("title") || titleDom.innerText
+    item = { "listTitle": title, "href": href, "no": i }
+        
+    if ("{{.ListPubtimeCss}}" != "") {
+        let pubtime = v.querySelector("{{.ListPubtimeCss}}")
+        if (pubtime) {
+            item["listPublishTime"] = pubtime.innerText
+        }
+    }
+    ret.push(item)
+})
+ret
+    `,
+        }, {
+            "name": "模版3",
             "tooltip": "发布日期需要加工，年月/日分开，需要拼装",
             "code": `
 var ret = []
@@ -60,7 +83,7 @@ document.querySelectorAll("{{.ListItemCss}}").forEach((v, i) => {
 ret
     `,
         }, {
-            "name": "模版3",
+            "name": "模版4",
             "tooltip": "发布日期有干扰，需要清洗",
             "code": `
 var ret = []
@@ -88,7 +111,37 @@ document.querySelectorAll("{{.ListItemCss}}").forEach((v, i) => {
 })
 ret
     `,
-        },
+        }, {
+            "name": "模版5",
+            "tooltip": "列表数据有干扰（多了），需要清理",
+            "code": `
+var ret = []
+document.querySelectorAll("{{.ListItemCss}}").forEach((v, i) => {
+    let item = {}
+    if ("{{.ListLinkCss}}" != "") {
+        let link = v.querySelector("{{.ListLinkCss}}")
+        if (link) {
+            var href = link.href
+            if (!href.startsWith("http")) href = window.location.origin + "/" + href
+            let title = link.getAttribute("title") || link.innerText
+            item = { "listTitle": title, "href": href, "no": i }
+        } else {
+            item = { "no": i }
+        }
+    }
+    if ("{{.ListPubtimeCss}}" != "") {
+        let pubtime = v.querySelector("{{.ListPubtimeCss}}")
+        if (pubtime) {
+            item["listPublishTime"] = pubtime.innerText
+        }
+    }
+    ret.push(item)
+})
+//清理过滤逻辑
+ret = ret.filter((item)=>item.listTitle)
+ret
+    `,
+        }
     ],
     //详情页代码集合
     ContentJsCodes: [
@@ -227,9 +280,9 @@ if ("{{.ContentCss}}" != "") {//正文内容
   tmp = document.querySelector("{{.ContentCss}}")
   if (tmp) {
     //TODO在这里写清洗逻辑
-    tmp.removeChild(tmp.children[0])
+    //tmp.removeChild(tmp.children[0])
     //或者
-    //tmp.removeChild(tmp.querySelector("CSS选择器"))
+    //if(tmp.querySelector("CSS选择器"))tmp.removeChild(tmp.querySelector("CSS选择器"))
     ret["content"] = tmp.innerText
     ret["contentHtml"] = tmp.innerHTML
     var patchContent = false
@@ -279,8 +332,7 @@ var link=document.querySelector("{{.ListNextPageCss}}");
 if(link)link.click();
 ""   
     `,
-        },
-        {
+        }, {
             "name": "模版2",
             "tooltip": "选择器匹配多个 翻页链接/按钮，可根据文本精准匹配",
             "code": `
@@ -289,8 +341,7 @@ document.querySelectorAll("{{.ListNextPageCss}}").forEach(link=>{
 })
 ""
     `,
-        },
-        {
+        }, {
             "name": "模版3",
             "tooltip": "选择器匹配多个 翻页链接/按钮，可根据文本模糊/包含匹配",
             "code": `
@@ -299,8 +350,7 @@ document.querySelectorAll("{{.ListNextPageCss}}").forEach(link=>{
 })
 ""
     `,
-        },
-        {
+        }, {
             "name": "模版4",
             "tooltip": "选择器匹配多个 翻页链接/按钮，可根据检测是否包含子对象匹配",
             "code": `
@@ -308,6 +358,31 @@ document.querySelectorAll("{{.ListNextPageCss}}").forEach(link=>{
   if(link.querySelector("自对象CSS选择器，请修改"))link.click();
 })
 ""
+    `,
+        }, {
+            "name": "模版5",
+            "tooltip": "翻页按钮文本匹配自定义变量",
+            "code": `
+//初始化定义变量
+//var currentPage4Spider=1;''
+currentPage4Spider+=1
+document.querySelectorAll("{{.ListNextPageCss}}").forEach(link=>{
+  if(link.innerText===String(currentPage4Spider))link.click();
+})
+""
+    `,
+        }, {
+            "name": "模版6",
+            "tooltip": "根据URL地址，模板化翻页",
+            "code": `
+var href=window.location.href;
+var pageStr = href.match(/\/\d+).html/)
+var currentPage = 1
+if(pageStr&&pageStr.length>1)currentPage = parseInt(pageStr[1])
+currentPage+=1
+href=""//拼装URL地址
+window.location=href
+''
     `,
         },
     ],

frontend/src/router/index.js

@@ -41,7 +41,7 @@ const routes = [
   },
   {
     path: '/third/collection/list',
-    name: 'reviewList',
+    name: 'collectionList',
     component: CollectionList,
     meta: {
       requiresAuth: false

frontend/src/store/modules/rulesList.js

@@ -3,6 +3,7 @@ import {
   ServerActionGetModifyUsers,
   ServerActionClaimCodes,
   ServerActionUpdateCode,
+  ServerActionQlmRecordList
 } from '../../../wailsjs/go/main/App'
 
 // 爬虫列表模块(局部模块)
@@ -42,6 +43,14 @@ export default {
       })
       return r
     },
+    async getQlmRecordList({ commit }, payload) {
+      const { pageSize, pageNum } = payload
+      const r = await ServerActionQlmRecordList({
+        start: (pageNum - 1) * pageSize,
+        limit: pageSize
+      })
+      return r
+    },
     // 爬虫认领
     async userClaimCodes() {
       const r = await ServerActionClaimCodes()

frontend/src/utils/date.js

@@ -0,0 +1,61 @@
+/*
+ * 时间格式化函数(将时间格式化为，2019年08月12日，2019-08-12，2019/08/12的形式)
+ *   pattern参数（想要什么格式的数据就传入什么格式的数据）
+ *     · 'yyyy-MM-dd'  ---> 输出如2019-09-20
+ *     · 'yyyy-MM-dd HH:mm'  --->  输出如2019-09-20 18:20
+ *     · 'yyyy-MM-dd HH:mm:ss'  --->  输出如2019-09-20 06:20:23
+ *     · 'yyyy/MM/dd'  --->  输出如2019/09/20
+ *     · 'yyyy年MM月dd日'  --->  输出如2019年09月20日
+ *     · 'yyyy年MM月dd日 HH时mm分'  --->  输出如2019年09月20日 18时20分
+ *     · 'yyyy年MM月dd日 HH时mm分ss秒'  --->  输出如2019年09月20日 18时20分23秒
+ *     · 'yyyy年MM月dd日 HH时mm分ss秒 EE'  --->  输出如2019年09月20日 18时20分23秒 周二
+ *     · 'yyyy年MM月dd日 HH时mm分ss秒 EEE'  --->  输出如2019年09月20日 18时20分23秒 星期二
+ *  参考: https://www.cnblogs.com/mr-wuxiansheng/p/6296646.html
+ */
+export function dateFormatter(date = '', fmt = 'yyyy-MM-dd HH:mm:ss') {
+    // 将传入的date转为时间对象
+    if (!date) return ''
+    // 处理ios不兼容'2022-6-6'类似的'-'问题
+    if (typeof data === 'string') {
+        date = date.replace(/-/g, '/')
+    }
+    date = new Date(date)
+    const o = {
+        'y+': date.getFullYear(),
+        'M+': date.getMonth() + 1, // 月份
+        'd+': date.getDate(), // 日
+        // 12小时制
+        'h+': date.getHours() % 12 === 0 ? 12 : date.getHours() % 12, // 小时
+        // 24小时制
+        'H+': date.getHours(), // 小时
+        'm+': date.getMinutes(), // 分
+        's+': date.getSeconds(), // 秒
+        'q+': Math.floor((date.getMonth() + 3) / 3), // 季度
+        S: date.getMilliseconds(), // 毫秒
+        'E+': date.getDay() // 周
+    }
+    const week = ['日', '一', '二', '三', '四', '五', '六']
+
+    if (/(y+)/.test(fmt)) {
+        fmt = fmt.replace(
+            RegExp.$1,
+            (date.getFullYear() + '').substr(4 - RegExp.$1.length)
+        )
+    }
+    if (/(E+)/.test(fmt)) {
+        fmt = fmt.replace(
+            RegExp.$1,
+            (RegExp.$1.length > 1 ? (RegExp.$1.length > 2 ? '星期' : '周') : '') +
+            week[date.getDay()]
+        )
+    }
+    for (const k in o) {
+        if (new RegExp('(' + k + ')').test(fmt)) {
+            fmt = fmt.replace(
+                RegExp.$1,
+                RegExp.$1.length === 1 ? o[k] : ('00' + o[k]).substr(('' + o[k]).length)
+            )
+        }
+    }
+    return fmt
+}

frontend/src/utils/index.js

@@ -1 +1,2 @@
-export * from './str'
+export * from './str'
+export * from './date'

frontend/src/views/CollectionList.vue

@@ -15,18 +15,20 @@
         </el-header>
         <el-main>
             <el-table ref="spiderTable" :data="listState.list" border stripe :row-style="getRowStyle" v-loading="listState.loading">
+                <el-table-column prop="comeintimeText" label="日期" width="110" align="left" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="proxyText" label="代理" align="left" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="headlessText" label="浏览器" align="left" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="imageText" label="图像" align="left" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="listdatanum" label="列表量" align="left" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="needdownloadnum" label="待采量" align="center" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="detaildatanum" label="成功量" align="center" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="pushnum" label="推送量" show-overflow-tooltip></el-table-column>
-                <el-table-column prop="stateText" label="处理状态" show-overflow-tooltip>
+                <el-table-column prop="stateText" label="处理状态" width="100" show-overflow-tooltip>
                     <template #default="scope">
-                        <div class="highlight-main">{{scope.stateText}}</div>
+                        <div class="highlight-main">{{scope.row.stateText}}</div>
                     </template>
                 </el-table-column>
-                <el-table-column label="功能" width="160" align="center">
+                <el-table-column label="功能" min-width="260" align="center">
                     <template #default="scope">
                         <el-tooltip content="列表页采集" placement="top">
                             <el-button size="small" :class="{ active: scope.row._action_clicked_list_collect }" @click="tableEvents.handleListCollect(scope.$index, scope.row)">
@@ -80,7 +82,7 @@
                 </el-radio-group>
             </el-form-item>
             <el-form-item label="显示图像" label-width="100px">
-                <el-radio-group v-model="addNewRecord.form.headless">
+                <el-radio-group v-model="addNewRecord.form.image">
                     <el-radio :value="true">显示</el-radio>
                     <el-radio :value="false">不显示</el-radio>
                 </el-radio-group>
@@ -89,6 +91,7 @@
         <template #footer>
             <div class="dialog-footer">
                 <el-button type="primary" :loading="false" @click="addNewRecordConfirm">确定</el-button>
+                <el-button @click="addNewRecord.dialog = false">取消</el-button>
             </div>
         </template>
     </el-dialog>
@@ -97,13 +100,13 @@
 
 <script setup>
 import { ref, computed, reactive, watch } from 'vue'
-import { useRouter } from 'vue-router';
 import { useStore } from 'vuex';
 import { ElMessage, ElMessageBox } from 'element-plus'
-import { ServerActionQlmRecordList, QlmListDownload, QlmDetailDownload, ServerActionQlmRemoveRepeat, ServerActionQlmPushData, ServerActionQlmClearData } from "../../wailsjs/go/main/App"
+import { ServerActionQlmAddRecord, QlmListDataDownload, QlmDetailDataDownload, ServerActionQlmRemoveRepeat, ServerActionQlmPushData, ServerActionQlmClearData } from "../../wailsjs/go/main/App"
 import Breadcrumb from "../components/Breadcrumb.vue"
 import { USER_ROLE_ADMIN, USER_ROLE_DEVELOPER, USER_ROLE_REVIEWER } from '../data/user'
 import { Refresh, DocumentAdd, Search, Box } from '@element-plus/icons-vue'
+import { dateFormatter } from "../utils/index.js";
 
 const store = useStore();
 const spiderTable = ref(null)
@@ -161,9 +164,9 @@ const addNewRecord = reactive({
     dialog: false,
     loading: false,
     form: {
-        proxy: true,
-        headless: true,
-        image: false,
+        proxy: false,
+        headless: false,
+        image: true,
     }
 })
 
@@ -196,11 +199,12 @@ const getRowStyle = ({ row }) => {
 
 const pushStateMap = {
     0: '未处理',
-    1: '未判重',
-    2: '判重结束',
-    3: '详情采集中',
-    4: '未推送',
-    5: '已完成',
+    1: '列表采集中',
+    2: '未判重',
+    3: '判重结束',
+    4: '详情采集中',
+    5: '未推送',
+    6: '已完成',
 }
 const calcStateText = (state) => {
     return pushStateMap[state] || ''
@@ -209,7 +213,10 @@ const calcStateText = (state) => {
 async function getTableList() {
     listState.loading = true
     try {
-        const r = await ServerActionQlmRecordList()
+        const r = await store.dispatch('rulesList/getQlmRecordList', {
+          pageSize: listState.pageSize,
+          pageNum: listState.pageNum
+        })
         const { data, err, msg } = r
         if (data) {
             console.log(data)
@@ -220,7 +227,9 @@ async function getTableList() {
                         ...t,
                         proxyText: t.proxy ? '使用' : '不使用',
                         headlessText: t.headless ? '无头' : '显示',
+                        imageText: t.image ? '显示' : '不显示',
                         stateText: calcStateText(t.state),
+                        comeintimeText: t.comeintime ? dateFormatter(t.comeintime * 1000, 'yyyy-MM-dd') : '',
                         // 操作按钮是否点击过
                         _action_clicked_list_collect: false,
                         _action_clicked_duplicate_remove: false,
@@ -235,7 +244,6 @@ async function getTableList() {
             }
         }
     } catch (error) {
-        console.log(error)
         listState.loaded = true
     } finally {
         listState.loading = false
@@ -284,28 +292,30 @@ const addNewRecordConfirm = () => {
         ...addNewRecord.form,
     }
     addNewRecord.loading = true
+  console.log(payload)
     ServerActionQlmAddRecord(payload).then(r => {
-        if (r.err === 1) {
-            ElMessage({
-                message: '新增成功',
-                type: 'success',
-                duration: 3000,
-            })
-            addNewRecord.dialog = false
-        } else {
-            return ElMessage({
-                message: r.msg || '新增失败',
-                type: 'error',
-                duration: 3000,
-            })
-        }
-    }).finally(() => {
-        addNewRecord.loading = false
-    })
+          if (r.err === 1) {
+              ElMessage({
+                  message: '新增成功',
+                  type: 'success',
+                  duration: 3000,
+              })
+              addNewRecord.dialog = false
+              refreshTableList()
+          } else {
+              return ElMessage({
+                  message: r.msg || '新增失败',
+                  type: 'error',
+                  duration: 3000,
+              })
+          }
+      }).finally(() => {
+          addNewRecord.loading = false
+      })
 }
 
 const getActionCommonParams = row => {
-    const params = {
+    const param = {
         id: row._id,
         state: row.state
     }
@@ -313,12 +323,29 @@ const getActionCommonParams = row => {
         headless: row.headless,
         showImage: row.image,
         proxyServer: row.proxy,
+        id: row._id,
     }
     return {
-        params,
+        param,
         other
     }
 }
+
+const confirmDialog = (conf = {}) => {
+  if (!conf) return
+  ElMessageBox.confirm(conf.content || '确认操作', '提示',
+      {
+        customClass: conf.customClass || 'j-confirm-message-box',
+        type: 'warning',
+        confirmButtonText: conf.confirmButtonText || '确定',
+        cancelButtonText: conf.cancelButtonText || '取消',
+        showCancelButton: conf.showCancelButton || false,
+      }
+  ).then(() => {
+    conf.success && conf.success()
+  })
+}
+
 // table的按钮事件集合
 const tableEvents = {
     handleListCollect: (index, row) => {
@@ -332,13 +359,9 @@ const tableEvents = {
             })
         }
         loading.value = true
-        const { params, other } = getActionCommonParams(row)
-        params.state = 1
-        const payload = {
-            params,
-            ...other
-        }
-        QlmListDownload(payload).then(r => {
+        const { param, other } = getActionCommonParams(row)
+        param.state = 1
+        QlmListDataDownload(param, other).then(r => {
             if (r.err === 1) {
                 ElMessage({
                     message: '操作成功',
@@ -367,29 +390,30 @@ const tableEvents = {
                 duration: 3000,
             })
         }
-        loading.value = true
-        const { params } = getActionCommonParams(row)
-        params.state = 2
-        const payload = {
-            ...params
-        }
-        ServerActionQlmRemoveRepeat(payload).then(r => {
-            if (r.err === 1) {
+        confirmDialog({
+          success: () => {
+            loading.value = true
+            const { param } = getActionCommonParams(row)
+            param.state = 2
+            ServerActionQlmRemoveRepeat(param).then(r => {
+              if (r.err === 1) {
                 ElMessage({
-                    message: '操作成功',
-                    type: 'success',
-                    duration: 3000,
+                  message: '操作成功',
+                  type: 'success',
+                  duration: 3000,
                 })
                 getTableList()
-            } else {
+              } else {
                 return ElMessage({
-                    message: r.msg || '操作失败',
-                    type: 'error',
-                    duration: 3000,
+                  message: r.msg || '操作失败',
+                  type: 'error',
+                  duration: 3000,
                 })
-            }
-        }).finally(() => {
-            loading.value = false
+              }
+            }).finally(() => {
+              loading.value = false
+            })
+          },
         })
     },
     handleDetailCollect(index, row) {
@@ -403,13 +427,9 @@ const tableEvents = {
             })
         }
         loading.value = true
-        const { params, other } = getActionCommonParams(row)
-        params.state = 4
-        const payload = {
-            params,
-            ...other
-        }
-        QlmDetailDownload(payload).then(r => {
+        const { param, other } = getActionCommonParams(row)
+        param.state = 4
+        QlmDetailDataDownload(param, other).then(r => {
             if (r.err === 1) {
                 ElMessage({
                     message: '操作成功',
@@ -438,54 +458,63 @@ const tableEvents = {
                 duration: 3000,
             })
         }
-        loading.value = true
-        const { params } = getActionCommonParams(row)
-        params.state = 5
-        const payload = {
-            ...params
-        }
-        ServerActionQlmPushData(payload).then(r => {
-            if (r.err === 1) {
+        confirmDialog({
+          success: () => {
+            loading.value = true
+            const { param } = getActionCommonParams(row)
+            param.state = 5
+            const payload = {
+              ...param
+            }
+            ServerActionQlmPushData(payload).then(r => {
+              if (r.err === 1) {
                 ElMessage({
-                    message: '操作成功',
-                    type: 'success',
-                    duration: 3000,
+                  message: '操作成功',
+                  type: 'success',
+                  duration: 3000,
                 })
                 getTableList()
-            } else {
+              } else {
                 return ElMessage({
-                    message: r.msg || '操作失败',
-                    type: 'error',
-                    duration: 3000,
+                  message: r.msg || '操作失败',
+                  type: 'error',
+                  duration: 3000,
                 })
-            }
-        }).finally(() => {
-            loading.value = false
+              }
+            }).finally(() => {
+              loading.value = false
+            })
+          },
         })
     },
     handleRemoveHistory(_, row) {
         onlyClickHighlight(row, '_action_clicked_remove_history')
-        loading.value = true
-        const payload = {
-            id: row._id,
-        }
-        ServerActionQlmClearData(payload).then(r => {
-            if (r.err === 1) {
+
+        confirmDialog({
+          success: () => {
+            loading.value = true
+            const payload = {
+              id: row._id,
+            }
+            ServerActionQlmClearData(payload).then(r => {
+              if (r.err === 1) {
                 ElMessage({
-                    message: '操作成功',
-                    type: 'success',
-                    duration: 3000,
+                  message: '操作成功',
+                  type: 'success',
+                  duration: 3000,
                 })
                 getTableList()
-            } else {
+              } else {
                 return ElMessage({
-                    message: r.msg || '操作失败',
-                    type: 'error',
-                    duration: 3000,
+                  message: r.msg || '操作失败',
+                  type: 'error',
+                  duration: 3000,
                 })
-            }
-        }).finally(() => {
-            loading.value = false
+              }
+            }).finally(() => {
+              loading.value = false
+            })
+          },
         })
     },
 }
@@ -534,6 +563,6 @@ const tableActionDisabled = {
     white-space: nowrap;
 }
 .highlight-main {
-    color: var(--el-color-primary);
+    color: var(--el-color-success);
 }
 </style>

frontend/wailsjs/go/main/App.d.ts

@@ -1,8 +1,8 @@
 // Cynhyrchwyd y ffeil hon yn awtomatig. PEIDIWCH Â MODIWL
 // This file is automatically generated. DO NOT EDIT
 import {backend} from '../models';
-import {list} from '../models';
 import {main} from '../models';
+import {list} from '../models';
 
 export function CountYestodayArts(arg1:string,arg2:number,arg3:number,arg4:boolean,arg5:boolean):Promise<void>;
 
@@ -32,9 +32,9 @@ export function LoadJob(arg1:string):Promise<backend.Job>;
 
 export function LoadSpiderConfigAll(arg1:number,arg2:number):Promise<Array<backend.SpiderConfig>>;
 
-export function QlmDetailDownload():Promise<void>;
+export function QlmDetailDataDownload(arg1:{[key: string]: any},arg2:{[key: string]: any}):Promise<main.Result>;
 
-export function QlmListDownload():Promise<void>;
+export function QlmListDataDownload(arg1:{[key: string]: any},arg2:{[key: string]: any}):Promise<main.Result>;
 
 export function RunExportEpubFile(arg1:string,arg2:string,arg3:list.List):Promise<void>;
 
@@ -64,13 +64,13 @@ export function ServerActionCurrentOpenTab(arg1:{[key: string]: any}):Promise<vo
 
 export function ServerActionGetModifyUsers():Promise<main.Result>;
 
-export function ServerActionQlmClearData(arg1:{[key: string]: any}):Promise<main.Result>;
+export function ServerActionQlmAddRecord(arg1:{[key: string]: any}):Promise<main.Result>;
 
-export function ServerActionQlmNewRecord(arg1:{[key: string]: any}):Promise<main.Result>;
+export function ServerActionQlmClearData(arg1:{[key: string]: any}):Promise<main.Result>;
 
 export function ServerActionQlmPushData(arg1:{[key: string]: any}):Promise<main.Result>;
 
-export function ServerActionQlmRecordList():Promise<main.Result>;
+export function ServerActionQlmRecordList(arg1:{[key: string]: any}):Promise<main.Result>;
 
 export function ServerActionQlmRemoveRepeat(arg1:{[key: string]: any}):Promise<main.Result>;
 

frontend/wailsjs/go/main/App.js

@@ -58,12 +58,12 @@ export function LoadSpiderConfigAll(arg1, arg2) {
   return window['go']['main']['App']['LoadSpiderConfigAll'](arg1, arg2);
 }
 
-export function QlmDetailDownload() {
-  return window['go']['main']['App']['QlmDetailDownload']();
+export function QlmDetailDataDownload(arg1, arg2) {
+  return window['go']['main']['App']['QlmDetailDataDownload'](arg1, arg2);
 }
 
-export function QlmListDownload() {
-  return window['go']['main']['App']['QlmListDownload']();
+export function QlmListDataDownload(arg1, arg2) {
+  return window['go']['main']['App']['QlmListDataDownload'](arg1, arg2);
 }
 
 export function RunExportEpubFile(arg1, arg2, arg3) {
@@ -122,20 +122,20 @@ export function ServerActionGetModifyUsers() {
   return window['go']['main']['App']['ServerActionGetModifyUsers']();
 }
 
-export function ServerActionQlmClearData(arg1) {
-  return window['go']['main']['App']['ServerActionQlmClearData'](arg1);
+export function ServerActionQlmAddRecord(arg1) {
+  return window['go']['main']['App']['ServerActionQlmAddRecord'](arg1);
 }
 
-export function ServerActionQlmNewRecord(arg1) {
-  return window['go']['main']['App']['ServerActionQlmNewRecord'](arg1);
+export function ServerActionQlmClearData(arg1) {
+  return window['go']['main']['App']['ServerActionQlmClearData'](arg1);
 }
 
 export function ServerActionQlmPushData(arg1) {
   return window['go']['main']['App']['ServerActionQlmPushData'](arg1);
 }
 
-export function ServerActionQlmRecordList() {
-  return window['go']['main']['App']['ServerActionQlmRecordList']();
+export function ServerActionQlmRecordList(arg1) {
+  return window['go']['main']['App']['ServerActionQlmRecordList'](arg1);
 }
 
 export function ServerActionQlmRemoveRepeat(arg1) {

main.go

@@ -25,7 +25,7 @@ var (
 	currentSpiderConfig  *be.SpiderConfig = new(be.SpiderConfig)
 	currentResults                        = list.New() //b.ResultItems = make(b.ResultItems, 0)
 	vm                   *bvm.VM
-	qlmVm                *script.GLVm
+	glvm                 *script.GLVm
 	ws                   *bws.WebService
 )
 

qianlima.go

@@ -1,18 +1,63 @@
 package main
 
-// QlmListDownload 千里马列表页数据下载
-func (a *App) QlmListDownload() {
-	//读取列表页lua脚本
-	qlmVm.LoadScript("list")
+import (
+	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+)
+
+// QlmListDataDownload 千里马列表页数据下载
+func (a *App) QlmListDataDownload(param map[string]interface{}, record map[string]interface{}) *Result {
+	qu.Debug(param, record)
+	r := &Result{}
+	if User != nil {
+		getResult(map[string]interface{}{"param": param, "user": User}, r, "qlm/updateRecord")
+		if r.Err == 1 {
+			go DownloadListData(record) //下载
+		}
+	} else {
+		r.Msg = "用户登录异常，请重新登录！"
+		qu.Debug(r.Msg)
+	}
+	return r
+}
+
+// QlmDetailDataDownload 千里马详情页数据下载
+func (a *App) QlmDetailDataDownload(param map[string]interface{}, record map[string]interface{}) *Result {
+	qu.Debug(param, record)
+	r := &Result{}
+	if User != nil {
+		getResult(map[string]interface{}{"param": param, "user": User}, r, "qlm/updateRecord")
+		if r.Err == 1 {
+			go DownloadDetailData(record) //下载
+		}
+	} else {
+		r.Msg = "用户登录异常，请重新登录！"
+		qu.Debug(r.Msg)
+	}
+	return r
 }
 
-// QlmDetailDownload 千里马详情页数据下载
-func (a *App) QlmDetailDownload() {
-	//读取详情页lua脚本
-	qlmVm.LoadScript("detail")
+// DownloadListData 列表页下载
+func DownloadListData(record map[string]interface{}) {
+	scriptText := glvm.LoadScript("list")
+	if scriptText == "" {
+		return
+	}
+	glvm.ProxyServer, _ = record["proxy"].(bool)
+	glvm.Headless, _ = record["headless"].(bool)
+	glvm.ShowImage, _ = record["image"].(bool)
+	glvm.RunScript(scriptText)
+	//采集完成更新记录
+	//go qlmUpdateRecord(map[string]interface{}{"id": record["id"], "state": 2})
 }
 
-// QlmDataSave 采集数据保存
-func QlmDataSave() {
+// DownloadDetailData 详情页下载
+func DownloadDetailData(record map[string]interface{}) {
+	glvm.LoadScript("detail")
+	//采集完成更新记录
+	qlmUpdateRecord(map[string]interface{}{"id": record["id"], "state": 5})
+}
 
+// 更新记录状态
+func qlmUpdateRecord(record map[string]interface{}) {
+	getResult(map[string]interface{}{"param": record}, nil, "qlm/updateRecord")
 }

server.go

@@ -118,13 +118,26 @@ func (a *App) ServerActionUpdateCodeState(param map[string]interface{}) *Result
 	return r
 }
 
-// ServerActionQlmNewRecord 新增千里马采集记录
-func (a *App) ServerActionQlmNewRecord(param map[string]interface{}) *Result {
+// ServerActionQlmRecordList 千里马采集记录
+func (a *App) ServerActionQlmRecordList(param map[string]interface{}) *Result {
+	r := &Result{}
+	//前期校验
+	if User != nil {
+		getResult(map[string]interface{}{"param": param, "user": User}, r, "qlm/recordList")
+	} else {
+		r.Msg = "用户登录异常，请重新登录！"
+		qu.Debug(r.Msg)
+	}
+	return r
+}
+
+// ServerActionQlmAddRecord 新增千里马采集记录
+func (a *App) ServerActionQlmAddRecord(param map[string]interface{}) *Result {
 	qu.Debug("param---", param)
 	r := &Result{}
 	//前期校验
 	if User != nil {
-		getResult(map[string]interface{}{"param": param, "user": User}, r, "newRecord")
+		getResult(map[string]interface{}{"param": param, "user": User}, r, "qlm/newRecord")
 	} else {
 		r.Msg = "用户登录异常，请重新登录！"
 		qu.Debug(r.Msg)
@@ -138,7 +151,7 @@ func (a *App) ServerActionQlmRemoveRepeat(param map[string]interface{}) *Result
 	r := &Result{}
 	//前期校验
 	if User != nil {
-		getResult(map[string]interface{}{"param": param}, r, "removeRepeat")
+		getResult(map[string]interface{}{"param": param}, r, "qlm/removeRepeat")
 	} else {
 		r.Msg = "用户登录异常，请重新登录！"
 		qu.Debug(r.Msg)
@@ -152,7 +165,7 @@ func (a *App) ServerActionQlmPushData(param map[string]interface{}) *Result {
 	r := &Result{}
 	//前期校验
 	if User != nil {
-		getResult(map[string]interface{}{"param": param}, r, "pushData")
+		getResult(map[string]interface{}{"param": param}, r, "qlm/pushData")
 	} else {
 		r.Msg = "用户登录异常，请重新登录！"
 		qu.Debug(r.Msg)
@@ -167,7 +180,7 @@ func (a *App) ServerActionQlmClearData(param map[string]interface{}) *Result {
 	//前期校验
 	if User != nil {
 		qu.Debug("param---", param)
-		getResult(map[string]interface{}{"param": param}, r, "clearData")
+		getResult(map[string]interface{}{"param": param}, r, "qlm/clearData")
 	} else {
 		r.Msg = "用户登录异常，请重新登录！"
 		qu.Debug(r.Msg)
@@ -222,5 +235,4 @@ func getResult(param, result interface{}, route string) {
 		qu.Debug("Error unmarshaling response:", err)
 		return
 	}
-	qu.Debug(result)
 }