feat: 新增千里马采集页面

frontend/src/router/index.js

@@ -5,6 +5,7 @@ import store from '../store';
 // import Home from "../views/Home.vue"
 import CodeList from "../views/CodeList.vue"
 import ReviewList from "../views/ReviewList.vue"
+import CollectionList from "../views/CollectionList.vue"
 import Setting from "../views/Setting.vue"
 import Login from "../views/Login.vue"
 import Logout from '../views/Logout.vue'
@@ -38,6 +39,14 @@ const routes = [
       requiresAuth: true
     }
   },
+  {
+    path: '/third/collection/list',
+    name: 'reviewList',
+    component: CollectionList,
+    meta: {
+      requiresAuth: false
+    }
+  },
   {
     path: '/setting',
     name: 'setting',

frontend/src/store/index.js

@@ -25,6 +25,7 @@ export default createStore({
             [USER_ROLE_ADMIN]: [
                 { title: '爬虫列表', icon: 'List', path: '/code/list' },
                 { title: '审核列表', icon: 'Checked', path: '/review/list' },
+                { title: '千里马采集', icon: 'Checked', path: '/third/collection/list' },
                 // { title: '系统设置', icon: 'Help', path: '/setting' },
             ],
             // 开发者菜单

frontend/src/views/CollectionList.vue

@@ -0,0 +1,539 @@
+<template>
+    <Breadcrumb pageTitle="千里马采集"></Breadcrumb>
+    <div class="space"></div>
+    <el-card v-loading="loading">
+        <el-header>
+            <div class="action-bar-container">
+                <el-space class="action-bar-item-container action-bar-action-left">
+                    <el-button-group class="ml-4">
+                        <!-- <el-button type="primary" :icon="Refresh" @click="resetFilterAndRefreshTableList">刷新</el-button> -->
+                        <el-button type="primary" :icon="DocumentAdd" @click="showAddNewRecordDialog">新增记录</el-button>
+                    </el-button-group>
+                </el-space>
+                <el-space class="action-bar-item-container action-bar-action-right"></el-space>
+            </div>
+        </el-header>
+        <el-main>
+            <el-table ref="spiderTable" :data="listState.list" border stripe :row-style="getRowStyle" v-loading="listState.loading">
+                <el-table-column prop="proxyText" label="代理" align="left" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="headlessText" label="浏览器" align="left" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="listdatanum" label="列表量" align="left" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="needdownloadnum" label="待采量" align="center" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="detaildatanum" label="成功量" align="center" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="pushnum" label="推送量" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="stateText" label="处理状态" show-overflow-tooltip>
+                    <template #default="scope">
+                        <div class="highlight-main">{{scope.stateText}}</div>
+                    </template>
+                </el-table-column>
+                <el-table-column label="功能" width="160" align="center">
+                    <template #default="scope">
+                        <el-tooltip content="列表页采集" placement="top">
+                            <el-button size="small" :class="{ active: scope.row._action_clicked_list_collect }" @click="tableEvents.handleListCollect(scope.$index, scope.row)">
+                                <el-icon><Edit /></el-icon>
+                            </el-button>
+                        </el-tooltip>
+                        <el-tooltip content="去重" placement="top">
+                            <el-button size="small" :class="{ active: scope.row._action_clicked_duplicate_remove }" @click="tableEvents.handleDuplicateRemove(scope.$index, scope.row)">
+                                <el-icon><SetUp /></el-icon>
+                            </el-button>
+                        </el-tooltip>
+                        <el-tooltip content="详情页采集" placement="top">
+                            <el-button size="small" :class="{ active: scope.row._action_clicked_detail_collect }" @click="tableEvents.handleDetailCollect(scope.$index, scope.row)">
+                                <el-icon><Aim /></el-icon>
+                            </el-button>
+                        </el-tooltip>
+                        <el-tooltip content="推送" placement="top">
+                            <el-button size="small" :class="{ active: scope.row._action_clicked_pushed }" @click="tableEvents.handlePushed(scope.$index, scope.row)">
+                                <el-icon><DocumentCopy /></el-icon>
+                            </el-button>
+                        </el-tooltip>
+                        <el-tooltip content="清除记录" placement="top">
+                            <el-button size="small" :class="{ active: scope.row._action_clicked_remove_history }" @click="tableEvents.handleRemoveHistory(scope.$index, scope.row)">
+                                <el-icon><DArrowLeft /></el-icon>
+                            </el-button>
+                        </el-tooltip>
+                    </template>
+                </el-table-column>
+            </el-table>
+            <div class="space"></div>
+            <div class="pagination-container">
+                <el-pagination align="right" @size-change="handleSizeChange" @current-change="handleCurrentChange"
+                    :current-page="listState.pageNum" :page-sizes="[10, 20, 30, 40]" :page-size="listState.pageSize"
+                    layout="total, sizes, prev, pager, next, jumper" :total="listState.total">
+                </el-pagination>
+            </div>
+        </el-main>
+    </el-card>
+    <el-dialog v-model="addNewRecord.dialog" title="新增记录">
+        <el-form>
+            <el-form-item label="配置使用代理" label-width="100px">
+                <el-radio-group v-model="addNewRecord.form.proxy">
+                    <el-radio :value="true">是</el-radio>
+                    <el-radio :value="false">否</el-radio>
+                </el-radio-group>
+            </el-form-item>
+            <el-form-item label="无头浏览器" label-width="100px">
+                <el-radio-group v-model="addNewRecord.form.headless">
+                    <el-radio :value="true">是</el-radio>
+                    <el-radio :value="false">否</el-radio>
+                </el-radio-group>
+            </el-form-item>
+            <el-form-item label="显示图像" label-width="100px">
+                <el-radio-group v-model="addNewRecord.form.headless">
+                    <el-radio :value="true">显示</el-radio>
+                    <el-radio :value="false">不显示</el-radio>
+                </el-radio-group>
+            </el-form-item>
+        </el-form>
+        <template #footer>
+            <div class="dialog-footer">
+                <el-button type="primary" :loading="false" @click="addNewRecordConfirm">确定</el-button>
+            </div>
+        </template>
+    </el-dialog>
+    <div></div>
+</template>
+
+<script setup>
+import { ref, computed, reactive, watch } from 'vue'
+import { useRouter } from 'vue-router';
+import { useStore } from 'vuex';
+import { ElMessage, ElMessageBox } from 'element-plus'
+import { ServerActionQlmRecordList, QlmListDownload, QlmDetailDownload, ServerActionQlmRemoveRepeat, ServerActionQlmPushData, ServerActionQlmClearData } from "../../wailsjs/go/main/App"
+import Breadcrumb from "../components/Breadcrumb.vue"
+import { USER_ROLE_ADMIN, USER_ROLE_DEVELOPER, USER_ROLE_REVIEWER } from '../data/user'
+import { Refresh, DocumentAdd, Search, Box } from '@element-plus/icons-vue'
+
+const store = useStore();
+const spiderTable = ref(null)
+
+const loading = ref(false)
+
+// 选择器数据
+const filters = reactive({
+    // 搜索
+    search: '',
+    // 爬虫状态
+    state: -1,
+    // 维护人
+    modifyuser: '-1',
+    // 认领状态
+    claimtype: -1,
+})
+// 选择器数据(用来重置)
+const defaultFilters = {
+    search: '',
+    state: -1,
+    modifyuser: '-1',
+    claimtype: -1,
+}
+
+// 列表数据
+const listState = reactive({
+    loaded: false,
+    loading: false,
+    pageNum: 1, // 页码
+    pageSize: 10, // 每页多少条
+    total: 0, // 返回的总数据条数
+    list: [ // 数据列表
+        // {
+        //     code: '爬虫代码', 
+        //     site: '网站',
+        //     channel: '栏目',
+        //     href: '栏目地址',
+        //     modifyuser: '维护人',
+        //     _id: 'asfasf',
+        // },
+    ],
+})
+// 列表默认数据(用来重置)
+const defaultListState = {
+    loaded: false,
+    loading: false,
+    pageNum: 1,
+    pageSize: 10,
+    total: 0,
+    list: [],
+}
+
+const addNewRecord = reactive({
+    dialog: false,
+    loading: false,
+    form: {
+        proxy: true,
+        headless: true,
+        image: false,
+    }
+})
+
+// 当前编辑的row的数据
+const currentEditRow = ref({})
+// 上一个点击的row的数据
+const prevClickedRow = ref({})
+
+// 用户身份标识
+const userRole = computed(() => store.getters.userRole)
+
+// 取消prevClicked的高亮
+const cancelOtherHighlight = () => {
+    for (const key in prevClickedRow.value) {
+        if (key.includes('_action_clicked')) {
+            prevClickedRow.value[key] = false
+        }
+    }
+}
+// 页面按钮点击高亮
+const onlyClickHighlight = (row, key) => {
+    cancelOtherHighlight()
+    row[key] = true
+    prevClickedRow.value = row
+}
+
+const getRowStyle = ({ row }) => {
+    return row.selected ? { backgroundColor: '#F7F7F7' } : {};
+};
+
+const pushStateMap = {
+    0: '未处理',
+    1: '未判重',
+    2: '判重结束',
+    3: '详情采集中',
+    4: '未推送',
+    5: '已完成',
+}
+const calcStateText = (state) => {
+    return pushStateMap[state] || ''
+}
+// 获取列表数据
+async function getTableList() {
+    listState.loading = true
+    try {
+        const r = await ServerActionQlmRecordList()
+        const { data, err, msg } = r
+        if (data) {
+            console.log(data)
+            listState.total = data.total || 0
+            if (Array.isArray(data.list)) {
+                const sList = data.list.map(t => {
+                    return {
+                        ...t,
+                        proxyText: t.proxy ? '使用' : '不使用',
+                        headlessText: t.headless ? '无头' : '显示',
+                        stateText: calcStateText(t.state),
+                        // 操作按钮是否点击过
+                        _action_clicked_list_collect: false,
+                        _action_clicked_duplicate_remove: false,
+                        _action_clicked_detail_collect: false,
+                        _action_clicked_pushed: false,
+                        _action_clicked_remove_history: false,
+                        // _action_clicked_submit: false,
+                    }
+                })
+
+                listState.list = sList || []
+            }
+        }
+    } catch (error) {
+        console.log(error)
+        listState.loaded = true
+    } finally {
+        listState.loading = false
+    }
+}
+
+// 重置列表数据
+const resetListState = () => {
+    Object.assign(listState, defaultListState)
+}
+// 重置选择器数据
+const resetFilterState = () => {
+    Object.assign(filters, defaultFilters)
+}
+
+// 刷新列表(不重置选择器)
+function refreshTableList() {
+    resetListState()
+    getTableList()
+}
+// 刷新列表(并重置选择器)
+function resetFilterAndRefreshTableList() {
+    resetFilterState()
+    refreshTableList()
+}
+
+getTableList()
+
+const handleSizeChange = (val) => {
+    listState.pageSize = val;
+    listState.pageNum = 1;
+    getTableList()
+};
+
+const handleCurrentChange = (val) => {
+    listState.pageNum = val;
+    getTableList()
+};
+
+
+const showAddNewRecordDialog = () => {
+    addNewRecord.dialog = true
+}
+const addNewRecordConfirm = () => {
+    const payload = {
+        ...addNewRecord.form,
+    }
+    addNewRecord.loading = true
+    ServerActionQlmAddRecord(payload).then(r => {
+        if (r.err === 1) {
+            ElMessage({
+                message: '新增成功',
+                type: 'success',
+                duration: 3000,
+            })
+            addNewRecord.dialog = false
+        } else {
+            return ElMessage({
+                message: r.msg || '新增失败',
+                type: 'error',
+                duration: 3000,
+            })
+        }
+    }).finally(() => {
+        addNewRecord.loading = false
+    })
+}
+
+const getActionCommonParams = row => {
+    const params = {
+        id: row._id,
+        state: row.state
+    }
+    const other = {
+        headless: row.headless,
+        showImage: row.image,
+        proxyServer: row.proxy,
+    }
+    return {
+        params,
+        other
+    }
+}
+// table的按钮事件集合
+const tableEvents = {
+    handleListCollect: (index, row) => {
+        onlyClickHighlight(row, '_action_clicked_list_collect')
+        const pass = row.state === 0
+        if (!pass) {
+            return ElMessage({
+                message: '请先完成上一步操作！',
+                type: 'error',
+                duration: 3000,
+            })
+        }
+        loading.value = true
+        const { params, other } = getActionCommonParams(row)
+        params.state = 1
+        const payload = {
+            params,
+            ...other
+        }
+        QlmListDownload(payload).then(r => {
+            if (r.err === 1) {
+                ElMessage({
+                    message: '操作成功',
+                    type: 'success',
+                    duration: 3000,
+                })
+                getTableList()
+            } else {
+                return ElMessage({
+                    message: r.msg || '操作失败',
+                    type: 'error',
+                    duration: 3000,
+                })
+            }
+        }).finally(() => {
+            loading.value = false
+        })
+    },
+    handleDuplicateRemove(index, row) {
+        onlyClickHighlight(row, '_action_clicked_duplicate_remove')
+        const pass = row.state === 2 || row.state === 3
+        if (!pass) {
+            return ElMessage({
+                message: '请先完成上一步操作！',
+                type: 'error',
+                duration: 3000,
+            })
+        }
+        loading.value = true
+        const { params } = getActionCommonParams(row)
+        params.state = 2
+        const payload = {
+            ...params
+        }
+        ServerActionQlmRemoveRepeat(payload).then(r => {
+            if (r.err === 1) {
+                ElMessage({
+                    message: '操作成功',
+                    type: 'success',
+                    duration: 3000,
+                })
+                getTableList()
+            } else {
+                return ElMessage({
+                    message: r.msg || '操作失败',
+                    type: 'error',
+                    duration: 3000,
+                })
+            }
+        }).finally(() => {
+            loading.value = false
+        })
+    },
+    handleDetailCollect(index, row) {
+        onlyClickHighlight(row, '_action_clicked_detail_collect')
+        const pass = row.state === 3 || row.state === 5
+        if (!pass) {
+            return ElMessage({
+                message: '请先完成上一步操作！',
+                type: 'error',
+                duration: 3000,
+            })
+        }
+        loading.value = true
+        const { params, other } = getActionCommonParams(row)
+        params.state = 4
+        const payload = {
+            params,
+            ...other
+        }
+        QlmDetailDownload(payload).then(r => {
+            if (r.err === 1) {
+                ElMessage({
+                    message: '操作成功',
+                    type: 'success',
+                    duration: 3000,
+                })
+                getTableList()
+            } else {
+                return ElMessage({
+                    message: r.msg || '操作失败',
+                    type: 'error',
+                    duration: 3000,
+                })
+            }
+        }).finally(() => {
+            loading.value = false
+        })
+    },
+    handlePushed(index, row) {
+        onlyClickHighlight(row, '_action_clicked_pushed')
+        const pass = row.state === 5
+        if (!pass) {
+            return ElMessage({
+                message: '请先完成上一步操作！',
+                type: 'error',
+                duration: 3000,
+            })
+        }
+        loading.value = true
+        const { params } = getActionCommonParams(row)
+        params.state = 5
+        const payload = {
+            ...params
+        }
+        ServerActionQlmPushData(payload).then(r => {
+            if (r.err === 1) {
+                ElMessage({
+                    message: '操作成功',
+                    type: 'success',
+                    duration: 3000,
+                })
+                getTableList()
+            } else {
+                return ElMessage({
+                    message: r.msg || '操作失败',
+                    type: 'error',
+                    duration: 3000,
+                })
+            }
+        }).finally(() => {
+            loading.value = false
+        })
+    },
+    handleRemoveHistory(_, row) {
+        onlyClickHighlight(row, '_action_clicked_remove_history')
+        loading.value = true
+        const payload = {
+            id: row._id,
+        }
+        ServerActionQlmClearData(payload).then(r => {
+            if (r.err === 1) {
+                ElMessage({
+                    message: '操作成功',
+                    type: 'success',
+                    duration: 3000,
+                })
+                getTableList()
+            } else {
+                return ElMessage({
+                    message: r.msg || '操作失败',
+                    type: 'error',
+                    duration: 3000,
+                })
+            }
+        }).finally(() => {
+            loading.value = false
+        })
+    },
+}
+
+// 表格按钮是否可用
+const tableActionDisabled = {
+    // 此按钮是开发者才会展示
+    submitDisabled(row) {
+        // 开发者-爬虫状态是待完成，才可点击提交
+        const waitingComplete = row.state === 0
+        // 开发者-未通过的爬虫可点击提交
+        const notPass = row.state === 2
+        const canSubmit = waitingComplete || notPass
+        return !canSubmit
+    },
+}
+</script>
+
+<style lang="scss" scoped>
+::v-deep {
+    .el-button {
+        &.active {
+            background-color: var(--el-button-hover-bg-color);
+            border-color: var(--el-button-hover-border-color);
+            color: var(--el-button-hover-text-color);
+            outline: none;
+        }
+    }
+}
+
+.pagination-container {
+    display: flex;
+    align-items: center;
+    justify-content: flex-end;
+}
+.action-bar-container {
+    display: flex;
+    justify-content: space-between;
+}
+.action-bar-item {
+    display: flex;
+    align-items: center;
+}
+.action-bar-name {
+    font-size: 14px;
+    white-space: nowrap;
+}
+.highlight-main {
+    color: var(--el-color-primary);
+}
+</style>

frontend/wailsjs/go/main/App.d.ts

@@ -32,6 +32,10 @@ export function LoadJob(arg1:string):Promise<backend.Job>;
 
 export function LoadSpiderConfigAll(arg1:number,arg2:number):Promise<Array<backend.SpiderConfig>>;
 
+export function QlmDetailDownload():Promise<void>;
+
+export function QlmListDownload():Promise<void>;
+
 export function RunExportEpubFile(arg1:string,arg2:string,arg3:list.List):Promise<void>;
 
 export function RunExportExcelFile(arg1:string,arg2:string,arg3:list.List):Promise<void>;
@@ -60,6 +64,16 @@ export function ServerActionCurrentOpenTab(arg1:{[key: string]: any}):Promise<vo
 
 export function ServerActionGetModifyUsers():Promise<main.Result>;
 
+export function ServerActionQlmClearData(arg1:{[key: string]: any}):Promise<main.Result>;
+
+export function ServerActionQlmNewRecord(arg1:{[key: string]: any}):Promise<main.Result>;
+
+export function ServerActionQlmPushData(arg1:{[key: string]: any}):Promise<main.Result>;
+
+export function ServerActionQlmRecordList():Promise<main.Result>;
+
+export function ServerActionQlmRemoveRepeat(arg1:{[key: string]: any}):Promise<main.Result>;
+
 export function ServerActionUpdateCode(arg1:{[key: string]: any}):Promise<main.Result>;
 
 export function ServerActionUpdateCodeState(arg1:{[key: string]: any}):Promise<main.Result>;

frontend/wailsjs/go/main/App.js

@@ -58,6 +58,14 @@ export function LoadSpiderConfigAll(arg1, arg2) {
   return window['go']['main']['App']['LoadSpiderConfigAll'](arg1, arg2);
 }
 
+export function QlmDetailDownload() {
+  return window['go']['main']['App']['QlmDetailDownload']();
+}
+
+export function QlmListDownload() {
+  return window['go']['main']['App']['QlmListDownload']();
+}
+
 export function RunExportEpubFile(arg1, arg2, arg3) {
   return window['go']['main']['App']['RunExportEpubFile'](arg1, arg2, arg3);
 }
@@ -114,6 +122,26 @@ export function ServerActionGetModifyUsers() {
   return window['go']['main']['App']['ServerActionGetModifyUsers']();
 }
 
+export function ServerActionQlmClearData(arg1) {
+  return window['go']['main']['App']['ServerActionQlmClearData'](arg1);
+}
+
+export function ServerActionQlmNewRecord(arg1) {
+  return window['go']['main']['App']['ServerActionQlmNewRecord'](arg1);
+}
+
+export function ServerActionQlmPushData(arg1) {
+  return window['go']['main']['App']['ServerActionQlmPushData'](arg1);
+}
+
+export function ServerActionQlmRecordList() {
+  return window['go']['main']['App']['ServerActionQlmRecordList']();
+}
+
+export function ServerActionQlmRemoveRepeat(arg1) {
+  return window['go']['main']['App']['ServerActionQlmRemoveRepeat'](arg1);
+}
+
 export function ServerActionUpdateCode(arg1) {
   return window['go']['main']['App']['ServerActionUpdateCode'](arg1);
 }