功能调整

backend/vm/check.go

@@ -97,10 +97,17 @@ import (
 //		}
 //		qu.Debug("列表采集条数结果：", verifyResult.Len())
 //		//TODO 6.翻页
-//		if err = trunPage(sc, 2000, ctx); err != nil {
-//			//if err = trunPage(sc, sc.ListTurnDelayTime, ctx); err != nil {
-//			ret.ListTrunPage = false
-//			break T
+//		if verifyResult.Len() > 0 {
+//			if sc.MaxPages == 1 { //最大页为1，不校验翻页
+//				ret.ListTrunPage = true
+//				break
+//			} else if sc.MaxPages > 1 { //&& !ret.ListTrunPage {
+//				if err = trunPage(sc, sc.ListTurnDelayTime, ctx); err != nil { //翻页失败
+//					break T
+//				} else {
+//					ret.ListTrunPage = true
+//				}
+//			}
 //		}
 //	}
 //	//检查
@@ -124,7 +131,7 @@ import (
 //	}
 //	qu.Debug(verifyResult.Len())
 //	if ret.ListItems {
-//		ret.ListItems = verifyResult.Len() > 2
+//		ret.ListItems = (sc.MaxPages == 1 && verifyResult.Len() > 0) || (sc.MaxPages > 1 && verifyResult.Len() > 2)
 //	}
 //
 //	//TODO:每次验证结果存库、内存？
@@ -174,7 +181,7 @@ T:
 		//TODO 5.操作详情页
 		qu.Debug("列表采集条数：", len(listResult))
 		for contentIndex, r := range listResult {
-			if contentIndex > 1 { //每页只采集2条
+			if contentIndex > 1 { //每页只校验2条
 				break
 			}
 			qu.Debug("当前列表页第" + fmt.Sprint(contentIndex+1) + "条")
@@ -195,11 +202,16 @@ T:
 		}
 		qu.Debug("列表采集条数结果：", verifyResult.Len())
 		//TODO 6.翻页
-		if len(listResult) > 0 && !ret.ListTrunPage {
-			if err = trunPage(sc, sc.ListTurnDelayTime, ctx); err != nil { //翻页失败
-				break T
-			} else {
+		if verifyResult.Len() > 0 {
+			if sc.MaxPages == 1 { //最大页为1，不校验翻页
 				ret.ListTrunPage = true
+				break
+			} else if sc.MaxPages > 1 { // && !ret.ListTrunPage {
+				if err = trunPage(sc, sc.ListTurnDelayTime, ctx); err != nil { //翻页失败
+					break T
+				} else {
+					ret.ListTrunPage = true
+				}
 			}
 		}
 	}
@@ -211,9 +223,8 @@ T:
 		qu.Debug("Check PublishTime:", ret.PublishTime, r.PublishTime, r.ListPubTime)
 		ret.PublishTime = r.PublishTime != ""
 	}
-	qu.Debug(verifyResult.Len())
 	if ret.ListItems {
-		ret.ListItems = verifyResult.Len() > 2
+		ret.ListItems = (sc.MaxPages == 1 && verifyResult.Len() > 0) || (sc.MaxPages > 1 && verifyResult.Len() > 2)
 	}
 
 	return ret, nil

backend/vm/single.go

@@ -74,12 +74,13 @@ func (vm *VM) RunSpiderTmp(url string, maxPages int, listDealy, trunPageDelay, c
 			case <-exit:
 				return
 			default:
+				qu.Debug(v.No, v.ListTitle, v.Href, v.Title, v.PublishTime, v.ListPubTime)
 				v.Site = sc.Site
 				v.Channel = sc.Channel
 				v.Title = v.ListTitle
 				v.Content = "详见正文"
+				v.PublishTime = v.ListPubTime
 				v.No = no
-				qu.Debug(v.No, v.ListTitle, v.Href)
 				no++
 				currentResult.PushBack(v)
 			}

frontend/src/components/spider/jscodetpl.js

@@ -10,7 +10,7 @@ document.querySelectorAll("{{.ListItemCss}}").forEach((v, i) => {
             var href = link.href
             if (!href.startsWith("http")) href = window.location.origin + "/" + href
             let title = link.getAttribute("title") || link.innerText
-            item = { "title": title, "href": href, "no": i }
+            item = { "listTitle": title, "href": href, "no": i }
         } else {
             item = { "no": i }
         }
@@ -18,7 +18,7 @@ document.querySelectorAll("{{.ListItemCss}}").forEach((v, i) => {
     if ("{{.ListPubtimeCss}}" != "") {
         let pubtime = v.querySelector("{{.ListPubtimeCss}}")
         if (pubtime) {
-            item["pubtime"] = pubtime.innerText
+            item["listPublishTime"] = pubtime.innerText
         }
     }
     ret.push(item)
@@ -67,7 +67,7 @@ ret
 `,
     //列表页翻页代码
     ListTurnPageJsCode: `
-var link=document.querySelector("{{.ListNextPageCss}}");if(link)link.click();
+var link=document.querySelector("{{.ListNextPageCss}}");if(link)link.click();"";
 `
 
 }

frontend/src/views/CodeList.vue

@@ -56,9 +56,9 @@
         </el-header>
         <el-main>
             <el-table ref="spiderTable" :data="listState.list" stripe :row-style="getRowStyle" v-loading="listState.loading">
-                <el-table-column prop="site" label="网站" align="center" show-overflow-tooltip></el-table-column>
-                <el-table-column prop="channel" label="栏目" align="center" show-overflow-tooltip></el-table-column>
-                <el-table-column prop="code" label="代码" align="center" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="site" label="网站" align="left" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="channel" label="栏目" align="left" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="code" label="代码" align="left" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="stateText" label="爬虫状态" width="80" align="center" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="claimText" label="认领状态" width="100" align="center" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="href" label="栏目地址" show-overflow-tooltip></el-table-column>

frontend/src/views/ReviewList.vue

@@ -9,7 +9,7 @@
                         <!-- <el-button type="primary" :icon="Refresh" @click="resetFilterAndRefreshTableList">刷新</el-button> -->
                         <el-button type="primary" :icon="Box" @click="doBatchRollback" v-if="showBatchListing">批量打回</el-button>
                         <el-button type="primary" :icon="Box" @click="doBatchListing" v-if="showBatchListing">批量上线</el-button>
-                        <el-button type="primary" :icon="Box" @click="doUnBatchListing" v-if="showBatchListing">批量下线</el-button>
+                        <el-button type="primary" :icon="Box" @click="doUnBatchListing" v-if="showBatchListing">批量下架</el-button>
                     </el-button-group>
                 </el-space>
                 <el-space class="action-bar-item-container action-bar-action-right">
@@ -59,9 +59,9 @@
         <el-main>
             <el-table ref="spiderTable" :data="listState.list" stripe :row-style="getRowStyle" @selection-change="handleSelectionChange" v-loading="listState.loading">
                 <el-table-column type="selection" width="55" :selectable="tableRowSelectable" v-if="isAdmin" />
-                <el-table-column prop="site" label="网站" align="center" show-overflow-tooltip></el-table-column>
-                <el-table-column prop="channel" label="栏目" align="center" show-overflow-tooltip></el-table-column>
-                <el-table-column prop="code" label="代码" align="center" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="site" label="网站" align="left" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="channel" label="栏目" align="left" show-overflow-tooltip></el-table-column>
+                <el-table-column prop="code" label="代码" align="left" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="stateText" label="爬虫状态" width="80" align="center" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="claimText" label="认领状态" width="100" align="center" show-overflow-tooltip></el-table-column>
                 <el-table-column prop="href" label="栏目地址" show-overflow-tooltip></el-table-column>
@@ -94,9 +94,9 @@
                     <template #default="scope">
                         <!-- 管理员：上线和退回 -->
                         <template v-if="tableActionShow.adminGroup(scope.row)">
-                            <el-tooltip content="下线" placement="top" v-if="tableActionShow.downCode(scope.row)">
+                            <el-tooltip content="下架" placement="top" v-if="tableActionShow.downCode(scope.row)">
                                 <el-button size="small" :class="{ active: scope.row._action_clicked_down_code }" :disabled="actionButtonDisabled.adminDownCode(scope.row)" @click="tableEvents.adminDownCode(scope.$index, scope.row)">
-                                    <el-icon><Download /></el-icon>
+                                  <el-icon><CircleCloseFilled /></el-icon>
                                 </el-button>
                             </el-tooltip>
                             <el-tooltip content="上线" placement="top" v-if="tableActionShow.adminSubmit(scope.row)">
@@ -116,8 +116,8 @@
                                 <el-icon><CircleCheck /></el-icon>
                             </el-button>
                         </el-tooltip>
-                        <el-tooltip content="打回" placement="top">
-                            <el-button size="small" :class="{ active: scope.row._action_clicked_rollback }" :disabled="actionButtonDisabled.reviewerRollback(scope.row)" @click="tableEvents.reviewerRollback(scope.$index, scope.row)">
+                        <el-tooltip content="打回" placement="top" v-if="!actionButtonDisabled.reviewerRollback(scope.row)">
+                            <el-button size="small" :class="{ active: scope.row._action_clicked_rollback }" @click="tableEvents.reviewerRollback(scope.$index, scope.row)">
                                 <el-icon><CircleClose /></el-icon>
                             </el-button>
                         </el-tooltip>
@@ -460,9 +460,9 @@ const openEditDialog = (row) => {
 // 操作按钮组的显示隐藏控制
 const tableActionShow = {
     reviewerGroup(row) {
-        // 已通过、已下线和已上线的不展示
+        // 已通过、已下架和已上线的不展示
         const canOnline = row.state === 3 // 已通过
-        const alreadyOffline = row.state === 6 // 已下线
+        const alreadyOffline = row.state === 6 // 已下架
         const alreadyOnline = row.state === 11 // 已上线
         const pass = canOnline || alreadyOffline || alreadyOnline
         return !pass
@@ -475,12 +475,12 @@ const tableActionShow = {
         }
     },
     adminSubmit(row) {
-        // 只有审核通过或者下线的才能上线，否则不展示
+        // 只有审核通过或者下架的才能上线，否则不展示
         const canOnline = row.state === 3 || row.state === 6
         return canOnline
     },
     downCode(row) {
-        // 下线按钮，只有已上线爬虫展示
+        // 下架按钮，只有已上线爬虫展示
         return row.state === 11
     },
     adminRollback(row) {
@@ -698,7 +698,7 @@ const tableEvents = {
         })
     },
     // 上线和批量上线操作 stateText=up
-    // 下线和批量下线操作 stateText=down
+    // 下架和批量下架操作 stateText=down
     batchListing(list = [], stateText = 'up') {
         const stateMap = {
             up: {
@@ -707,7 +707,7 @@ const tableEvents = {
             },
             down: {
                 code: 6,
-                text: '下线',
+                text: '下架',
             },
         }
         const stateInfo = stateMap[stateText]
@@ -807,10 +807,10 @@ const tableEvents = {
             this.batchListing([row], 'up')
         })
     },
-    // 管理控制下线
+    // 管理控制下架
     adminDownCode(_, row) {
         onlyClickHighlight(row, '_action_clicked_down_code')
-        ElMessageBox.confirm('确认下线？', '提示',
+        ElMessageBox.confirm('确认下架？', '提示',
             {
                 customClass: 'j-confirm-message-box',
                 type: 'warning',
@@ -857,7 +857,7 @@ const toggleRow = (rowList = []) => {
 }
 
 const tableRowSelectable = (row) => {
-    // 待审核(管理员的批量打回)、已通过(管理员的批量上线)、已下架的批量上线或者已上线(管理员的批量下线)的才能选
+    // 待审核(管理员的批量打回)、已通过(管理员的批量上线)、已下架的批量上线或者已上线(管理员的批量下架)的才能选
     return row.state === 1 || row.state === 3 || row.state === 6 || row.state === 11
 }
 
@@ -912,7 +912,7 @@ const confirmBatchListing = () => {
     toggleRow(dropList)
     if (list.length <= 0) {
         return ElMessage({
-            message: '至少要选中1条有效数据: 当前未选中已通过或已下线爬虫',
+            message: '至少要选中1条有效数据: 当前未选中已通过或已下架爬虫',
             type: 'error',
             duration: 3000,
         })
@@ -934,7 +934,7 @@ const doBatchListing = () => {
     })
 }
 
-// 执行批量下线操作
+// 执行批量下架操作
 const confirmBatchUnListing = () => {
     // 过滤一下，找到已上线爬虫，仅提交此类爬虫
     const list = listState.selected.filter(v => v.state === 11)
@@ -950,7 +950,7 @@ const confirmBatchUnListing = () => {
     tableEvents.batchListing(list, 'down')
 }
 const doUnBatchListing = () => {
-    ElMessageBox.confirm('确认批量下线？', '提示',
+    ElMessageBox.confirm('确认批量下架？', '提示',
         {
             customClass: 'j-confirm-message-box',
             type: 'warning',

server.go

@@ -30,41 +30,51 @@ type Data struct {
 // ServerActionCodeList 获取爬虫列表
 func (a *App) ServerActionCodeList(param map[string]interface{}) *Result {
 	qu.Debug("param---", param)
-	data := &Result{}
+	r := &Result{}
 	if User != nil {
-		getResult(map[string]interface{}{"param": param, "user": User}, data, "getcodes")
+		getResult(map[string]interface{}{"param": param, "user": User}, r, "getcodes")
 	} else {
-		data.Msg = "用户登录异常，请重新登录！"
-		qu.Debug(data.Msg)
+		r.Msg = "用户登录异常，请重新登录！"
+		qu.Debug(r.Msg)
 	}
-	return data
+	return r
 }
 
 // ServerActionClaimCodes 爬虫认领
 func (a *App) ServerActionClaimCodes() *Result {
-	data := &Result{}
+	r := &Result{}
 	if User != nil && User.Auth == 1 {
-		getResult(User, data, "claimcode")
+		getResult(User, r, "claimcode")
 	} else {
-		data.Msg = "认领失败！"
+		r.Msg = "认领失败！"
 	}
-	return data
+	return r
+}
+
+// ServerActionCopyCode 复制爬虫
+func (a *App) ServerActionCopyCode(param map[string]interface{}) *Result {
+	r := &Result{}
+	if User != nil && User.Auth == 1 {
+		getResult(map[string]interface{}{"param": param, "user": User}, r, "copycode")
+	} else {
+		r.Msg = "复制爬虫失败！"
+	}
+	return r
 }
 
 // ServerActionUpdateCode 爬虫更新
 func (a *App) ServerActionUpdateCode(param map[string]interface{}) *Result {
 	qu.Debug("param---", param)
-	data := &Result{}
+	r := &Result{}
 	if User != nil {
 		//formatUser(param)
 		qu.Debug("param---", param)
-		getResult(map[string]interface{}{"param": param, "user": User}, data, "updatecode")
+		getResult(map[string]interface{}{"param": param, "user": User}, r, "updatecode")
 	} else {
-		data.Msg = "用户登录异常，请重新登录！"
-		qu.Debug(data.Msg)
+		r.Msg = "用户登录异常，请重新登录！"
+		qu.Debug(r.Msg)
 	}
-	qu.Debug(*data)
-	return data
+	return r
 }
 
 // ServerActionUpdateCode 爬虫状态更新
@@ -77,7 +87,7 @@ func (a *App) ServerActionUpdateCodeState(param map[string]interface{}) *Result
 		4、审核通过
 		5、上线
 	*/
-	data := &Result{}
+	r := &Result{}
 	//前期校验
 	if User != nil {
 		//提交审核时，验证校验清单
@@ -85,26 +95,26 @@ func (a *App) ServerActionUpdateCodeState(param map[string]interface{}) *Result
 		if qu.IntAll(lua["state"]) == 1 {
 			code := qu.ObjToString(lua["code"])
 			if vr := be.VerifyResults[code]; vr == nil { //没有检验清单，不允许提交
-				data.Msg = "未验证，无法提交！"
+				r.Msg = "未验证，无法提交！"
 				//} else if !vr.ListItems || !vr.Content || !vr.Title || !vr.PublishTime { //校验检验清单必通过项
 			} else if !vr.ListItems || !vr.Content || !vr.Title || !vr.PublishTime || !vr.ListTrunPage { //校验检验清单必通过项
-				data.Msg = "验证清单未通过！"
+				r.Msg = "验证清单未通过！"
 			} else {
 				lua["verify"] = vr
+				be.VerifyResults[code] = nil //清空验证结果
 			}
 		}
 	} else {
-		data.Msg = "用户登录异常，请重新登录！"
-		qu.Debug(data.Msg)
+		r.Msg = "用户登录异常，请重新登录！"
+		qu.Debug(r.Msg)
 	}
-	if data.Msg == "" {
+	if r.Msg == "" {
 		param["user"] = User
 		qu.Debug("param---", param)
 		//formatUser(param)
-		getResult(param, data, "updatecodestate")
+		getResult(param, r, "updatecodestate")
 	}
-	qu.Debug(*data)
-	return data
+	return r
 }
 
 // 格式化User对象