插件集成大模型

backend/vm/check.go

@@ -48,8 +48,9 @@ func (vm *VM) VerifySpiderConfig(sc *be.SpiderConfig) (*be.SpiderConfigVerifyRes
 		return ret, errors.New("初始化列表页失败")
 	}
 	no := 1
+	ret.ListTrunPage = true
 T:
-	for j := 0; j < 2; j++ { //最多检查2页
+	for j := 0; j < VERIVY_MAX_TRUN_PAGE && j < int(sc.MaxPages); j++ { //最多检查2页
 		qu.Debug("开始检查第" + fmt.Sprint(j+1) + "页...")
 		listResult := make(be.ResultItems, 0)
 		err := chromedp.Run(ctx, chromedp.Tasks{
@@ -105,18 +106,16 @@ T:
 		qu.Debug("第"+fmt.Sprint(j+1)+"页校验成功数据条数：", verifyResult.Len())
 		//翻页
 		if verifyResult.Len() > 0 {
-			if sc.MaxPages == 1 { //最大页为1，不校验翻页
-				ret.ListTrunPage = true
-				break
-			} else if sc.MaxPages > 1 { //&& !ret.ListTrunPage {
+			if sc.MaxPages > 1 && j < VERIVY_MAX_TRUN_PAGE-1 && j < int(sc.MaxPages)-1 { //&& !ret.ListTrunPage {
 				if err = trunPage(sc, sc.ListTurnDelayTime, ctx); err != nil { //翻页失败
 					qu.Debug("第" + fmt.Sprint(j+1) + "页翻页失败")
 					ret.ListTrunPage = false
 					break T
-				} else {
-					ret.ListTrunPage = true
 				}
 			}
+		} else {
+			ret.ListTrunPage = false
+			break T
 		}
 	}
 	//检查

backend/vm/vm.go

@@ -22,7 +22,8 @@ import (
 )
 
 const (
-	MAX_TRUN_PAGE = 1000
+	MAX_TRUN_PAGE        = 1000
+	VERIVY_MAX_TRUN_PAGE = 3
 )
 
 type (

backend/webservice/webservice.go

@@ -2,19 +2,24 @@
 package webservice
 
 import (
+	"bytes"
 	"crypto/tls"
 	_ "embed"
 	"encoding/json"
 	"fmt"
-	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"io/ioutil"
 	"log"
 	"net/http"
 	be "spider_creator/backend"
 	bdb "spider_creator/backend/db"
+
+	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 )
 
 const (
-	LISTEN_ADDR = ":8080"
+	LISTEN_ADDR       = ":8080"
+	AI_GEN_LIST_CSS   = "http://39.106.73.135:5005/listpage"
+	AI_GEN_DETAIL_CSS = "http://39.106.73.135:5005/detailpage"
 )
 
 type (
@@ -67,6 +72,9 @@ func (ws *WebService) RunHttpServe() {
 	//这里注册HTTP服务
 	mux.HandleFunc("/save", ws.SaveSpiderConfig)
 	mux.HandleFunc("/load", ws.LoadSpiderConfig)
+	mux.HandleFunc("/loadListCss", ws.FindListCssSelector)
+	mux.HandleFunc("/loadDetailCss", ws.FindDetailCssSelector)
+
 	//
 	qu.Debug("Starting HTTPS server on ", LISTEN_ADDR)
 	err = server.ListenAndServeTLS("", "")
@@ -146,3 +154,173 @@ func SetCurrentTabCssMark(cssMark map[string]interface{}) {
 	currentTabSpiderConfig = sc
 	//qu.Debug("当前编辑爬虫链接：", *currentTabSpiderConfig)
 }
+
+// FindListCssSelector
+func (ws *WebService) FindListCssSelector(w http.ResponseWriter, r *http.Request) {
+	qu.Debug("AI生成列表页CSS...")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Content-Type", "application/json")
+	reqData := struct {
+		Data string `json:"data"`
+		User string `json:"user"`
+	}{}
+	err := json.NewDecoder(r.Body).Decode(&reqData)
+	if err != nil {
+		return
+	}
+
+	//发起远程请求，进行大模型计算
+	reqData.User = "jianyu"
+	jsonData, err := json.Marshal(reqData)
+	if err != nil {
+		fmt.Println("Error marshalling data:", err)
+		return
+	}
+	// 创建HTTP客户端
+	client := &http.Client{}
+	// 创建POST请求
+	req, err := http.NewRequest("POST", AI_GEN_LIST_CSS, bytes.NewBuffer(jsonData))
+	if err != nil {
+		fmt.Println("Error creating request:", err)
+		return
+	}
+	// 设置请求头
+	req.Header.Set("Content-Type", "application/json")
+	// 发送请求
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Println("Error sending request:", err)
+		return
+	}
+	defer resp.Body.Close()
+	// 读取响应
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Println("Error reading response:", err)
+		return
+	}
+	log.Println("AI返回", string(body))
+	resultCss := struct {
+		Data struct {
+			ListBodyCss        string `json:"listBodyCss"`
+			ListItemCss        string `json:"listItemCss"`
+			ListLinkCss        string `json:"listLinkCss"`
+			ListPublishTimeCss string `json:"listPublishTimeCss"`
+			ListNextPageCss    string `json:"listNextPageCss"`
+			listTurnPageJs     string `json:"listTurnPageJs"`
+		} `json:"data"`
+	}{}
+	err = json.Unmarshal(body, &resultCss)
+	if err != nil {
+		fmt.Println("Error decodejson response:", err, string(body))
+		return
+	}
+
+	//TODO 通知开发工具端，CSS选择器有变动
+	currentTabSpiderConfig.ListBodyCss = resultCss.Data.ListBodyCss
+	currentTabSpiderConfig.ListItemCss = resultCss.Data.ListItemCss
+	currentTabSpiderConfig.ListLinkCss = resultCss.Data.ListLinkCss
+	currentTabSpiderConfig.ListNextPageCss = resultCss.Data.ListNextPageCss
+	currentTabSpiderConfig.ListPubtimeCss = resultCss.Data.ListPublishTimeCss
+	currentTabSpiderConfig.ListTurnPageJSCode = resultCss.Data.listTurnPageJs
+
+	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listBodyCss", "css": resultCss.Data.ListBodyCss})
+	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listItemCss", "css": resultCss.Data.ListItemCss})
+	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listLinkCss", "css": resultCss.Data.ListLinkCss})
+	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listPublishTimeCss", "css": resultCss.Data.ListPublishTimeCss})
+	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listNextPageCss", "css": resultCss.Data.ListNextPageCss})
+	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listTurnPageJs", "css": resultCss.Data.listTurnPageJs})
+
+	err = json.NewEncoder(w).Encode(currentTabSpiderConfig)
+	if err != nil {
+		log.Println("反向序列化失败")
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+}
+
+// FindDetailCssSelector
+func (ws *WebService) FindDetailCssSelector(w http.ResponseWriter, r *http.Request) {
+	qu.Debug("AI生成详情页CSS...")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Content-Type", "application/json")
+	reqData := struct {
+		Data string `json:"data"`
+		User string `json:"user"`
+	}{}
+	err := json.NewDecoder(r.Body).Decode(&reqData)
+	if err != nil {
+		return
+	}
+
+	//发起远程请求，进行大模型计算
+	reqData.User = "jianyu"
+	jsonData, err := json.Marshal(reqData)
+	if err != nil {
+		fmt.Println("Error marshalling data:", err)
+		return
+	}
+	// 创建HTTP客户端
+	client := &http.Client{}
+	// 创建POST请求
+	req, err := http.NewRequest("POST", AI_GEN_DETAIL_CSS, bytes.NewBuffer(jsonData))
+	if err != nil {
+		fmt.Println("Error creating request:", err)
+		return
+	}
+	// 设置请求头
+	req.Header.Set("Content-Type", "application/json")
+	// 发送请求
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Println("Error sending request:", err)
+		return
+	}
+	defer resp.Body.Close()
+	// 读取响应
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Println("Error reading response:", err)
+		return
+	}
+	log.Println("AI返回", string(body))
+	resultCss := struct {
+		Data struct {
+			TitleCss       string `json:"titleCss"`
+			ContentCss     string `json:"contentCss"`
+			PublishTimeCss string `json:"publishTimeCss"`
+			PublishUnitCss string `json:"publishUnitCss"`
+			AttachCss      string `json:"attachCss"`
+		} `json:"data"`
+	}{}
+	err = json.Unmarshal(body, &resultCss)
+	if err != nil {
+		fmt.Println("Error decodejson response:", err, string(body))
+		return
+	}
+
+	//TODO 通知开发工具端，CSS选择器有变动
+	currentTabSpiderConfig.TitleCss = resultCss.Data.TitleCss
+	currentTabSpiderConfig.PublishTimeCss = resultCss.Data.PublishTimeCss
+	currentTabSpiderConfig.PublishUnitCss = resultCss.Data.PublishUnitCss
+	currentTabSpiderConfig.ContentCss = resultCss.Data.ContentCss
+	currentTabSpiderConfig.AttachCss = resultCss.Data.AttachCss
+
+	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "titleCss",
+		"css": resultCss.Data.TitleCss})
+	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "contentCss",
+		"css": resultCss.Data.ContentCss})
+	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "publishTimeCss",
+		"css": resultCss.Data.PublishTimeCss})
+	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "publishUnitCss",
+		"css": resultCss.Data.PublishUnitCss})
+	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "attachCss",
+		"css": resultCss.Data.AttachCss})
+
+	err = json.NewEncoder(w).Encode(currentTabSpiderConfig)
+	if err != nil {
+		log.Println("反向序列化失败")
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+}

frontend/src/views/CodeList.vue

@@ -87,7 +87,7 @@
                         </el-tooltip>
                     </template>
                 </el-table-column>
-                <el-table-column label="功能" width="160" align="center">
+                <el-table-column v-if="!isAdmin" label="功能" width="160" align="center">
                     <template #default="scope">
                         <el-tooltip content="复制" placement="top" v-if="showListCopyButton">
                             <el-button size="small" :disabled="tableActionDisabled.copyDisabled(scope.row)" :class="{ active: scope.row._action_clicked_copy }" @click="tableEvents.handleCopy(scope.$index, scope.row)">
@@ -143,7 +143,7 @@ import VerifySpider from "../components/spider/VerifySpider.vue"
 import { useCodeListFiltersWithRole } from '../composables/filter-options'
 import { USER_ROLE_ADMIN, USER_ROLE_DEVELOPER, USER_ROLE_REVIEWER } from '../data/user'
 import { Refresh, Search, Box } from '@element-plus/icons-vue'
-
+const isAdmin = computed(() => [USER_ROLE_ADMIN].includes(userRole.value))
 const router = useRouter();
 const store = useStore();
 const spiderTable = ref(null)

frontend/src/views/ReviewList.vue

@@ -90,7 +90,7 @@
                         </el-tooltip>
                     </template>
                 </el-table-column>
-                <el-table-column label="功能" width="120" align="center">
+                <el-table-column label="功能" :width="isAdmin ? 160 : 120" align="center">
                     <template #default="scope">
                         <!-- 管理员：上线和退回 -->
                         <template v-if="tableActionShow.adminGroup(scope.row)">
@@ -99,6 +99,11 @@
                                   <el-icon><CircleCloseFilled /></el-icon>
                                 </el-button>
                             </el-tooltip>
+                            <el-tooltip content="打回" placement="top" v-if="tableActionShow.adminReject(scope.row)">
+                              <el-button size="small" :class="{ active: scope.row._action_clicked_reject_code }" :disabled="actionButtonDisabled.adminReject(scope.row)" @click="tableEvents.adminReject(scope.$index, scope.row)">
+                                <el-icon><CircleClose /></el-icon>
+                              </el-button>
+                            </el-tooltip>
                             <el-tooltip content="上线" placement="top" v-if="tableActionShow.adminSubmit(scope.row)">
                                 <el-button size="small" :class="{ active: scope.row._action_clicked_submit }" :disabled="actionButtonDisabled.adminSubmit(scope.row)" @click="tableEvents.adminSubmit(scope.$index, scope.row)">
                                     <el-icon><UploadFilled /></el-icon>
@@ -345,6 +350,7 @@ async function getTableList() {
                         _action_clicked_debug: false,
                         _action_clicked_verify: false,
                         _action_clicked_down_code: false,
+                        _action_clicked_reject_code: false,
                         _action_clicked_submit: false,
                         _action_clicked_rollback: false,
                     }
@@ -483,6 +489,10 @@ const tableActionShow = {
         // 下架按钮，只有已上线爬虫展示
         return row.state === 11
     },
+    adminReject(row) {
+      // 下架按钮，只有已上线爬虫展示
+      return row.state === 11
+    },
     adminRollback(row) {
         // 只有已上线，才展示退回
         return row.state === 11
@@ -510,6 +520,9 @@ const actionButtonDisabled = {
     },
     adminDownCode(row) {
         return row.state !== 11
+    },
+    adminReject(row) {
+      return row.state !== 11
     }
 }
 
@@ -843,6 +856,26 @@ const tableEvents = {
             })
         })
     },
+    // 管理员打回
+    adminReject(_, row) {
+      console.log(row)
+      onlyClickHighlight(row, '_action_clicked_reject_code')
+      ElMessageBox.prompt('请输入打回原因', '打回', {
+        confirmButtonText: '确认打回',
+        cancelButtonText: '取消',
+        inputPattern: /^.+$/, // 非空
+        inputErrorMessage: '不能为空',
+      })
+      .then(({ value }) => {
+        this.updateCodeStateAction([row], {
+          reason: value,
+          stype: '打回',
+          state: 2,
+          successTip: '打回成功',
+          errorTip: '打回失败',
+        })
+      })
+    }
 }
 
 const toggleRow = (rowList = []) => {
@@ -868,8 +901,8 @@ const confirmBatchRollback = () => {
     let dropList = []
     if (isAdmin.value) {
         // 管理员可以打回待审核和已通过爬虫
-        list = listState.selected.filter(v => v.state === 1 || v.state === 3)
-        dropList = listState.selected.filter(v => !(v.state === 1 || v.state === 3))
+        list = listState.selected.filter(v => v.state === 1 || v.state === 3 || v.state === 11)
+        dropList = listState.selected.filter(v => !(v.state === 1 || v.state === 3 || v.state === 11))
     } else {
         // 审核员只能打回待审核爬虫
         list = listState.selected.filter(v => v.state === 1)

server.go

@@ -12,7 +12,7 @@ import (
 	"time"
 )
 
-const HREF = "http://127.0.0.1:8091/%s"
+const HREF = "http://127.0.0.1:8091/%s" //线下测试环境
 
 //const HREF = "http://visualizeld.spdata.jianyu360.com/%s" //正式库
 //const HREF = "http://visualize.spdata.jianyu360.com/%s" //临时库