// 对外服务 package webservice import ( "bytes" "crypto/tls" _ "embed" "encoding/json" "fmt" "io/ioutil" "log" "net/http" be "spider_creator/backend" bdb "spider_creator/backend/db" qu "jygit.jydev.jianyu360.cn/data_processing/common_utils" ) const ( LISTEN_ADDR = ":8080" AI_GEN_LIST_CSS = "http://182.92.66.70:5005/listpage" AI_GEN_DETAIL_CSS = "http://182.92.66.70:5005/detailpage" ) type ( SpiderConfigItem struct { Key string `json:"key"` Css string `json:"css"` Url string `json:"url"` } WebService struct { db *bdb.SpiderDb enf be.EventNotifyFace currentSpiderConfig *be.SpiderConfig } ) var ( //go:embed cert.pem certBytes []byte //go:embed key.pem keyBytes []byte currentTabSpiderConfig = &be.SpiderConfig{} ) // NewWebService func NewWebService(db *bdb.SpiderDb, enf be.EventNotifyFace, csf *be.SpiderConfig) *WebService { return &WebService{db, enf, csf} } func (ws *WebService) RunHttpServe() { // 设置HTTP服务器 mux := http.NewServeMux() // 解析证书 cert, err := tls.X509KeyPair(certBytes, keyBytes) if err != nil { qu.Debug(err.Error()) return } // 创建一个TLS配置 tlsConfig := &tls.Config{ // 可以在这里添加其他TLS配置 Certificates: []tls.Certificate{cert}, ServerName: "localhost", InsecureSkipVerify: true, } server := &http.Server{ Addr: LISTEN_ADDR, Handler: mux, TLSConfig: tlsConfig, } //这里注册HTTP服务 mux.HandleFunc("/save", ws.SaveSpiderConfig) mux.HandleFunc("/load", ws.LoadSpiderConfig) mux.HandleFunc("/loadListCss", ws.FindListCssSelector) mux.HandleFunc("/loadDetailCss", ws.FindDetailCssSelector) // qu.Debug("Starting HTTPS server on ", LISTEN_ADDR) err = server.ListenAndServeTLS("", "") if err != nil { qu.Debug("Failed to start server: ", err.Error()) return } } // SaveSpiderConfig LoadCurrentSpiderConfig,json处理 func (ws *WebService) SaveSpiderConfig(w http.ResponseWriter, r *http.Request) { qu.Debug("保存设置") w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") var req = new(SpiderConfigItem) err := json.NewDecoder(r.Body).Decode(req) if err != nil { qu.Debug("序列化失败") http.Error(w, err.Error(), http.StatusBadRequest) return } switch req.Key { case "listItemCss": currentTabSpiderConfig.ListItemCss = req.Css case "listLinkCss": currentTabSpiderConfig.ListLinkCss = req.Css case "listPublishTimeCss": currentTabSpiderConfig.ListPubtimeCss = req.Css case "listNextPageCss": currentTabSpiderConfig.ListNextPageCss = req.Css case "listBodyCss": currentTabSpiderConfig.ListBodyCss = req.Css case "titleCss": currentTabSpiderConfig.TitleCss = req.Css case "publishUnitCss": currentTabSpiderConfig.PublishUnitCss = req.Css case "publishTimeCss": currentTabSpiderConfig.PublishTimeCss = req.Css case "contentCss": currentTabSpiderConfig.ContentCss = req.Css case "attachCss": currentTabSpiderConfig.AttachCss = req.Css } qu.Debug("CSS", req.Key, req.Css, req.Url) fmt.Fprint(w, "{'code':200}") //TODO 通知开发工具端,CSS选择器有变动 ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": req.Key, "css": req.Css, "url": req.Url}) } // LoadSpiderConfig LoadCurrentSpiderConfig,加载,返回当前配置项 func (ws *WebService) LoadSpiderConfig(w http.ResponseWriter, r *http.Request) { qu.Debug("加载当前配置项") w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") var req = new(SpiderConfigItem) err := json.NewDecoder(r.Body).Decode(req) if err != nil { qu.Debug("序列化失败") http.Error(w, err.Error(), http.StatusBadRequest) return } else { qu.Debug("高亮:", req.Url, *currentTabSpiderConfig) err = json.NewEncoder(w).Encode(currentTabSpiderConfig) if err != nil { log.Println("反向序列化失败") http.Error(w, err.Error(), http.StatusBadRequest) return } } } func SetCurrentTabCssMark(cssMark map[string]interface{}) { sc, err := be.NewSpiderConfig(cssMark) if err != nil { qu.Debug("标注信息传输失败!") } currentTabSpiderConfig = sc //qu.Debug("当前编辑爬虫链接:", *currentTabSpiderConfig) } // FindListCssSelector func (ws *WebService) FindListCssSelector(w http.ResponseWriter, r *http.Request) { qu.Debug("AI生成列表页CSS...") w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") reqData := struct { Data string `json:"data"` User string `json:"user"` }{} err := json.NewDecoder(r.Body).Decode(&reqData) if err != nil { return } //发起远程请求,进行大模型计算 reqData.User = "jianyu" jsonData, err := json.Marshal(reqData) if err != nil { fmt.Println("Error marshalling data:", err) return } // 创建HTTP客户端 client := &http.Client{} // 创建POST请求 req, err := http.NewRequest("POST", AI_GEN_LIST_CSS, bytes.NewBuffer(jsonData)) if err != nil { fmt.Println("Error creating request:", err) return } // 设置请求头 req.Header.Set("Content-Type", "application/json") // 发送请求 resp, err := client.Do(req) if err != nil { fmt.Println("Error sending request:", err) return } defer resp.Body.Close() // 读取响应 body, err := ioutil.ReadAll(resp.Body) if err != nil { fmt.Println("Error reading response:", err) return } log.Println("AI返回", string(body)) resultCss := struct { Data struct { ListBodyCss string `json:"listBodyCss"` ListItemCss string `json:"listItemCss"` ListLinkCss string `json:"listLinkCss"` ListPublishTimeCss string `json:"listPublishTimeCss"` ListNextPageCss string `json:"listNextPageCss"` listTurnPageJs string `json:"listTurnPageJs"` } `json:"data"` }{} err = json.Unmarshal(body, &resultCss) if err != nil { fmt.Println("Error decodejson response:", err, string(body)) return } //TODO 通知开发工具端,CSS选择器有变动 currentTabSpiderConfig.ListBodyCss = resultCss.Data.ListBodyCss currentTabSpiderConfig.ListItemCss = resultCss.Data.ListItemCss currentTabSpiderConfig.ListLinkCss = resultCss.Data.ListLinkCss currentTabSpiderConfig.ListNextPageCss = resultCss.Data.ListNextPageCss currentTabSpiderConfig.ListPubtimeCss = resultCss.Data.ListPublishTimeCss currentTabSpiderConfig.ListTurnPageJSCode = resultCss.Data.listTurnPageJs ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listBodyCss", "css": resultCss.Data.ListBodyCss}) ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listItemCss", "css": resultCss.Data.ListItemCss}) ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listLinkCss", "css": resultCss.Data.ListLinkCss}) ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listPublishTimeCss", "css": resultCss.Data.ListPublishTimeCss}) ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listNextPageCss", "css": resultCss.Data.ListNextPageCss}) ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listTurnPageJs", "css": resultCss.Data.listTurnPageJs}) err = json.NewEncoder(w).Encode(currentTabSpiderConfig) if err != nil { log.Println("反向序列化失败") http.Error(w, err.Error(), http.StatusBadRequest) return } } // FindDetailCssSelector func (ws *WebService) FindDetailCssSelector(w http.ResponseWriter, r *http.Request) { qu.Debug("AI生成详情页CSS...") w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") reqData := struct { Data string `json:"data"` User string `json:"user"` }{} err := json.NewDecoder(r.Body).Decode(&reqData) if err != nil { return } //发起远程请求,进行大模型计算 reqData.User = "jianyu" jsonData, err := json.Marshal(reqData) if err != nil { fmt.Println("Error marshalling data:", err) return } // 创建HTTP客户端 client := &http.Client{} // 创建POST请求 req, err := http.NewRequest("POST", AI_GEN_DETAIL_CSS, bytes.NewBuffer(jsonData)) if err != nil { fmt.Println("Error creating request:", err) return } // 设置请求头 req.Header.Set("Content-Type", "application/json") // 发送请求 resp, err := client.Do(req) if err != nil { fmt.Println("Error sending request:", err) return } defer resp.Body.Close() // 读取响应 body, err := ioutil.ReadAll(resp.Body) if err != nil { fmt.Println("Error reading response:", err) return } log.Println("AI返回", string(body)) resultCss := struct { Data struct { TitleCss string `json:"titleCss"` ContentCss string `json:"contentCss"` PublishTimeCss string `json:"publishTimeCss"` PublishUnitCss string `json:"publishUnitCss"` AttachCss string `json:"attachCss"` } `json:"data"` }{} err = json.Unmarshal(body, &resultCss) if err != nil { fmt.Println("Error decodejson response:", err, string(body)) return } //TODO 通知开发工具端,CSS选择器有变动 currentTabSpiderConfig.TitleCss = resultCss.Data.TitleCss currentTabSpiderConfig.PublishTimeCss = resultCss.Data.PublishTimeCss currentTabSpiderConfig.PublishUnitCss = resultCss.Data.PublishUnitCss currentTabSpiderConfig.ContentCss = resultCss.Data.ContentCss currentTabSpiderConfig.AttachCss = resultCss.Data.AttachCss ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "titleCss", "css": resultCss.Data.TitleCss}) ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "contentCss", "css": resultCss.Data.ContentCss}) ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "publishTimeCss", "css": resultCss.Data.PublishTimeCss}) ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "publishUnitCss", "css": resultCss.Data.PublishUnitCss}) ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "attachCss", "css": resultCss.Data.AttachCss}) err = json.NewEncoder(w).Encode(currentTabSpiderConfig) if err != nil { log.Println("反向序列化失败") http.Error(w, err.Error(), http.StatusBadRequest) return } }