spider_creater/backend/webservice/webservice.go

// 对外服务
package webservice

import (
	"bytes"
	"crypto/tls"
	_ "embed"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	be "spider_creator/backend"
	bdb "spider_creator/backend/db"

	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
)

const (
	LISTEN_ADDR       = ":8080"
	AI_GEN_LIST_CSS   = "http://182.92.66.70:5005/listpage"
	AI_GEN_DETAIL_CSS = "http://182.92.66.70:5005/detailpage"
)

type (
	SpiderConfigItem struct {
		Key string `json:"key"`
		Css string `json:"css"`
		Url string `json:"url"`
	}
	WebService struct {
		db                  *bdb.SpiderDb
		enf                 be.EventNotifyFace
		currentSpiderConfig *be.SpiderConfig
	}
)

var (
	//go:embed cert.pem
	certBytes []byte
	//go:embed key.pem
	keyBytes               []byte
	currentTabSpiderConfig = &be.SpiderConfig{}
)

// NewWebService
func NewWebService(db *bdb.SpiderDb, enf be.EventNotifyFace, csf *be.SpiderConfig) *WebService {
	return &WebService{db, enf, csf}
}

func (ws *WebService) RunHttpServe() {
	// 设置HTTP服务器
	mux := http.NewServeMux()
	// 解析证书
	cert, err := tls.X509KeyPair(certBytes, keyBytes)
	if err != nil {
		qu.Debug(err.Error())
		return
	}
	// 创建一个TLS配置
	tlsConfig := &tls.Config{
		// 可以在这里添加其他TLS配置
		Certificates:       []tls.Certificate{cert},
		ServerName:         "localhost",
		InsecureSkipVerify: true,
	}
	server := &http.Server{
		Addr:      LISTEN_ADDR,
		Handler:   mux,
		TLSConfig: tlsConfig,
	}
	//这里注册HTTP服务
	mux.HandleFunc("/save", ws.SaveSpiderConfig)
	mux.HandleFunc("/load", ws.LoadSpiderConfig)
	mux.HandleFunc("/loadListCss", ws.FindListCssSelector)
	mux.HandleFunc("/loadDetailCss", ws.FindDetailCssSelector)

	//
	qu.Debug("Starting HTTPS server on ", LISTEN_ADDR)
	err = server.ListenAndServeTLS("", "")
	if err != nil {
		qu.Debug("Failed to start server:  ", err.Error())
		return
	}
}

// SaveSpiderConfig LoadCurrentSpiderConfig,json处理
func (ws *WebService) SaveSpiderConfig(w http.ResponseWriter, r *http.Request) {
	qu.Debug("保存设置")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json")
	var req = new(SpiderConfigItem)
	err := json.NewDecoder(r.Body).Decode(req)
	if err != nil {
		qu.Debug("序列化失败")
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	switch req.Key {
	case "listItemCss":
		currentTabSpiderConfig.ListItemCss = req.Css
	case "listLinkCss":
		currentTabSpiderConfig.ListLinkCss = req.Css
	case "listPublishTimeCss":
		currentTabSpiderConfig.ListPubtimeCss = req.Css
	case "listNextPageCss":
		currentTabSpiderConfig.ListNextPageCss = req.Css
	case "listBodyCss":
		currentTabSpiderConfig.ListBodyCss = req.Css
	case "titleCss":
		currentTabSpiderConfig.TitleCss = req.Css
	case "publishUnitCss":
		currentTabSpiderConfig.PublishUnitCss = req.Css
	case "publishTimeCss":
		currentTabSpiderConfig.PublishTimeCss = req.Css
	case "contentCss":
		currentTabSpiderConfig.ContentCss = req.Css
	case "attachCss":
		currentTabSpiderConfig.AttachCss = req.Css
	}
	qu.Debug("CSS", req.Key, req.Css, req.Url)
	fmt.Fprint(w, "{'code':200}")
	//TODO 通知开发工具端，CSS选择器有变动
	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": req.Key, "css": req.Css, "url": req.Url})
}

// LoadSpiderConfig LoadCurrentSpiderConfig,加载，返回当前配置项
func (ws *WebService) LoadSpiderConfig(w http.ResponseWriter, r *http.Request) {
	qu.Debug("加载当前配置项")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json")
	var req = new(SpiderConfigItem)
	err := json.NewDecoder(r.Body).Decode(req)
	if err != nil {
		qu.Debug("序列化失败")
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	} else {
		qu.Debug("高亮：", req.Url, *currentTabSpiderConfig)
		err = json.NewEncoder(w).Encode(currentTabSpiderConfig)
		if err != nil {
			log.Println("反向序列化失败")
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
	}
}

func SetCurrentTabCssMark(cssMark map[string]interface{}) {
	sc, err := be.NewSpiderConfig(cssMark)
	if err != nil {
		qu.Debug("标注信息传输失败！")
	}
	currentTabSpiderConfig = sc
	//qu.Debug("当前编辑爬虫链接：", *currentTabSpiderConfig)
}

// FindListCssSelector
func (ws *WebService) FindListCssSelector(w http.ResponseWriter, r *http.Request) {
	qu.Debug("AI生成列表页CSS...")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json")
	reqData := struct {
		Data string `json:"data"`
		User string `json:"user"`
	}{}
	err := json.NewDecoder(r.Body).Decode(&reqData)
	if err != nil {
		return
	}

	//发起远程请求，进行大模型计算
	reqData.User = "jianyu"
	jsonData, err := json.Marshal(reqData)
	if err != nil {
		fmt.Println("Error marshalling data:", err)
		return
	}
	// 创建HTTP客户端
	client := &http.Client{}
	// 创建POST请求
	req, err := http.NewRequest("POST", AI_GEN_LIST_CSS, bytes.NewBuffer(jsonData))
	if err != nil {
		fmt.Println("Error creating request:", err)
		return
	}
	// 设置请求头
	req.Header.Set("Content-Type", "application/json")
	// 发送请求
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("Error sending request:", err)
		return
	}
	defer resp.Body.Close()
	// 读取响应
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("Error reading response:", err)
		return
	}
	log.Println("AI返回", string(body))
	resultCss := struct {
		Data struct {
			ListBodyCss        string `json:"listBodyCss"`
			ListItemCss        string `json:"listItemCss"`
			ListLinkCss        string `json:"listLinkCss"`
			ListPublishTimeCss string `json:"listPublishTimeCss"`
			ListNextPageCss    string `json:"listNextPageCss"`
			listTurnPageJs     string `json:"listTurnPageJs"`
		} `json:"data"`
	}{}
	err = json.Unmarshal(body, &resultCss)
	if err != nil {
		fmt.Println("Error decodejson response:", err, string(body))
		return
	}

	//TODO 通知开发工具端，CSS选择器有变动
	currentTabSpiderConfig.ListBodyCss = resultCss.Data.ListBodyCss
	currentTabSpiderConfig.ListItemCss = resultCss.Data.ListItemCss
	currentTabSpiderConfig.ListLinkCss = resultCss.Data.ListLinkCss
	currentTabSpiderConfig.ListNextPageCss = resultCss.Data.ListNextPageCss
	currentTabSpiderConfig.ListPubtimeCss = resultCss.Data.ListPublishTimeCss
	currentTabSpiderConfig.ListTurnPageJSCode = resultCss.Data.listTurnPageJs

	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listBodyCss", "css": resultCss.Data.ListBodyCss})
	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listItemCss", "css": resultCss.Data.ListItemCss})
	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listLinkCss", "css": resultCss.Data.ListLinkCss})
	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listPublishTimeCss", "css": resultCss.Data.ListPublishTimeCss})
	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listNextPageCss", "css": resultCss.Data.ListNextPageCss})
	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "listTurnPageJs", "css": resultCss.Data.listTurnPageJs})

	err = json.NewEncoder(w).Encode(currentTabSpiderConfig)
	if err != nil {
		log.Println("反向序列化失败")
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
}

// FindDetailCssSelector
func (ws *WebService) FindDetailCssSelector(w http.ResponseWriter, r *http.Request) {
	qu.Debug("AI生成详情页CSS...")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json")
	reqData := struct {
		Data string `json:"data"`
		User string `json:"user"`
	}{}
	err := json.NewDecoder(r.Body).Decode(&reqData)
	if err != nil {
		return
	}

	//发起远程请求，进行大模型计算
	reqData.User = "jianyu"
	jsonData, err := json.Marshal(reqData)
	if err != nil {
		fmt.Println("Error marshalling data:", err)
		return
	}
	// 创建HTTP客户端
	client := &http.Client{}
	// 创建POST请求
	req, err := http.NewRequest("POST", AI_GEN_DETAIL_CSS, bytes.NewBuffer(jsonData))
	if err != nil {
		fmt.Println("Error creating request:", err)
		return
	}
	// 设置请求头
	req.Header.Set("Content-Type", "application/json")
	// 发送请求
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("Error sending request:", err)
		return
	}
	defer resp.Body.Close()
	// 读取响应
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("Error reading response:", err)
		return
	}
	log.Println("AI返回", string(body))
	resultCss := struct {
		Data struct {
			TitleCss       string `json:"titleCss"`
			ContentCss     string `json:"contentCss"`
			PublishTimeCss string `json:"publishTimeCss"`
			PublishUnitCss string `json:"publishUnitCss"`
			AttachCss      string `json:"attachCss"`
		} `json:"data"`
	}{}
	err = json.Unmarshal(body, &resultCss)
	if err != nil {
		fmt.Println("Error decodejson response:", err, string(body))
		return
	}

	//TODO 通知开发工具端，CSS选择器有变动
	currentTabSpiderConfig.TitleCss = resultCss.Data.TitleCss
	currentTabSpiderConfig.PublishTimeCss = resultCss.Data.PublishTimeCss
	currentTabSpiderConfig.PublishUnitCss = resultCss.Data.PublishUnitCss
	currentTabSpiderConfig.ContentCss = resultCss.Data.ContentCss
	currentTabSpiderConfig.AttachCss = resultCss.Data.AttachCss

	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "titleCss",
		"css": resultCss.Data.TitleCss})
	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "contentCss",
		"css": resultCss.Data.ContentCss})
	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "publishTimeCss",
		"css": resultCss.Data.PublishTimeCss})
	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "publishUnitCss",
		"css": resultCss.Data.PublishUnitCss})
	ws.enf.Dispatch("spiderConfigChange", map[string]interface{}{"key": "attachCss",
		"css": resultCss.Data.AttachCss})

	err = json.NewEncoder(w).Encode(currentTabSpiderConfig)
	if err != nil {
		log.Println("反向序列化失败")
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
}