package main import ( "bytes" "encoding/json" "fmt" "io/ioutil" qu "jygit.jydev.jianyu360.cn/data_processing/common_utils" "net/http" "reflect" be "spider_creator/backend" hc "spider_creator/httpclient" "time" ) type Result struct { Msg string `json:"msg"` Err int `json:"err"` Data *Data `json:"data"` } type Data struct { List []map[string]interface{} `json:"list"` Total int `json:"total"` } // ServerActionCodeList 获取爬虫列表 func (a *App) ServerActionCodeList(param map[string]interface{}) *Result { qu.Debug("param---", param) r := &Result{} if User != nil { getResult(map[string]interface{}{"param": param, "user": User}, r, "getcodes") } else { r.Msg = "用户登录异常,请重新登录!" qu.Debug(r.Msg) } return r } // ServerActionClaimCodes 爬虫认领 func (a *App) ServerActionClaimCodes() *Result { r := &Result{} if User != nil && User.Auth == 1 { getResult(User, r, "claimcode") } else { r.Msg = "认领失败!" } return r } // ServerActionCopyCode 复制爬虫 func (a *App) ServerActionCopyCode(param map[string]interface{}) *Result { r := &Result{} if User != nil && User.Auth == 1 { getResult(map[string]interface{}{"param": param, "user": User}, r, "copycode") } else { r.Msg = "复制爬虫失败!" } return r } // ServerActionUpdateCode 爬虫更新 func (a *App) ServerActionUpdateCode(param map[string]interface{}) *Result { qu.Debug("param---", param) r := &Result{} if User != nil { //formatUser(param) qu.Debug("param---", param) getResult(map[string]interface{}{"param": param, "user": User}, r, "updatecode") } else { r.Msg = "用户登录异常,请重新登录!" qu.Debug(r.Msg) } return r } // ServerActionUpdateCodeState 爬虫状态更新 func (a *App) ServerActionUpdateCodeState(param map[string]interface{}) *Result { qu.Debug("param---", param) /* 1、提交审核 2、回退 3、打回 4、审核通过 5、上线 */ r := &Result{} //前期校验 if User != nil { //提交审核时,验证校验清单 p, _ := param["param"].(map[string]interface{}) if qu.ObjToString(p["stype"]) == "提交" { code := qu.ObjToString(p["code"]) vr := be.VerifyResults[code] if vr == nil { //没有检验清单,不允许提交 r.Msg = "未验证,无法提交!" //} else if !vr.ListItems || !vr.Content || !vr.Title || !vr.PublishTime { //校验检验清单必通过项 } else if !vr.ListItems || !vr.Content || !vr.Title || !vr.PublishTime || !vr.ListTrunPage { //校验检验清单必通过项 r.Msg = "验证清单未通过!" } else { p["verify"] = vr be.VerifyResults[code] = nil //清空验证结果 } } } else { r.Msg = "用户登录异常,请重新登录!" qu.Debug(r.Msg) } if r.Msg == "" { param["user"] = User qu.Debug("param---", param) //formatUser(param) getResult(param, r, "updatecodestate") } return r } // ServerActionQlmRecordList 千里马采集记录 func (a *App) ServerActionQlmRecordList(param map[string]interface{}) *Result { r := &Result{} //前期校验 if User != nil { getResult(map[string]interface{}{"param": param, "user": User}, r, "qlm/recordList") } else { r.Msg = "用户登录异常,请重新登录!" qu.Debug(r.Msg) } return r } // ServerActionQlmAddRecord 新增千里马采集记录 func (a *App) ServerActionQlmAddRecord(param map[string]interface{}) *Result { qu.Debug("param---", param) r := &Result{} //前期校验 if User != nil { getResult(map[string]interface{}{"param": param, "user": User}, r, "qlm/newRecord") } else { r.Msg = "用户登录异常,请重新登录!" qu.Debug(r.Msg) } return r } // ServerActionQlmRemoveRepeat 千里马数据去重 func (a *App) ServerActionQlmRemoveRepeat(param map[string]interface{}) *Result { qu.Debug("param---", param) r := &Result{} //前期校验 if User != nil { getResult(map[string]interface{}{"param": param}, r, "qlm/removeRepeat") } else { r.Msg = "用户登录异常,请重新登录!" qu.Debug(r.Msg) } return r } // ServerActionQlmPushData 千里马数据推送 func (a *App) ServerActionQlmPushData(param map[string]interface{}) *Result { qu.Debug("param---", param) r := &Result{} //前期校验 if User != nil { getResult(map[string]interface{}{"param": param}, r, "qlm/pushData") } else { r.Msg = "用户登录异常,请重新登录!" qu.Debug(r.Msg) } return r } // ServerActionQlmClearData 千里马数据清理 func (a *App) ServerActionQlmClearData(param map[string]interface{}) *Result { qu.Debug("param---", param) r := &Result{} //前期校验 if User != nil { getResult(map[string]interface{}{"param": param}, r, "qlm/clearData") } else { r.Msg = "用户登录异常,请重新登录!" qu.Debug(r.Msg) } return r } // 格式化User对象 func formatUser(tmp map[string]interface{}) { v := reflect.ValueOf(User) if v.Kind() == reflect.Ptr { v = v.Elem() } t := v.Type() for i := 0; i < v.NumField(); i++ { field := t.Field(i).Tag.Get("json") value := v.Field(i).Interface() tmp[field] = value } } func getResult(param, result interface{}, route string) { jsonData, err := json.Marshal(param) if err != nil { qu.Debug("Error marshaling request:", err) return } req, err := http.NewRequest("POST", fmt.Sprintf(serverAddress, route), bytes.NewBuffer(jsonData)) if err != nil { qu.Debug("Error creating request:", err) return } // 设置请求头 req.Header.Set("Content-Type", "application/json") // 发送请求 //client := &http.Client{ // Timeout: 10 * time.Second, //} client := hc.HttpClient() client.Timeout = 10 * time.Second resp, err := client.Do(req) if err != nil { qu.Debug("Error sending request:", err) return } defer resp.Body.Close() // 读取响应体 body, err := ioutil.ReadAll(resp.Body) if err != nil { qu.Debug("Error reading response body:", err) return } if err := json.Unmarshal(body, &result); err != nil { qu.Debug("Error unmarshaling response:", err) return } }