支持传参

src/analysiscode/client.go

@@ -3,7 +3,6 @@ package analysiscode
 import (
 	"analysiscode/proto"
 	"context"
-	"encoding/json"
 	"log"
 	qu "qfw/util"
 	"spiderutil"
@@ -12,17 +11,6 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 )
 
-type ResultInfo struct {
-	Msg  string `json:"msg"`
-	Code int    `json:"code"`
-	Info Result `json:"r"`
-}
-
-type Result struct {
-	Time float64 `json:"time"`
-	Code string  `json:"code"`
-}
-
 var Client proto.GetCodeClient
 
 func InitCodeGrpcClient() {
@@ -30,34 +18,30 @@ func InitCodeGrpcClient() {
 	log.Println("address:", spiderutil.Config.ServerAddress)
 	conn, err := grpc.Dial(spiderutil.Config.ServerAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
-		log.Fatalf("Init Connect Error: %v", err)
+		log.Println("Init Connect Error: ", err)
 	}
-	log.Println(conn == nil)
 	Client = proto.NewGetCodeClient(conn)
 }
 
-func GetCodeByPath(path, head string) (code string) {
-	log.Println(path)
+func GetCodeByPath(path, stype, head, cookie string) (string, string, string) {
+	log.Println("path:", path)
 	defer qu.Catch()
 	getCodeReq := &proto.GetCodebyImgPathReq{
 		Imgpath: path,
+		Stype:   stype,
 		Head:    head,
+		Cookie:  cookie,
 	}
 	resp, err := Client.GetCodeByImgPath(context.Background(), getCodeReq)
 	if err != nil {
-		log.Fatalf("Get Code Err: %v", err)
-		return
+		log.Println("Get Code Err: ", err)
+		return "", "", ""
 	}
-	tmp := &ResultInfo{}
-	if err := json.Unmarshal([]byte(resp.Result), &tmp); err == nil {
-		log.Println(resp.Result, resp.String())
-		if tmp.Msg == "success" {
-			code = tmp.Info.Code
-		}
-	} else {
-		log.Fatalf("Get Result Err: %v", err)
+	log.Println(resp)
+	if resp.Code == "" {
+		log.Println("Get Result Err: ", err)
 	}
-	return
+	return resp.Code, resp.Head, resp.Cookie
 }
 
 // func main() {

src/analysiscode/config.json

@@ -1,6 +1,9 @@
 {
-  "serverport": ":8030",
-  "serveraddress": "codeserver.spdata.jianyu360.com:8030",
-  "servercodeaddress": "http://123.57.163.80:2119/v1/images/verify",
-  "timeout": 100
-}
+	"serverport": ":8030",
+	"serveraddress": "codeserver.spdata.jianyu360.com:8030",
+	"servercodeaddress": "http://123.57.163.80:2119/v1/images/discern?pic_type=",
+	"username":"jianyu001",
+	"password": "123qwe!A",
+	"timeout": 100
+}
+

src/analysiscode/proto/code.pb.go

@@ -27,7 +27,9 @@ type GetCodebyImgPathReq struct {
 	unknownFields protoimpl.UnknownFields
 
 	Imgpath string `protobuf:"bytes,1,opt,name=imgpath,proto3" json:"imgpath,omitempty"`
-	Head    string `protobuf:"bytes,2,opt,name=head,proto3" json:"head,omitempty"`
+	Stype   string `protobuf:"bytes,2,opt,name=stype,proto3" json:"stype,omitempty"`
+	Head    string `protobuf:"bytes,3,opt,name=head,proto3" json:"head,omitempty"`
+	Cookie  string `protobuf:"bytes,4,opt,name=cookie,proto3" json:"cookie,omitempty"`
 }
 
 func (x *GetCodebyImgPathReq) Reset() {
@@ -69,6 +71,13 @@ func (x *GetCodebyImgPathReq) GetImgpath() string {
 	return ""
 }
 
+func (x *GetCodebyImgPathReq) GetStype() string {
+	if x != nil {
+		return x.Stype
+	}
+	return ""
+}
+
 func (x *GetCodebyImgPathReq) GetHead() string {
 	if x != nil {
 		return x.Head
@@ -76,12 +85,21 @@ func (x *GetCodebyImgPathReq) GetHead() string {
 	return ""
 }
 
+func (x *GetCodebyImgPathReq) GetCookie() string {
+	if x != nil {
+		return x.Cookie
+	}
+	return ""
+}
+
 type GetCodebyImgPathResp struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Result string `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"`
+	Code   string `protobuf:"bytes,1,opt,name=code,proto3" json:"code,omitempty"`
+	Head   string `protobuf:"bytes,2,opt,name=head,proto3" json:"head,omitempty"`
+	Cookie string `protobuf:"bytes,3,opt,name=cookie,proto3" json:"cookie,omitempty"`
 }
 
 func (x *GetCodebyImgPathResp) Reset() {
@@ -116,9 +134,23 @@ func (*GetCodebyImgPathResp) Descriptor() ([]byte, []int) {
 	return file_proto_code_proto_rawDescGZIP(), []int{1}
 }
 
-func (x *GetCodebyImgPathResp) GetResult() string {
+func (x *GetCodebyImgPathResp) GetCode() string {
+	if x != nil {
+		return x.Code
+	}
+	return ""
+}
+
+func (x *GetCodebyImgPathResp) GetHead() string {
+	if x != nil {
+		return x.Head
+	}
+	return ""
+}
+
+func (x *GetCodebyImgPathResp) GetCookie() string {
 	if x != nil {
-		return x.Result
+		return x.Cookie
 	}
 	return ""
 }
@@ -127,21 +159,26 @@ var File_proto_code_proto protoreflect.FileDescriptor
 
 var file_proto_code_proto_rawDesc = []byte{
 	0x0a, 0x10, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x43, 0x0a, 0x13, 0x47, 0x65, 0x74,
+	0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x71, 0x0a, 0x13, 0x47, 0x65, 0x74,
 	0x43, 0x6f, 0x64, 0x65, 0x62, 0x79, 0x49, 0x6d, 0x67, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71,
 	0x12, 0x18, 0x0a, 0x07, 0x69, 0x6d, 0x67, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x07, 0x69, 0x6d, 0x67, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65,
-	0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x65, 0x61, 0x64, 0x22, 0x2e,
-	0x0a, 0x14, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x62, 0x79, 0x49, 0x6d, 0x67, 0x50, 0x61,
-	0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x32, 0x56,
-	0x0a, 0x07, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x4b, 0x0a, 0x10, 0x47, 0x65, 0x74,
-	0x43, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x49, 0x6d, 0x67, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1a, 0x2e,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x62, 0x79, 0x49,
-	0x6d, 0x67, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x1a, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74,
-	0x6f, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x62, 0x79, 0x49, 0x6d, 0x67, 0x50, 0x61,
-	0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x09, 0x52, 0x07, 0x69, 0x6d, 0x67, 0x70, 0x61, 0x74, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74,
+	0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x79, 0x70, 0x65,
+	0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
+	0x68, 0x65, 0x61, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x18, 0x04,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x22, 0x56, 0x0a, 0x14,
+	0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x62, 0x79, 0x49, 0x6d, 0x67, 0x50, 0x61, 0x74, 0x68,
+	0x52, 0x65, 0x73, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x61, 0x64,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x65, 0x61, 0x64, 0x12, 0x16, 0x0a, 0x06,
+	0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f,
+	0x6f, 0x6b, 0x69, 0x65, 0x32, 0x56, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12,
+	0x4b, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x49, 0x6d, 0x67, 0x50,
+	0x61, 0x74, 0x68, 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x43,
+	0x6f, 0x64, 0x65, 0x62, 0x79, 0x49, 0x6d, 0x67, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x1a,
+	0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x62,
+	0x79, 0x49, 0x6d, 0x67, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x42, 0x08, 0x5a, 0x06,
+	0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

src/analysiscode/proto/code.proto

@@ -8,10 +8,14 @@ package proto;
 //model
 message GetCodebyImgPathReq {
   string imgpath  = 1;
-  string head = 2;
+  string stype = 2;
+  string head = 3;
+  string cookie = 4;
 }
 message GetCodebyImgPathResp {
-  string  result = 1;
+  string code = 1;
+  string head = 2;
+  string cookie = 3;
 }
 
 //service

src/analysiscode/server.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"log"
 	"net"
+	"net/http"
 	"qfw/util"
 	"spiderutil"
 	"time"
@@ -28,7 +29,7 @@ func main1() {
 	util.Debug("port", spiderutil.Config.ServerPort)
 	lis, err := net.Listen("tcp", spiderutil.Config.ServerPort)
 	if err != nil {
-		log.Fatalf("failed to listen: %v", err)
+		log.Println("failed to listen: ", err)
 	}
 	//create a server
 	s := grpc.NewServer()
@@ -40,36 +41,71 @@ func main1() {
 func (cs *CodeService) GetCodeByImgPath(ctx context.Context, req *proto.GetCodebyImgPathReq) (*proto.GetCodebyImgPathResp, error) {
 	//根据图片地址解析出code
 	util.Debug("Img Path:", req.Imgpath)
-	code := getCode(req.Imgpath, req.Head)
+	code, headers, cookies := getCode(req.Imgpath, req.Stype, req.Head, req.Cookie)
+	head, _ := json.Marshal(headers)
+	cookie, _ := json.Marshal(cookies)
 	return &proto.GetCodebyImgPathResp{
-		Result: code,
+		Code:   code,
+		Head:   string(head),
+		Cookie: string(cookie),
 	}, nil
 }
 
-func getCode(path, head string) (code string) {
+func getCode(path, stype, head, cookie string) (code string, respheader http.Header, respcookie []*http.Cookie) {
+	// log.Println("cookie---", cookie)
+	// log.Println("stype---", stype)
 	client := req.C().SetTimeout(spiderutil.Config.TimeOut * time.Second)
 	headers := map[string]string{}
 	if head != "" {
 		json.Unmarshal([]byte(head), &headers)
 	}
+	cookies := []*http.Cookie{}
+	if cookie != "" {
+		json.Unmarshal([]byte(cookie), &cookies)
+	}
+	// log.Println("headers:", headers)
+	// log.Println("cookies:", cookies)
 	for times := 1; times <= 3; times++ { //重试三次
-		getCodeResp, err := client.R().SetHeaders(headers).Get(path)
+		request := client.R()
+		if len(headers) > 0 {
+			request.SetHeaders(headers)
+		}
+		if len(cookies) > 0 {
+			request.SetCookies(cookies...)
+		}
+		//下载验证码
+		getCodeResp, err := request.Get(path)
 		if err != nil {
-			log.Fatalf("get code by path err: %v", err)
+			log.Println("Get Code By Path Error: ", err)
 			continue
 		}
-
-		resuleResp, err := client.R().
+		//解析验证码
+		data := map[string]string{
+			"grant_type":     "",
+			"username":       spiderutil.Config.UserName,
+			"password":       spiderutil.Config.PassWord,
+			"scope":          "",
+			"client_id":      "",
+			"client_secret ": "",
+		}
+		codeResp, err := client.R().
 			SetHeader("accept", "application/json").
 			SetFileReader("file", "1", bytes.NewReader(getCodeResp.Bytes())).
+			SetFormData(data).
+			Post(spiderutil.Config.ServerCodeAddress + stype)
 			//SetFile("file", "C:/Users/topnet/Desktop/code.jpg").
-			Post(spiderutil.Config.ServerCodeAddress)
+			// Post(spiderutil.Config.ServerCodeAddress)
 		if err != nil {
-			log.Fatalf("analysis code by path err: %v", err)
+			log.Println("analysis code by path err: ", err)
 			continue
 		}
-		code = resuleResp.String()
-		if code != "" {
+		codeResult := map[string]interface{}{}
+		json.Unmarshal(codeResp.Bytes(), &codeResult)
+		log.Println("codeResult:", codeResult)
+		if yzm, ok := codeResult["r"].(map[string]interface{})["pic_str"].(string); ok && yzm != "" {
+			code = yzm
+			respheader = getCodeResp.Header
+			respcookie = getCodeResp.Cookies()
 			return
 		}
 	}

src/spiderutil/sysconfig.go

@@ -32,10 +32,12 @@ type config struct {
 	DetailChansize    int                        `json:"detailchansize"` //下载三级页的线程数
 	LogLevel          int                        `json:"logLevel"`       //日志基本1debug 2info 3warn
 	DayNum            int                        `json:"daynum"`
-	Modal             int                        `json:"Modal"`          //1列表页三级页分开采集，0原始采完列表采三级页（7000,7700）
-	IsHistoryEvent    bool                       `json:"ishistoryevent"` //只有7000为true
-	SiteType          []string                   `json:"sitetype"`       //网站类型
-	SiteColl          string                     `json:"sitecoll"`       //网站类型
+	Modal             int                        `json:"Modal"`            //1列表页三级页分开采集，0原始采完列表采三级页（7000,7700）
+	IsHistoryEvent    bool                       `json:"ishistoryevent"`   //只有7000为true
+	SiteType          []string                   `json:"sitetype"`         //网站类型
+	SiteColl          string                     `json:"sitecoll"`         //网站表名
+	ThreadBaseNum     int                        `json:"threadbasenum"`    //开启线程的数据基数
+	ThreadUpperLimit  int                        `json:"threadupperlimit"` //总线程上限
 	//补漏
 	Checkmaxpage  map[string]int `json:"checkmaxpage"`
 	CheckHourTime int            `json:"checkhourtime"`