验证码服务

src/analysiscode/README.md

@@ -0,0 +1 @@
+为爬虫采集提供下载、解析验证码服务

src/analysiscode/client.go

@@ -0,0 +1,89 @@
+package analysiscode
+
+import (
+	"analysiscode/proto"
+	"context"
+	"encoding/json"
+	"log"
+	qu "qfw/util"
+	"spiderutil"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+type ResultInfo struct {
+	Msg  string `json:"msg"`
+	Code int    `json:"code"`
+	Info Result `json:"r"`
+}
+
+type Result struct {
+	Time float64 `json:"time"`
+	Code string  `json:"code"`
+}
+
+var Client proto.GetCodeClient
+
+func InitCodeGrpcClient() {
+	defer qu.Catch()
+	log.Println("address:", spiderutil.Config.ServerAddress)
+	conn, err := grpc.Dial(spiderutil.Config.ServerAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		log.Fatalf("Init Connect Error: %v", err)
+	}
+	log.Println(conn == nil)
+	Client = proto.NewGetCodeClient(conn)
+}
+
+func GetCodeByPath(path, head string) (code string) {
+	log.Println(path)
+	defer qu.Catch()
+	getCodeReq := &proto.GetCodebyImgPathReq{
+		Imgpath: path,
+		Head:    head,
+	}
+	resp, err := Client.GetCodeByImgPath(context.Background(), getCodeReq)
+	if err != nil {
+		log.Fatalf("Get Code Err: %v", err)
+		return
+	}
+	tmp := &ResultInfo{}
+	if err := json.Unmarshal([]byte(resp.Result), &tmp); err == nil {
+		log.Println(resp.Result, resp.String())
+		if tmp.Msg == "success" {
+			code = tmp.Info.Code
+		}
+	} else {
+		log.Fatalf("Get Result Err: %v", err)
+	}
+	return
+}
+
+// func main() {
+// 	InitCodeGrpcClient()
+// 	code := GetCodeByPath("https://gdgpo.czt.gd.gov.cn/freecms/verify/verifyCode.do?createTypeFlag=n")
+// 	log.Println(code)
+// }
+
+// func main_client() {
+// 	//conn, err := grpc.Dial("127.0.0.1:8010", grpc.WithInsecure())//grpc.WithInsecure()方法弃用
+// 	conn, err := grpc.Dial("127.0.0.1:8010", grpc.WithTransportCredentials(insecure.NewCredentials()))
+// 	if err != nil {
+// 		log.Fatalf("did not connect: %v", err)
+// 	}
+// 	defer conn.Close()
+// 	client := proto.NewGetCodeClient(conn)
+// 	getCodeReq := &proto.GetCodebyImgPathReq{
+// 		Imgpath: "http://www.ccgp-tianjin.gov.cn/commons/image.jsp",
+// 	}
+// 	resp, err := client.GetCodeByImgPath(context.Background(), getCodeReq)
+// 	if err != nil {
+// 		log.Fatalf("get code err: %v", err)
+// 	}
+// 	tmp := &ResultInfo{}
+// 	json.Unmarshal([]byte(resp.Result), &tmp)
+// 	log.Println(resp.Result, resp.String())
+
+// 	log.Println(tmp.Msg, tmp.Info.Code)
+// }

src/analysiscode/config.json

@@ -0,0 +1,6 @@
+{
+  "serverport": ":8030",
+  "serveraddress": "codeserver.spdata.jianyu360.com:8030",
+  "servercodeaddress": "http://123.57.163.80:2119/v1/images/verify",
+  "timeout": 100
+}

src/analysiscode/proto/code.pb.go

@@ -0,0 +1,223 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.28.0
+// 	protoc        v3.19.4
+// source: proto/code.proto
+
+package proto
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+//model
+type GetCodebyImgPathReq struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Imgpath string `protobuf:"bytes,1,opt,name=imgpath,proto3" json:"imgpath,omitempty"`
+	Head    string `protobuf:"bytes,2,opt,name=head,proto3" json:"head,omitempty"`
+}
+
+func (x *GetCodebyImgPathReq) Reset() {
+	*x = GetCodebyImgPathReq{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_code_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *GetCodebyImgPathReq) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GetCodebyImgPathReq) ProtoMessage() {}
+
+func (x *GetCodebyImgPathReq) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_code_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use GetCodebyImgPathReq.ProtoReflect.Descriptor instead.
+func (*GetCodebyImgPathReq) Descriptor() ([]byte, []int) {
+	return file_proto_code_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *GetCodebyImgPathReq) GetImgpath() string {
+	if x != nil {
+		return x.Imgpath
+	}
+	return ""
+}
+
+func (x *GetCodebyImgPathReq) GetHead() string {
+	if x != nil {
+		return x.Head
+	}
+	return ""
+}
+
+type GetCodebyImgPathResp struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Result string `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"`
+}
+
+func (x *GetCodebyImgPathResp) Reset() {
+	*x = GetCodebyImgPathResp{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_code_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *GetCodebyImgPathResp) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GetCodebyImgPathResp) ProtoMessage() {}
+
+func (x *GetCodebyImgPathResp) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_code_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use GetCodebyImgPathResp.ProtoReflect.Descriptor instead.
+func (*GetCodebyImgPathResp) Descriptor() ([]byte, []int) {
+	return file_proto_code_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *GetCodebyImgPathResp) GetResult() string {
+	if x != nil {
+		return x.Result
+	}
+	return ""
+}
+
+var File_proto_code_proto protoreflect.FileDescriptor
+
+var file_proto_code_proto_rawDesc = []byte{
+	0x0a, 0x10, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x43, 0x0a, 0x13, 0x47, 0x65, 0x74,
+	0x43, 0x6f, 0x64, 0x65, 0x62, 0x79, 0x49, 0x6d, 0x67, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71,
+	0x12, 0x18, 0x0a, 0x07, 0x69, 0x6d, 0x67, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x07, 0x69, 0x6d, 0x67, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65,
+	0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x65, 0x61, 0x64, 0x22, 0x2e,
+	0x0a, 0x14, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x62, 0x79, 0x49, 0x6d, 0x67, 0x50, 0x61,
+	0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x32, 0x56,
+	0x0a, 0x07, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x4b, 0x0a, 0x10, 0x47, 0x65, 0x74,
+	0x43, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x49, 0x6d, 0x67, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1a, 0x2e,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x62, 0x79, 0x49,
+	0x6d, 0x67, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x1a, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x62, 0x79, 0x49, 0x6d, 0x67, 0x50, 0x61,
+	0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_proto_code_proto_rawDescOnce sync.Once
+	file_proto_code_proto_rawDescData = file_proto_code_proto_rawDesc
+)
+
+func file_proto_code_proto_rawDescGZIP() []byte {
+	file_proto_code_proto_rawDescOnce.Do(func() {
+		file_proto_code_proto_rawDescData = protoimpl.X.CompressGZIP(file_proto_code_proto_rawDescData)
+	})
+	return file_proto_code_proto_rawDescData
+}
+
+var file_proto_code_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_proto_code_proto_goTypes = []interface{}{
+	(*GetCodebyImgPathReq)(nil),  // 0: proto.GetCodebyImgPathReq
+	(*GetCodebyImgPathResp)(nil), // 1: proto.GetCodebyImgPathResp
+}
+var file_proto_code_proto_depIdxs = []int32{
+	0, // 0: proto.GetCode.GetCodeByImgPath:input_type -> proto.GetCodebyImgPathReq
+	1, // 1: proto.GetCode.GetCodeByImgPath:output_type -> proto.GetCodebyImgPathResp
+	1, // [1:2] is the sub-list for method output_type
+	0, // [0:1] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_proto_code_proto_init() }
+func file_proto_code_proto_init() {
+	if File_proto_code_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_proto_code_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetCodebyImgPathReq); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_code_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetCodebyImgPathResp); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_proto_code_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_proto_code_proto_goTypes,
+		DependencyIndexes: file_proto_code_proto_depIdxs,
+		MessageInfos:      file_proto_code_proto_msgTypes,
+	}.Build()
+	File_proto_code_proto = out.File
+	file_proto_code_proto_rawDesc = nil
+	file_proto_code_proto_goTypes = nil
+	file_proto_code_proto_depIdxs = nil
+}

src/analysiscode/proto/code.proto

@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+option go_package = "/proto";
+
+package proto;
+
+
+//model
+message GetCodebyImgPathReq {
+  string imgpath  = 1;
+  string head = 2;
+}
+message GetCodebyImgPathResp {
+  string  result = 1;
+}
+
+//service
+service GetCode {
+  rpc GetCodeByImgPath(GetCodebyImgPathReq) returns(GetCodebyImgPathResp);
+}

src/analysiscode/proto/code_grpc.pb.go

@@ -0,0 +1,105 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.2.0
+// - protoc             v3.19.4
+// source: proto/code.proto
+
+package proto
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
+const _ = grpc.SupportPackageIsVersion7
+
+// GetCodeClient is the client API for GetCode service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type GetCodeClient interface {
+	GetCodeByImgPath(ctx context.Context, in *GetCodebyImgPathReq, opts ...grpc.CallOption) (*GetCodebyImgPathResp, error)
+}
+
+type getCodeClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewGetCodeClient(cc grpc.ClientConnInterface) GetCodeClient {
+	return &getCodeClient{cc}
+}
+
+func (c *getCodeClient) GetCodeByImgPath(ctx context.Context, in *GetCodebyImgPathReq, opts ...grpc.CallOption) (*GetCodebyImgPathResp, error) {
+	out := new(GetCodebyImgPathResp)
+	err := c.cc.Invoke(ctx, "/proto.GetCode/GetCodeByImgPath", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// GetCodeServer is the server API for GetCode service.
+// All implementations must embed UnimplementedGetCodeServer
+// for forward compatibility
+type GetCodeServer interface {
+	GetCodeByImgPath(context.Context, *GetCodebyImgPathReq) (*GetCodebyImgPathResp, error)
+	mustEmbedUnimplementedGetCodeServer()
+}
+
+// UnimplementedGetCodeServer must be embedded to have forward compatible implementations.
+type UnimplementedGetCodeServer struct {
+}
+
+func (UnimplementedGetCodeServer) GetCodeByImgPath(context.Context, *GetCodebyImgPathReq) (*GetCodebyImgPathResp, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetCodeByImgPath not implemented")
+}
+func (UnimplementedGetCodeServer) mustEmbedUnimplementedGetCodeServer() {}
+
+// UnsafeGetCodeServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to GetCodeServer will
+// result in compilation errors.
+type UnsafeGetCodeServer interface {
+	mustEmbedUnimplementedGetCodeServer()
+}
+
+func RegisterGetCodeServer(s grpc.ServiceRegistrar, srv GetCodeServer) {
+	s.RegisterService(&GetCode_ServiceDesc, srv)
+}
+
+func _GetCode_GetCodeByImgPath_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetCodebyImgPathReq)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(GetCodeServer).GetCodeByImgPath(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/proto.GetCode/GetCodeByImgPath",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(GetCodeServer).GetCodeByImgPath(ctx, req.(*GetCodebyImgPathReq))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// GetCode_ServiceDesc is the grpc.ServiceDesc for GetCode service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var GetCode_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "proto.GetCode",
+	HandlerType: (*GetCodeServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetCodeByImgPath",
+			Handler:    _GetCode_GetCodeByImgPath_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "proto/code.proto",
+}

src/analysiscode/server.go

@@ -0,0 +1,93 @@
+package analysiscode
+
+import (
+	"analysiscode/proto"
+	"bytes"
+	"context"
+	"encoding/json"
+	"log"
+	"net"
+	"qfw/util"
+	"spiderutil"
+	"time"
+
+	"github.com/imroc/req"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	util.ReadConfig(&spiderutil.Config) //初始化参数
+}
+
+type CodeService struct {
+	proto.GetCodeServer
+}
+
+//server
+func main1() {
+	util.Debug("port", spiderutil.Config.ServerPort)
+	lis, err := net.Listen("tcp", spiderutil.Config.ServerPort)
+	if err != nil {
+		log.Fatalf("failed to listen: %v", err)
+	}
+	//create a server
+	s := grpc.NewServer()
+	//注册事件
+	proto.RegisterGetCodeServer(s, &CodeService{})
+	s.Serve(lis)
+}
+
+func (cs *CodeService) GetCodeByImgPath(ctx context.Context, req *proto.GetCodebyImgPathReq) (*proto.GetCodebyImgPathResp, error) {
+	//根据图片地址解析出code
+	util.Debug("Img Path:", req.Imgpath)
+	code := getCode(req.Imgpath, req.Head)
+	return &proto.GetCodebyImgPathResp{
+		Result: code,
+	}, nil
+}
+
+func getCode(path, head string) (code string) {
+	client := req.C().SetTimeout(spiderutil.Config.TimeOut * time.Second)
+	headers := map[string]string{}
+	if head != "" {
+		json.Unmarshal([]byte(head), &headers)
+	}
+	for times := 1; times <= 3; times++ { //重试三次
+		getCodeResp, err := client.R().SetHeaders(headers).Get(path)
+		if err != nil {
+			log.Fatalf("get code by path err: %v", err)
+			continue
+		}
+
+		resuleResp, err := client.R().
+			SetHeader("accept", "application/json").
+			SetFileReader("file", "1", bytes.NewReader(getCodeResp.Bytes())).
+			//SetFile("file", "C:/Users/topnet/Desktop/code.jpg").
+			Post(spiderutil.Config.ServerCodeAddress)
+		if err != nil {
+			log.Fatalf("analysis code by path err: %v", err)
+			continue
+		}
+		code = resuleResp.String()
+		if code != "" {
+			return
+		}
+	}
+	return
+}
+
+//func main() {
+//	log.Println(util.Config)
+//	conn, err := grpc.Dial("127.0.0.1"+util.Config.Port, grpc.WithInsecure())
+//	if err != nil {
+//		log.Fatalf("did not connect: %v", err)
+//	}
+//	defer conn.Close()
+//	client := proto.NewGetCodeServerClient(conn)
+//	codeImgReq := &proto.GetCodebyImgPathReq{
+//		Imgpath:
+//	}
+//	getCodeByImg(client, codeImgReq)
+//
+//}
+//

src/spiderutil/sysconfig.go

@@ -3,6 +3,10 @@
 **/
 package spiderutil
 
+import (
+	"time"
+)
+
 //系统配置
 type config struct {
 	Webport           string                     `json:"webport"`
@@ -77,6 +81,11 @@ type config struct {
 	JkMail map[string]string `json:"jkmail"`
 	//es
 	Es map[string]interface{} `json:"es"`
+	//获取验证码相关
+	ServerAddress     string        `json:"serveraddress"`
+	ServerPort        string        `json:"serverport"`
+	ServerCodeAddress string        `json:"servercodeaddress"`
+	TimeOut           time.Duration `json:"timeout"`
 }
 
 var Config config