hb:代码提交

invoiceScript/login.lua

@@ -0,0 +1,39 @@
+-- 登录
+local login_url = "https://tpass.beijing.chinatax.gov.cn:8443/#/login?redirect_uri=https%3A%2F%2Fetax.beijing.chinatax.gov.cn%2Fkxsfrz-cjpt-web%2Ftpass%2FtpassLogin.do&client_id=dcR7baR2cd6f428R8b7f9dfRRfc6bRd6&response_type=code&state=test"
+local browser_timeout = 1000*60*60*24
+-- login 登录
+function login()
+	browser_navagite(browser_timeout, login_url)
+	local login_success_flag,reflush_qr_times = false ,0
+	repeat
+		browser_sleep(1000)
+		print("获取二维码")
+		local ok = browser_wait_visible(1,"#qrcodeDiv img",1000*5)
+		local ok,img = browser_executejs(1000*5,0,'document.querySelector("#qrcodeDiv img").src')
+		--发送二维码图像，并等待扫码，超时后自动刷新页面
+		browser_send_img_chatbot('all','https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=f416abfc-2e3b-419c-a0a7-2f5427d81c9d',img)
+		--检查页面是否切换
+		local check_retry_times = 0
+		-- 循环检查页面是否变化，扫码登录是否成功，这里间隔5秒，尝试检查10次，50秒二维码也会失效，需要刷新tab页
+		repeat
+			print("检查扫码状态")
+			local ok,current_tab_url = browser_executejs(1000*5,0,'window.location.href')
+			if current_tab_url~=login_url then
+				login_success_flag=true
+			else
+				check_retry_times = check_retry_times+1
+				browser_sleep(1000*5)
+			end
+			check_retry_times=check_retry_times+1
+		until login_success_flag or check_retry_times>10 --检查扫码结束
+		--检查上一个循环是因为超时，还是真正登录成功了
+		if check_retry_times>10 then
+			print("超时了，重新刷新二维码")
+			browser_reload(browser_timeout)
+		else
+			print("登录成功")
+		end
+		reflush_qr_times = reflush_qr_times+1
+	until login_success_flag  or reflush_qr_times > 5 --登录检查结束
+	return login_success_flag
+end

invoiceScript/main.lua

@@ -0,0 +1,9 @@
+--剑鱼电子税票脚本
+require "scripts/inc/util"
+require "scripts/inc/fetch"
+--前边已经导入的包，下边可以直接使用，要控制好依赖关系
+require "logs/jy/login"
+
+--1. 登录
+local login_ok = login()
+print(login_ok)

program/REAdME.md

@@ -0,0 +1 @@
+剑鱼chromedp平台封装

program/bmgr/README.md

@@ -0,0 +1 @@
+浏览器资源管理

program/bmgr/browser.go

@@ -0,0 +1,234 @@
+/**
+ * 远程浏览器资源管理
+ * 申请浏览器，释放浏览器
+ */
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/chromedp/cdproto/page"
+
+	"github.com/chromedp/chromedp"
+)
+
+const (
+	//浏览器状态常量设置
+	browser_status_normal           = iota //初始值
+	browser_status_idle                    //空闲
+	browser_status_busy                    //忙
+	browser_status_waitting_destory        //待销毁，不可以分
+	//销毁服务器常量设置
+	destory_status_normal   = 0
+	destory_status_waitting = 1
+	//竞价服务器常量设置
+	append_machine_threshold  = 0.9 //扩容竞价服务器的临界值
+	destory_machine_threshold = 0.6 //缩容竞价服务器的临界值
+	append_machine_amount     = 5   //扩容每批次扩的数量
+	destory_machine_amount    = 2   //缩容竞价服务器每批次缩的数量
+	machine_monitor_interval  = 30  //竞价服务器检测间隔时间，这个时间要大于部署时间
+	//其他
+	default_browseres_per_machine = 4 //每台竞价服务器部署的浏览器数量
+)
+
+type (
+	//RemoteBrowser 远程浏览器
+	RemoteBrowser struct {
+		HttpAddr string `yaml:"httpAddr"` //Http访问地址
+		WsAddr   string `yaml:"wsAddr"`   //websocket地址
+		TunAddr  string `yaml:"tunAddr"`
+		//浏览器运行 状态
+		RunStatus int
+		//销毁状态，只有销毁状态是1，并且RunStatus是browser_state_idle空闲，才可以销毁
+		DestoryStatus int
+		Ctx           context.Context
+		Fn            context.CancelFunc
+	}
+	// 远程主机，竞价服务器
+	RemoteMachine struct {
+		Id string //服务器ID，在申请竞价服务器时获得
+		//TODO 与竞价服务器相关的其他属性
+		/**
+		释放状态，只有释放状态是1，
+		并且其名下所有浏览器释放状态是1+名下所有浏览器运行状态是闲的时候，
+		可以直接释放主机
+		*/
+		DestoryStatus int
+		browseres     []*RemoteBrowser
+	}
+	//远程浏览器在线管理，这里汇聚所有浏览器，以及其状态
+	OnlineRemoteBrowser struct {
+		Cap   int       //总容量
+		Busy  int       //忙的浏览器
+		Idle  int       //闲的浏览器
+		cache *sync.Map //缓存所有浏览器
+	}
+	//远程主机在线管理
+	OnlineRemoteMachine struct {
+		Cap   int       //申请的竞价服务器总量
+		cache *sync.Map //缓存所有竞价服务器
+	}
+)
+
+var (
+	//
+	olrb = &OnlineRemoteBrowser{
+		0, 0, 0, new(sync.Map),
+	}
+	olrm = &OnlineRemoteMachine{
+		0, new(sync.Map),
+	}
+)
+
+// init
+func init() {
+	//根据参数，自动控制扩容、缩容
+	go olrb.Monitor()
+}
+
+// Apply 申请远程浏览器
+func Apply(timeout int64) (b *RemoteBrowser, err error) {
+	ctx, _ := context.WithTimeout(context.Background(),
+		time.Duration(timeout)*time.Second)
+
+	for {
+		select {
+		case <-ctx.Done():
+			err = errors.New("Apply remotebrowser timeout")
+			return
+		default:
+			olrm.cache.Range(func(k, v interface{}) bool {
+				//待销毁的浏览器不再分配
+				if tmp, ok := v.(*RemoteBrowser); ok &&
+					tmp.RunStatus == browser_status_idle &&
+					tmp.DestoryStatus == destory_status_normal {
+					tmp.RunStatus = browser_status_busy
+					err = nil
+					b = tmp
+					log.Println("申请到浏览器", b.WsAddr)
+					return false
+				}
+				return true
+			})
+			if b == nil {
+				time.Sleep(time.Second)
+			} else {
+				return
+			}
+		}
+	}
+	return
+}
+
+// Repay 释放浏览器
+func Repay(b *RemoteBrowser) {
+	//destory(ws)
+	if b.Fn != nil {
+		b.Fn()
+		b.Fn = nil
+	}
+	b.RunStatus = browser_status_idle
+	log.Println("释放浏览器", b.WsAddr)
+}
+
+// destory 清理远程浏览器
+func destory(ws string) {
+	allocCtx, _ := chromedp.NewRemoteAllocator(context.Background(),
+		ws)
+	incCtx, _ := chromedp.NewContext(allocCtx)
+	ctx, _ := context.WithTimeout(incCtx, 5*time.Second)
+	ts, err := chromedp.Targets(ctx)
+	if err != nil {
+		return
+	}
+	for _, t := range ts {
+		newCtx, _ := chromedp.NewContext(ctx, chromedp.WithTargetID(t.TargetID))
+		chromedp.Run(newCtx,
+			page.Close(),
+		)
+
+	}
+}
+
+// AppendMachine 扩容
+func AppendMachine() {
+	//TODO 1. 调阿里云接口，扩竞价ECS
+	//TODO 2. 安装镜像，包含headless的docker
+	//TODO 3. 访问tun服务器的apply接口，获取tun资源
+	//TODO 4. 拼装docker run 参数，启动新申请竞价服务器的docker服务
+	//TODO 5. 注册竞价主机、远程浏览器到资源池
+	//TODO 6. 修正olrb,olrm的cap,idle参数
+
+}
+
+// Destory 主机释放
+func (rm *RemoteMachine) Destory() (err error) {
+	//TODO 设置名下远程浏览器状态
+	for _, v := range rm.browseres {
+		v.DestoryStatus = destory_status_waitting //该状态下，不允许再次分配
+	}
+	//TODO 等待名下所有浏览器状态变为空闲
+	idle := make(chan bool, 1)
+	go func() {
+		for {
+			status := true
+			for _, v := range rm.browseres {
+				status = status && (v.RunStatus == browser_status_idle)
+			}
+			if status {
+				idle <- true
+				return
+			}
+			time.Sleep(time.Second)
+		}
+	}()
+	select {
+	case <-idle:
+		//TODO 1.删除该竞价服务器绑定的浏览器资源
+		for _, v := range rm.browseres {
+			olrb.cache.Delete(v.WsAddr) //下掉资源池中的浏览器
+		}
+		//TODO 2.下掉资源池中的远程竞价服务器
+		olrm.cache.Delete(rm.Id)
+		//TODO 3.调用阿里云API物理删除竞价服务器
+		//待实现
+	}
+	return nil
+}
+
+// Monitor 远程浏览器资源监控，控制竞价服务器的扩容与缩容
+func (olrb *OnlineRemoteBrowser) Monitor() {
+	for {
+		if float32(olrb.Busy)/float32(olrb.Cap) >= append_machine_threshold {
+			//TODO 需要扩容
+		}
+		if float32(olrb.Idle)/float32(olrb.Cap) >= destory_machine_threshold {
+			//TODO 需要缩容，空闲太多
+		}
+		time.Sleep(time.Duration(machine_monitor_interval) * time.Second)
+	}
+}
+
+// LoadRemoteBrowserCfg
+func LoadRemoteBrowserCfg(addr string) (string, error) {
+	resp, err := http.DefaultClient.Get(fmt.Sprintf("%s/json/version", addr))
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	msg := struct {
+		Ws string `json:"webSocketDebuggerUrl"`
+	}{}
+	err = json.NewDecoder(resp.Body).Decode(&msg)
+	if err != nil {
+		return "", err
+	}
+	return msg.Ws, nil
+}

program/bmgr/config.yaml

@@ -0,0 +1,15 @@
+publishWebsocktAddr : "ws://127.0.0.1:9090"
+maxUseTimeout : 360
+remotebrowser :
+  - 
+    httpAddr : "http://192.168.3.240:29221"
+    tunAddr : "127.0.0.1:30001"
+  - 
+    httpAddr : "http://192.168.3.240:29222"
+    tunAddr : "127.0.0.1:30002"
+  - 
+    httpAddr : "http://192.168.3.240:29223"
+    tunAddr : "127.0.0.1:30003"
+  - 
+    httpAddr : "http://192.168.3.240:29224"
+    tunAddr : "127.0.0.1:30004"

program/bmgr/main.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"flag"
+	"jycdp"
+	"log"
+	"net/http"
+)
+
+type (
+	//RemoteBrowserConfig
+	RemoteBrowserConfig struct {
+		MaxUseTimeout int64 `yaml:"maxUseTimeout"`
+		//公共ws代理地址
+		PublishWebsocktAddr string           `yaml:"publishWebsocktAddr"`
+		Browseres           []*RemoteBrowser `yaml:"remotebrowser"`
+	}
+)
+
+var (
+	addr    = flag.String("proxy", ":9090", "服务监听地址")
+	cfgFile = flag.String("cfg", "./config.yaml", "配置文件")
+	cfg     = new(RemoteBrowserConfig)
+)
+
+// init
+func init() {
+	flag.Parse()
+	//TODO默认加载配置文件，也可以调用接口动态加减
+	err := jycdp.LoadConfig("./config.yaml", cfg)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, c := range cfg.Browseres {
+		ws, err := LoadRemoteBrowserCfg(c.HttpAddr)
+		if err != nil {
+			continue
+		}
+		log.Println(ws)
+		c.WsAddr = ws
+		c.RunStatus = browser_status_idle
+		olrm.cache.Store(c.WsAddr, c)
+	}
+
+}
+
+// main
+func main() {
+	http.HandleFunc("/", ProxyHandler)
+	http.ListenAndServe(*addr, nil)
+}

program/bmgr/wsproxy.go

@@ -0,0 +1,156 @@
+/*
+- websocket 流量代理
+- 连通代理后，需要申请远程浏览器，
+- 在超时，或者操作完成后，释放远程浏览器（主进程会做一些释放操作，如关闭所有TAB页面）
+*/
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+
+	"strings"
+
+	"time"
+)
+
+// 这里是ws代理的核心代码
+func ProxyHandler(writer http.ResponseWriter, request *http.Request) {
+	if strings.Contains(request.RequestURI, "json/version") {
+		log.Println("请求json/verson")
+		fmt.Fprintf(writer, `
+		{
+		   "Browser": "Chrome/114.0.5735.199",
+		   "Protocol-Version": "1.3",
+		   "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.199 Safari/537.36",
+		   "V8-Version": "11.4.183.25",
+		   "WebKit-Version": "537.36 (@581ada08cf738a4eb44f712c6f8cd40030e5c1a0)",
+		   "webSocketDebuggerUrl": "%s"
+		}`, cfg.PublishWebsocktAddr)
+		return
+	}
+
+	if strings.ToLower(request.Header.Get("Connection")) != "upgrade" ||
+		strings.ToLower(request.Header.Get("Upgrade")) != "websocket" {
+		_, _ = writer.Write([]byte(`Must be a websocket request`))
+		log.Println("无效的请求，不是websocket")
+		return
+	}
+	log.Println("requestURI:", request.RequestURI)
+	//TODO这里可以获取URL参数，用于设定超时，浏览器群组等等
+	hijacker, ok := writer.(http.Hijacker)
+	if !ok {
+		log.Println("写入头失败")
+		return
+	}
+	conn, _, err := hijacker.Hijack()
+	if err != nil {
+		log.Println("写入头失败")
+		return
+	}
+	defer conn.Close()
+	//TODO 开始处理请求
+	//1. 申请远程浏览器
+	b, err := Apply(10)
+	if err != nil {
+		log.Println("申请远程浏览器失败", err.Error())
+		return
+	}
+	defer Repay(b)
+	//不在平台级管理切换IP，在脚本中访问特殊地址实现切换IP
+	//ChangeBrowserIp(b.BindChangeIpAddr)
+
+	//TODO
+	target, err := url.Parse(b.WsAddr)
+	if err != nil {
+		log.Println("解析配置远程浏览器WS地址失败")
+		return
+	}
+	//2.拼装请求
+	req := request.Clone(request.Context())
+	req.URL.Path, req.URL.RawPath, req.RequestURI =
+		target.Path, target.Path, target.Path
+	req.Host = target.Host
+
+	var remoteConn net.Conn
+	remoteConn, err = net.Dial("tcp", target.Host)
+	if err != nil {
+		_, _ = writer.Write([]byte(err.Error()))
+		log.Println("远程浏览器WS地址连接失败")
+		return
+	}
+	defer remoteConn.Close()
+
+	err = req.Write(remoteConn)
+	if err != nil {
+		log.Println("远程浏览器写入失败")
+		return
+	}
+	//IO	复制 ,这点写的太恶心了
+	erres := make(chan error, 2)
+	success := make(chan bool, 1)
+	copyConn := func(src, target net.Conn, ctx context.Context, read bool, write bool) {
+		buf := make([]byte, 128, 128)
+		for {
+			readn := 0
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(10 * time.Second): //2秒内没有交互，退出
+				if readn == 0 {
+					erres <- errors.New("read timeout")
+					return
+				}
+			default:
+				nr, err := src.Read(buf)
+				readn = nr
+				if err != nil {
+					if err == io.EOF {
+						if write { //写回方向，没有数据，即结束
+							success <- true
+						}
+						return
+					} else {
+						log.Println(err.Error())
+						erres <- err
+						return
+					}
+				}
+				if nr > 0 {
+					target.Write(buf[:nr])
+				}
+
+			}
+		}
+	}
+	log.Println("所有准备工作完成，将要做IO复制")
+	ctx, fn := context.WithTimeout(context.Background(), time.Duration(cfg.MaxUseTimeout)*time.Second)
+	b.Fn = fn
+	go copyConn(conn, remoteConn, ctx, false, true) // response
+	go copyConn(remoteConn, conn, ctx, true, false) // request
+	select {
+	case <-ctx.Done():
+		log.Print("操作完成")
+	case err := <-erres:
+		log.Println(err.Error())
+	case <-success:
+		log.Println("传输完成")
+	}
+}
+
+// 切换IP
+func ChangeBrowserIp(ip string) {
+	client := new(http.Client)
+	resp, err := client.Get(ip)
+	if err != nil {
+		return
+	}
+	resp.Body.Close()
+}

program/captcharecognition/imgsum.go

@@ -0,0 +1,186 @@
+package main
+
+// 指纹识别库
+var numSign = [][]string{
+
+	{"Q", "0100000000000000000000000000000000000000000000000000000000000000000000011110000000000011111110001000011111111100010011111111111001001110000011100100110000001110000111000000011000011000000001100001100000000110000110000000011100111100000001110010110000001111"},
+	{"Y", "0001111111111101100111111111100111101111111110011111001110100000111111000000000011111100000000000111111100000000000111111111100000011111111110000000001111111000000011111111100000011111000000000011110000000000011111110000000011110011000000001110001100000000"},
+	{"7", "1100000110000000011100011000000001110001100000010111000000000111001100000000111100110000111111110011000011111100001100001111000001110001110000000111001111000000001101111000000000111100000000000011110000000000001111000000000000011100000000000001110000000000"},
+	{"T", "1110000000000000111000000000000011100000000000001110000000000000111111111111111111111111111111111111111111111111111111111111111111111111111111111110000000000000111000000000000011100000000000001110000000000000111000000000000011100000000000001110000000000000"},
+	{"E", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111100011111000111110001111100011111000011110001111110001111000111111000111000011111100011100001111110001110000111110001111000011111000111100001111100011110000111110000000000011"},
+	{"M", "0000011000000000000001100110000011111111111111111111111111111111111111111111111111111100000000001111111110000000111111111111000000001111111111000000011111111111000000000111111100000000000011110000000000111111000000111111111000001111111100000111111110000000"},
+	{"S", "0111111111111111111111111111111111111111111111111111111111111111110011110000110011001111000011001100001110001100110000111001110011000001111110000110000111111000011000011111000000000000111100000000000010000001000000001000000100000000100000010000000010000001"},
+	{"J", "1111111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
+	{"K", "1111111111111111111111111111111111111111111111111111111111111111111111111111111100000001100000000000000110000000000000111100110000000111111001000000111111111100000111111111110000011110111111000011110001111100111110000011111111110000000111111111000001101111"},
+	{"2", "0000000000011100000001100000110001100110000011100110011000001110011000000001111011000010001111101100000000111110110000000111111111100000111101111110000011100111111000011100011111100011110011101111111110001110111111110000111101111111000011110011111000000111"},
+	{"2", "0111000000000111011100000000111111110000000111111111000000111111111000000011101111100000011100111110110011110011111001111110001111111111110000110111111110000011001111111000001100011110000000110000000000000000000000000000000000000000000000000000000011110000"},
+	{"8", "0111111011111111011111111111111101111111111111111111111111001111111111111100011111101111100001111110011110000111111001111100011111100011111000111110101111100011111111111111011101111110111111110111111011111111011111001111111101111100111111110011100001111110"},
+	{"Z", "0000000000000111111000000000011111100000000011111110000000011111111000000011111111100000011111111110001111111011111000111111001111100111111000111110011111100011111011111100001111111111000000111111111000000011111111100000001111111000000000111111000000000011"},
+	{"E", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111100001110000111110000111000011111000011100001111100001110000111110001111000011111000111100001111100011110000111110000110000011111000000000001100000110000000110000000000000000"},
+	{"U", "1111111111111000111111111111111111111111111111111111111111111111000000000000111100000000000011110000000000000111000000000000011100000000000001110000000000001111000000000000111111111111111111111111111111111110111111111111100000000111000000000000011100000000"},
+	{"T", "0110000000000000111000000000000011100000000000001111000000000000011111111111111101111111111111110111111111111111011111111111111101111111111111110110000000000000011000000000000001100000000000000110000000000000011000000000000001100000000000000110000000000000"},
+	{"4", "0000000001110000000000011111000000000001111100000000000111110000000001111111000000001111101100000001111100110000001111110011000001111111001100000111101100110000111100011011000011111111111111111111111111111111111111111111111111111111111111111111111111111111"},
+	{"Z", "0000000011000000000001100000011101100111000001110110001100001111011000000011111101100000011111111110000111111011111000011111001101100011111000110110111111000011011111110000001101111111000000110111110000000011011111000000001101110000000000110000000110000000"},
+	{"N", "1111111111111111111111111111111111111111111111111111111000000000011111100000000000011111100000000000111111000000000001111111000000000001111110000000000011111100000000000111111100000000001111111111111111111111111111111111111111111111111111111111111111111111"},
+	{"E", "1111111111111111111111111111111111111111111111111110000111000011111000011100001111100001110000111110000111000011111000011100001111100001110000111110000111000011111000011100001111100001110000111110000111000011111000011100011111100000000001111110000000000111"},
+	{"2", "0111000000000111011100000000011101100000000011111110000000011111111100000001111111110000001111111111000001111011111000000111001111100000111100111110001111100011111100111100001111111111110000110111111110000111011111111000011100111111100001110001110000000111"},
+	{"R", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111100000111000001110000011100000111000001110000011100001111100001110001111111100111111111111110011111111111111110111111110111111011111110000111100111110000001110000011000000111"},
+	{"K", "0001111111111111000111111111111100111111111111110011111111111111000000001100000000000000111100000000000111110000000000111111100000000111111111000000111100011110000111100011111100011110001111110001000000000111000000000000000100000000000111011100000000111100"},
+	{"Z", "1110000000111111111000000111111111100000011111111110000111111011111000011111101111100011111100111110001111100011111101111110001111111111110000111111111110000011111111111000001111111111000000111111111100000011111110000000001111111000000000111111000000000011"},
+	{"N", "1111111111111111111111111111111111111111111111111111111111111111111111000000000001111110000000000111111000000000000111111110000000011111111000000000011111100000000000111111000000000001111110000000000011111100000000000111111100000000001111110000000000011111"},
+	{"P", "1111111111111110111111111111111000111111111111110011111111111111001111111111111100111111111111110011100001100010001110000110000000111000011100000011100011110000001110001111000000111111111000000001111111100000000111111100110000000111000011000000000000001100"},
+	{"E", "1111111111111111111111111111111101111111111111110111111111111111011000001100001101100000110110110110000011011011011000001110011101100000111001110110000011100111011000001110011101100000000001110000000000000111000000000000000000011111110000001011111111110011"},
+	{"6", "0111111111111000011110111011100001110011001110000111001000011100111100100001110011110010000011001100011000001100110001100000110011000110000011001100011000001100110001110001110011000111000111001100111111111000011011111111100001100011111110000110000111110000"},
+	{"D", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111100000000001111110000000000111111000000000011111100000000001111110000000000111111000000000011111100000000001111110000000000111111100000000011111110000000011111111000000001111"},
+	{"R", "0011111111111111000111111111111111111111111111111111111111111111111111111111111111111111111111111111000011000000111100001100000011100000110000001110000111110000011100111111100001111111111111000111111110111110011111110001111101111110000011110000000000000111"},
+	{"J", "0000000000000001000000000000001111111111111111111111111111111111111111111111111111111111111111100011000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001110000000000000111000000"},
+	{"4", "0000011111111000000001111111100000001111101110000001111000111000001111100011100000111100001110000111100000111000111100000011100011110000001110001111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111110000000000110000"},
+	{"5", "0000010000000000000000000000000000000000000000000000000000000000000000000010000010000000000000000000111111100001000011111110000100001111111000010000110001100001000011000110000100001100011000010000110001110011000011000111111100001100011111110001110001111110"},
+	{"9", "0000011000110000000011110001000000111111100001110111111111000111111111111101001111110111111100111110011011110011111011011111001111101101111100111110110111000111111001011100111101111101100111110111111111111110011111111111110000011111111110000000111111110000"},
+	{"4", "0000000011111100000000001111110000000011111111000000011111111000000111110111100000111110001111000011110000111100011100000011101001111111111111110111111111111111011111111111111101111111111111110000000000111000000000000011110000000000000001001111111111111111"},
+	{"K", "1111111111111111111111111111111111111111111111110000001100000110000000111000000000000111111000000000011111100000000111111111000000011110111110000011110001111100011111000111111011110000001111101110000000011111110000000001111111000000000011110100000000000111"},
+	{"B", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111110001111000011111000011100001111100001110000111110001111000011111111110110011111111111011001111111111101111111011111110111111101111111011111110111100000111111"},
+	{"U", "1111111111111000111111111111111011111111111111111111111111111111111111111111111100000000000011110000000000000111000000000000001100000000000000111000000000000011100000000000001100000000000001110000011000000111101011110010111111111111111111111111111111111110"},
+	{"D", "1111111111111100111111111111110011111111111111001111111111111100110000100000110011000000100011001100000000001100110000000000110011000000000011001100000000001100111000000001110011110000001110000111000001111000011111111111000001111111111111000011111111111110"},
+	{"O", "0001111111111111001111100000111110111100000001111111100000000111111110000000001111110000000000111111000000000011111110000000001111111000000000111111100000000011001110000000011100111100000011110001111000001111000111111111111001011111111111000000011111111100"},
+	{"U", "1111111111111000111111111111111011111111111111101111111111111111111111111111111100000000000011110000000000000111000000000001011100000000000000110000000000100011000000000011001100000000001100110000000001100111000000000110011100000000011011111111111111111111"},
+	{"A", "0111111111111100000000000000011100000000110111110000000011111111000001111111100000011111111100001111111100110000111111110011000011110000001100001111110000110000111111110011000011111111111100000001111111111000000001111111111100010000111111110001110000011111"},
+	{"W", "0111111111110000011111111111111100011111111111111100000011111111110000000000011100000000001111110000000111111110000011111111000000111111110000000111111000000000011111111100000001111111111100000000111111111110011000011111111101111110011111110111111111100111"},
+	{"W", "0011111111111111001111111111111100111111100011110011100001111111000000111111111000111111111100001111111111100000111111001100000011111100100000001111111110000000111111111111000000111111111111100000011111111111000000000111111100000000000011110000001111111111"},
+	{"2", "0000011000000000000001100000000000000110000000000000000000000000011000010000011101110001000001111110000000001111111000000001111111100000001111111110000001111011111000000111001111100001111000111111001111100011111111111100001101111111100000110011111110000011"},
+	{"D", "0000001000000000000000000000000000111111111111110011111111111111011111111111111101111111111111110111000000000011011100000000001101110000000000110111000000000011011100000000001111110000000000111111100000000111111111000000011111111100000011111111111111111110"},
+	{"K", "1111111111111111111111111111111111111111111111111111111111111111000000011000000000000011110000000000011111110000000111111111000000011110111110000011111000111100111100000001111111110000000011111100000000001111111100000000101101110000000000110111000000000001"},
+	{"T", "1100000000000000110000000000000011000000000000001110000000000000111111111111111011111111111111101111111111111110111111111111111011111111111111101100000000000000110000000000000011000000000000001100000000000000110000000000000011000000000000001100000000000000"},
+	{"F", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111000111000000111000011100000011100001110000001110100111000000111010011100000011101001110000001110000111000000111000011100000011100101110000001110110111000000"},
+	{"3", "0110000000000110111000011000011111100001100001111110000110000111111000011000011111100001100001111110000110000111111000111100011111100011110001111110001111000111111111111110011111111111111011110111111011111110011111001111111001111100111111100011100011111100"},
+	{"7", "0000011000000000110001100000000011000010000011001100001001111100110000000111110011000001111111001100000111110000110000111000000011000111000000001100111000000000110111100000000011110000000000001111000000000000111000100000000000000011110100000000111111110011"},
+	{"9", "0111111111100011111100011110001111110000111100111111000011110011111100001110001111110000110000111111000011000111111100011001111101111001101111110111111111111110001111111111110000011111111110000000011111100000000000000000000000000000000000000000000000000000"},
+	{"Y", "0110000010000000011110001000000001111000100000001111111000000000111111110000000000111111100000000001111111000000000011111111111100000011111111110000000111111111000000011111111100000001111111110000011110000000000111111000000000111111000000000001111100000000"},
+	{"Y", "0001110000000000001110000000000000111000000000000011110000000000001111110000000000111111000000000000111111000000000001111111111100000001111111110000000011111111000000011111111100000011110011000000011110001100000011110000110000111111000011000011101100000000"},
+	{"S", "0111110000111000011111000001100011111110000111011110111000011100110011110001110011000111000110001100001100011000110000111011100111000011111110010110001111111001011000111111000000000000111000000000000000000001000000100000000100000010000000000011111111111110"},
+	{"M", "1111111111111111111110000000000011111111100000000111111111100000000111111111110000000011111111110000000001111111000000000100111100000000000011110000000001111111000000111111111000111111111100000111111110000000111111000000000011111111111111111111111111111111"},
+	{"C", "0000011111100000000111111111100000011111111110000011111111111110001111111111111001111100001111110111000000001111111100000000111111100000000001111110000000000111111000000000011111100000000001111110000000000111111000000000011111100000000001111110000000000111"},
+	{"R", "1100000000111000011000000011000001101111111111110000111111111111000011111111111100001111111111110000111000110000000011100011000000001100001100000000110000111000000011100111110000001111111111100000011111101111000001111100011100000011110001110000000000000111"},
+	{"3", "0000000000000000010000000011100011000111001110001100011100011000110000110001100011000011000111001100011100011100111001110001100011111111001110000111111111111000011110111111100001110011111100000000000011100000000000000000000000000000111110000000001111111100"},
+	{"C", "0011111111111100011111000111110001111000001111000111000000011110111000000000111111100000000011101110000000000110110000000000011011000000000001101100000000000110110000000000011011000000000001101100000000000110110000000000011011100000000001100110000000001100"},
+	{"U", "1111111111110000111111111111100011111111111111001111111111111100111111111111110000000000000111100000000000011110000000000000111000000000000001110000000000000111000000000000011100000000000001110000000000001110000000000001111000000000000111101111111111111110"},
+	{"R", "1111111111110000000000000000000000111111111111110011111111111111001111111111111100111111111111110011000001100000001100000110000000110000011100000011000011110000001110011111110000111111111111000011111111111111001111111010111100011111010001110000000001000011"},
+	{"Y", "0011100001100000001111000000000000111111000000000011111100000000000011111100000000000111111111110000000111111111000000001111111100000000111111110000001111100000000001111000000000001111000000000011111100000000001110001000000001111000100110000111100010011000"},
+	{"S", "0111111110110111011111111011011111110111101101111110011110100011111001111100001111100011110000111110001111000011111000111110001111100011111100111110000111110111111000011111111111100000111111111110000011111111011100001111111101110000011111110111000001111111"},
+	{"X", "1100000000000001111000000000011111111000000001111111110000001111111111000011111000111111001111000001111111111000000011111111000000000111111100000000011111111000000011111111110000011110011111100011111000111111011111000001111111111000000011111111010000000111"},
+	{"G", "0000111110000000000111111100000000111111111000000111111111100000011111001111000001111000011100001111100000110000111100000011100011010000001110001101000000011000110100000001100011000000000110001101000000011000110110011111100011000001111110000110000111111000"},
+	{"7", "0000100000000000001110000000000000111000000000010011100000001111011110000000111101111000000111110111100000111110011110001111000001111000111000000111111110000000011111111000000001111110000000000111110000000000101111000000000011000001111100001100011111111100"},
+	{"C", "1100111111111110110111110011111111011110001101111101110000100111101110000000001100111000000000110011100000000011001110000000001100111000000000110011100000000011001110000000001100111000000000110011100000000011001110000000001100111000000000110001100000000011"},
+	{"8", "0000000000011100000111100011111000111110011111100111111101111111011111111111111111111111110001111111111111000011111011111100001111100111110000111110001111000011111000111110001111111111111101110111111111111111011111111111111101111111111111110011100011111110"},
+	{"U", "1111111111111000111111111111111011111111111111111111111111111111111111111111111100000000000011110000000000001111000000000000101100000000000010110000000000000011000000000000011100000000000011110000000000011111111111111111111111111111111111101111111111111100"},
+	{"Z", "1110000000001111111000000001111111100000011111111110000011111111111000011111101111100111111110111110011111100111110111111000011111111111000001111111111100000111111110001000011111110000000001111110000000000111000000000000000000000000111111000000000111111111"},
+	{"C", "0011111111111110011111000111111101111000010011110111000000000111111100000001011111110000000001111110000000000011111000000001001111100000000100111110000000010011111000000000001111100000000000111110000000000011111000000000001111100000000000110111000000000111"},
+	{"S", "0001110000000111001111100000011101111110000000110111111110000011011111111000001111111111100000111111111110000011111001111100001111100011110000111110000111000011111000011110001111100001111101111110000011111111011000001111111101110000111111110111000001111110"},
+	{"3", "0000100000000000000010000000000000001000000000000110110000000111111011011100011111111101110000111110010111000011111100111100001111110011110000111111001111100011111111111110011111111111111111110111110011111111011110001111111011000000000111001100000000000000"},
+	{"4", "0000001110000000000001111000000000101111100000000011111110000000001110011000000001111001100000000111000110000000111000011000000011111111111110001111111111111000111111111111100011111111111110000000000110000000000000011000000000000000000000000000111111111111"},
+	{"B", "0011111111111111001111111111111100111111111111110011100011100011001110001110001100111000111000110011100011100011001110001110001110111111101000111011111110110111101111110011111100011111001111110001111100111111000011110001111100000000000111110000000000011110"},
+	{"2", "0110000000000000011000000001100001100000001110000110000000111000110000000111100011000000011110001100000011111000110000001101100111000001110110001100001110011000110000111001100011100111000110001111111100011000111111100001100001111110000110000111110000011000"},
+	{"R", "0000000001100110111111111111111111111111111111111111111111111111111111111111111111100000111000001110000011100000111000001110000011100000111000001110000111111000111100111111110011111111111111000111111110111111011111110000111100111110000011110000000000000111"},
+	{"U", "0011111111111100001111111111111000111111111111110011111111111111000000110000011100000011000000110000000000000011000000000000001100000001000000110000000000000011000000011000011100111111111111110011111111111110001111111111110000000000000000001111111111111110"},
+	{"E", "1111111111111111111111111111111111111111111111111110000111000011111000011100001111100001110000111110000111000011111000011100001111100001110000111110000111000011111000011100001111100001110000111110000111000011111000011100001111100000000000111110000000000011"},
+	{"4", "0000000000111000000000000111100000000000011110000000000011111000000000011111100000000011110110000000011110011000000011110001100000011111000110000001111100011000001110000001100100111111111111110011111111111111001111111111111100111111111111110011111111111111"},
+	{"6", "1000000001100000100000001111100010000011111111100000001111111110000000111111111100000111001101110000011110100011000011101010000100001100011000010000110001100001000011000110000100001100011100110000110001111111000001100011111100000110001111100110000000111110"},
+	{"T", "1100000000000000110000000000011011000000000001101100000000000110110000000000011011111111111110001111111111111000111111111111100011111111111110001100000000000000111000001000000011100000000000001110000010000000111000000000000011100000000000001101111111111111"},
+	{"P", "1111111111111111111111111111111111111111111111111110000011000100111000001100000011100000110000001110000111000000111000011100000011100011110000001111111111000000111111111000000001111111100000000111111110000000001111100000000000011110000000000000000000000000"},
+	{"T", "1110000000000000111000000000000011100000000000001110000000000000111000000000000011100000000000001110000000000000111000000000000011111111111111111111111111111111111111111111111111111111111111111111111111111111111000000000000011100000000000001110000000000000"},
+	{"3", "1100000000000000110000000000000011000000000000000001100000000111001110001110011100111000111000110011100011100011001110001110001100111000111000110011100011100011001111111110001100011111011111110001111100111111000011110111111000000000011111100000000010000000"},
+	{"4", "0000000001111000000000001111100000000001111110000000001111011000000001110001100000011111000110000001111100011000001111100011100001111111111111110111111111111111001111111111111100111111111111110000000000011000100000000001100010000000111110000000000011111000"},
+	{"4", "0000001111111100000001111111110000000111101111000001111100111100001111100011100001111110001111000111110000111110011100000011111001110000001111101111111111111111111111111111111111111111111111110111111111111111011111111111111101111111111111110000000000111000"},
+	{"K", "1111111111111111111111111111111111111111111111111111111111111111111111111111111100000001100000000000000110000000000000111100000001000111111100000100111111110000000111111111100000011111111110000111110000111100111110011011111111110001100111111111000111001111"},
+	{"M", "0000000000001110000000000000011011111111111111111111111111111111111111111111111111111100000000001111111110000000011111111111000000001111111111000000001111111111000000110111111100000000000011110000000000111111000000111111111000011111111100000111111110010000"},
+	{"E", "0111111111111111011111111111111101111111111111110111111111111111011100011100111001110001110001100111000110000110011100011000011001110001100001100111000110000110011100011000011001110000000001100000000000000110000000000000000000000000000000001111111111111100"},
+	{"M", "1111111111111111111110000000000011111111100000000111111111100000000111111111110000000011111111110000000001111111000000000100111100000000000011110000000001111111000000111111111000011111111100000111111110000000111111000000000011111111111111111111111111111111"},
+	{"4", "0000000001110000000000001111000000000001111100000000000111110000000001111111000000001111101100000001111000110000001111100011000001111100001100000111110000110000111100000011000011111111111111111111111111111111111111111111111111111111111111111111111111111111"},
+	{"D", "0000000000001100000000000011010011111111111111111111111111111111111111111111111111111111111111111101100000001111111000011000111111100000000001111110000110000111111000000000011111100000000001111110000000000111111100000000111101111000000111110111111111111110"},
+	{"N", "1111111111111111011111111111111100011111111111110001111100000000000011111000000000001111110000000000101111100000000000011111110000000000011111000000000001111110000000000001111100000000000011110001111111111111000111111111111100011111111111110000111110000110"},
+	{"9", "0111111111000111111101011100011111100001111001111110000011100111111000001110011111100000111001111110000011100111111000001100011111100000110001111111000011001111011100011001111101111001101111110111111111111110001111111111110000111111111111000001111111111000"},
+	{"E", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111110001110000111111000111000011111100011100001111110001110000111111000111000011111100011110001111110001111000111111000111000011111100011100001111110001110000111111000000000011"},
+	{"E", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111100001111000111110000111000011111000011100001111100001111100111110000111110011111000011111001111100001111000111110000111100011111000000001001100000000000100110000000000010000"},
+	{"P", "0111111111111111011111111111111101111111111111110111111111111111011000111110000001100000111000010110010011100001011100001110000101110011111000000111111111000000011111111100000001111111100000000001111100000000000000000000000000000000000000001111111111111111"},
+	{"R", "1111111111111110111111111111111011111111111111001100000110000000110000011000000111000001100000011100001110000000110000111100000011100111111000001111111111100000111111111111000001111110111111000111111001111100011111000011110000111100001111000000000000011100"},
+	{"B", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111110001111000011111000011100001111100001110000111110001111000011111111111110011111111111111001111111111001111111011111100111111101111110011111110011100000111111"},
+	{"M", "1000000000000000000000000000000011111111111111111111111111111111111111111111111111111000001101001111111100000100011111111110010000011111111111100000001111111111000000001111111100000000000111100000000011111110000000111111110000011111111100000111111100000000"},
+	{"H", "1111111111111111111111111111111111111111111111111111111111111111000011000110000100000000011000000000000001100000000010000110000000001000011000000001100001100000000110000110000000001111111111110001111111111111000111111111111100011111111111110000111111111111"},
+	{"N", "0000111111111111000011111000000000000111100000001000011111000000100000111110000000000001111000000000000111110000000000001111100000000000001111000000000000111110000000000011111000000000000011110000000000001111000001111111111100001111111111110000111111111111"},
+	{"W", "1110000000000000111111100000000011111111111100001111111111111111111111111111111100111111111111110000000111111111000100000000111100010000011111110000011111111110001111111111000011111111101000001111110100000000111110000000000011111111100000001111111111110000"},
+	{"U", "0000000011111111000000000001111111111111110001111111111111111111111111111111111111111111111111000000111111111000000011100111000000000100001100000000000000110000000000000011000000000000001100000000000001110000000000001111000011111111111100001111111111100000"},
+	{"Y", "1110000000000000111100000000000011111100000000001111111000000000001111111000000000011111111111110000011111111111000000111111111100000011111111110000111100000000001111110000000000111100000000001111100000000000111000000000000011000000000000001111111111111111"},
+	{"F", "1111111111111111111111111111111111111111111111111111111111111111111000011111000011100001110000001110000111000000111000011100000011100001110000001110000111000000111000011100000011100001111000001110000111100000111000011110000011100001111000001110000111100000"},
+	{"K", "0000000000000001111111111111000011111111111100101111111111110110111111111111010011111111111111100000110000000110000011100000000000001111000000000000111100000100000111111000000000111011110010000111100111000000011110011110000011100000111100001110000011111000"},
+	{"2", "1000000000111000000000000001100001100000001110000110100000111000011000000111100001100000111110000110000011111000011000011111100001100011101110000110011110011000011111110001100001111110000110000111110000011000000111000001110000000000000001000000000000000100"},
+	{"D", "1111111111111111111111111111111111111111111111111111111111111111111000000000011111100100000001111110000000000111111000000000011111100110000001111110011000000111111000000000011111110000000011110111110000011110011111111111110000111111111111000011111111111000"},
+	{"3", "1110000111100111111000011100001111100001110000111110000111000011111000011100001111100001110000111110001111000011111001111100001111110111111001111111011111100111111111111110011101111110111111110111111011111111011111000111111101111100011111110011110001111111"},
+	{"N", "1111111111111111111111111111111111111111111111111111111111111111111111000000000011111110000000001111111000000000000111111000000000011111110000000000011111100000000000111111000000000001111110000000000011111100000000000111111100000000001111110000000000011111"},
+	{"F", "1111111111111110111111111111111000111111111111110011111111111111001111111111111100111111111111110011100011000000001110001100000000110000110000000011010011000000001101001100000000110000110000000011000011000000001100100000000000000011100000000000001100100000"},
+	{"D", "1111111111111111111111111111111111111111111111111111111111111111111100011000001111110001100000111111000110000011111100000000001111110001100000111111000110000011111100000000011111110000000011110111100000001111011111111111111000111111111111000011111111111111"},
+	{"8", "0011111110111111001111111111111100111111111111110011111111101011001111011100001100111101111000010011110011100001011111101111001100011111111100110001111111111111000111110111111100011111011111110111111000111110001000000001111000000000000011001000000000000000"},
+	{"W", "1100000000000000111111000000000011111111111000001111111111111110111111111111111000111111111111100000001111111110000000000011111100000000111111110000011111111100001111111111000011111111000000001111101000000000111110000000000011111111001100001111111111110000"},
+	{"G", "0001111111111111000011111111111100011111111011110011111111111111011111111111111101111111111111000111111100011100111100000001111011000000000111101100000000001100110000000000110011000000000011001100000000001100110000011111111011000000111111100111000011111100"},
+	{"5", "1111111110000111111111111000001111111111100000111110001110000011111000111000001111100011100000111110001111000011111000011100001111100001111001111110000111111111111000001111111111100000111111101110000000111100000011100000000000001110000000000000111000000000"},
+	{"T", "1110000000000000111000000000000011100000000000001111111111111111111111111111111111111111111111111111111111111111111010000000000011100000000000001110000000000000111000000000000011100000000000001110000000000000111000000000000011100000000000000000000000000000"},
+	{"Q", "0000000000000000000011111000000000011111110000000011111111100000001111111111000001111111111100001111000011111010011000011111100011100000001111101100000000011000110000000001110011000000000111001100000000011000110000000011100011100000001110000110000001111100"},
+	{"8", "0111111111111110011111111110111000111111111101110011111111110011011111011111101001111111111111101111111100011100110011100001110011001111000111001100011110011100111111111011110001111101111110000111100111111000011100011111101000000001111100110000000000000001"},
+	{"J", "0000000000000001000000000000001111111111111111111111111111111111111111111111111111111111111111100000000000000000000000000000000000000000000000000001000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000001111111111111100"},
+	{"R", "0011111111111111001111111111111100110000011000000111000001100000011100000110000011110000111100001011100111111000001111111111110000111111111111100011111110011111000111111000111100011111000001110000000000000011000000000000000100000000000000110000000000000011"},
+	{"4", "0000000001110000000000001111000000000001111100000000000111110000000001111111000000001111111100000001111001110000001111100111000001111100001100000111100000110000111100000011000011111111111111111111111111111111111111111111111111111111111111111111111111111111"},
+	{"Y", "0000000011011000000000000000000011100000000000001111000000000000111111000000000011111111000000000011111110000000000111111111111100000111111111110000001111111111000000111111111100001111100000000001111100000000001111000000000011111100000000001111000000000000"},
+	{"8", "0001100000111100000111100011111000111111011111110011111111111111011111111110011101110111100000110111111111100011011111011110001101111111111101110011111111111111001111111111111100111110011111100000000000011100000000000000000000111000000111000111110000011100"},
+	{"S", "0111111110000011011111111000001111110111100000111110011110000011111001111100001111100011110000111110001111000011111000011110001111100001111000111110000111100111111000011111111111100000111111111110000011111111011100001111111101110000011111100111000001111110"},
+	{"X", "1100000000000001111000000000011111111000000001111111110000001111111111000001111000111111001111000001111111111000000011111111000000000111111100000000011111111000000011111111110000011110011111100011110000111111011111000001111111110000000011111111000000000111"},
+	{"Y", "1110000000000000111100000000000011111100000000001111111000000000111111110000000000111111100000000000111111111111000001111111111100000001111111110000001111111111000001111000000000011111000000000011111100000000001111000000000011111000000000001111000000000000"},
+	{"M", "1111111111111111111111111111111111111111111111111111110000000000111111111000000000111111111100000001111111111100000000011111111100000001011111110000000000000111110000000011111111000001111111101100111111111000001111111000000011111111000000001111111111111111"},
+	{"B", "1011111111111111101111111111111110111111111111111011001111101111101100011100101110110001110000111011000111000011101100011110001110111101111000111011110111100011101111111110001110111111011111111011111101111111101111100011111110111110001111111011110000111110"},
+	{"J", "0000100000000000000000000000000100000000000000010000000000000101000000000000000100000000000001110011111111111111001111111111111100111111111111110011111111111110000000000000001000000000000000100000000000000000000000000000000010000000000000101000000000000010"},
+	{"A", "0100000000000000000000000000011101000000000111110000000011111111000001111111100011011111111100001111111100110000111111000011000011110000001100001111110000110000111111110011000011111111111100000001111111111000000001111111111100000000111111110000000000011111"},
+	{"E", "1111111111111110111111111111111011111111111111101111111111111110110000111000011011000011100001101100001110000110110000111000011011000011100001111100001110000111110000111000011111000000000001110000000000000111000000000000000000000000000000000000000000000000"},
+	{"5", "0111111100000111011111110000011101111111000001110110001100000111011000110000011101100011000001110110001100000111011000110000011111100011100001111110001110000111111000011100111111100001111111101110000111111110111000001111111011100000111111101110000011111110"},
+	{"Q", "0000011111000000000111111111000000011111111110000011111111111100011111111111110001111000001111000111000000011110111100000000111011111000000011101100110000000110110000000000011011000000000001101100000000000111111000000000111111100000000011110111000000011111"},
+	{"F", "1111111111110011111111111110001101111111111111110011111111111111001111111111111100111111111111110011111111111111001100001110000000110000110000000011000111000000001100011100000001110001110000000111000011000000011100001100000001110000110000000111000011000000"},
+	{"F", "1111111111111111011111111111111101111111111111110111111111111111011000001100000001100000110000000110000011000000011000001100000001100000110000000110000011000000011000001100000001100000000000000110000000000000000000000000000000011111100000000011111111100000"},
+	{"C", "0011111111111100011111000111110001111000001111000111000000011110111100000000111011100000000011101110000000000110110000000000011011000000000001101100000000000110110000000000011011000000000001101100000000000110110000000000011011100000000001100110000000001111"},
+	{"K", "1111111111111111111111111111111111111111111111111111111111111111111111111111111100000001110000000000000111000000000000111100000000000111111100000000111111111000000111111111100000011110111110000011110000111100111111000011111111111000000111111111000000011111"},
+	{"N", "0000000000011011000000000000000111111111111111101111111111111110111111111111111011111100000000000111110000000000001111110000000000011111110000000000011111000000000000111111100000000001111110000000000001111100000000000011111011111111111111101111111111111110"},
+	{"K", "0111111111111111011111111111111101111111111111110111111111111111000000001000000000000001110000000000001111110000000011111111100000111111111111000011111100111100011110000001111101110000000011110110011000000111000001100000001100000000000000010000000000000000"},
+	{"7", "1110000000000001111000000000011111100000000011111110000000001111111000000011111111100000001111111110000001111100111000011111000011100001111000001110001111000000111011110000000011101111000000001110111100000000111110000000000011111000000000001111000000000000"},
+	{"J", "0000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000011010000000000001101000000000000111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111101111111111111110"},
+	{"7", "0000000001100000111000000110000011110000011000011110000001100111111000000111111111100000011111111110000001111111111000000111110011100001111100001110001111100000111001111000000011111111110000001111111011100000111110001100000011111100110000001110010000010100"},
+	{"5", "0100000000000000011111111000001101111111100000110111111110000011011100111000001101110011100000110110001110000011011001111100001101100110111001110110000011111111011000001111111101110010111111100111000000111100000100000000000000010000000001111111000000001111"},
+	{"Z", "1110000000111111111000000111111111100000011111111110000111111011111000011111101111100111111100111110011111100011111001111110001111101111110000111111111110000011111111110000001111111111000000111111111100000011111110000000001111111000000000111111000000000011"},
+	{"*", "0000000000011100000111000011111000111110011111100111111001111111011111111111111111111111110001111111111110000011111011111000001111101111110000111110101111000011111110111110001111111111111101110111111011111111011111101111111101111110111111110011111001111110"},
+	{"C", "0000000100000000000000111111000000001111111111000001111111111100001111111111111000111110000111110011100000001111011100001000111101110000100001110110000010000111011000001010001101100000010000110110000011000011011000001100001101100000000000111110000000000011"},
+	{"W", "1111111111000110111111111111111000111111111111100000001111111110000000001111111000000000111111100000011111111100001111111111000011111111110000001111100011000000111111111000000011111111110000000011111111111100000001111111111000111110111111100011111111111111"},
+	{"E", "0011111111111111011111111111111111111111111111111111111011100011111111001100001111110000110000110111000011000011001110001100001100111000110000110011000011000011001100001100001100110000110000110011000011000011001100001100001100110000010000110011000000000011"},
+	{"R", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111100001110000001110000011000000111000001100000011100000110000001110001111100001111000111110000111100011111110001111111111111011011111111111111001111111001111100111111100011111"},
+	{"G", "0000000000000011000001111110100100011111111110000011111111111000011111111111110001111001101111000111001100011100111000110000111011100011000011001100011100011110110001110001110011000111000111101100011110011100110001111111110001100011111111000110001111111000"},
+	{"U", "1111111111100000111111111111000011111111111110001111111111111000000000000011110000000000000111000000000000001100000000000000110000000000000011000000000000011100000000000011100011111111111110001111111111110000111111111110000000000000000000000000110000000011"},
+	{"2", "1110000000001111111000000001111111100000000111111110000000111111111000000011101111100000011110111110000001110011111000001111001111100000111000111111001111100011111100111100001111111111110000111111111110000011011111111000001100111111000000110011111100000011"},
+	{"H", "1111111111111111111111111111111111111111111111111111111111111111111111111111111100000001111000000000000111100000000000011110000000000001110000000000000111000000000000011100000000000001110000000000000111000000000000111100000000000011110000001111111111111111"},
+	{"E", "0000111111111111000011111111111100111111111111000011111111111100001111111111110000111111111111000011001110001100001100111000110000110111100011000011011110001100001101111000110000110111100011000011101110001100001101000000110000001100000011001000110000000000"},
+	{"V", "1111111100000000111111110000000001111111110000000001111111111000000001111111111000000111111111110000011100111111000001110000011100000111001111110000001111111110000011111111100000011111110000000111111100000000111111000000000011111000000000001011111111111110"},
+	{"U", "0111111111111111011111111111111101111111111111110110011111100111011000000000011101100000000000110110000000000011011000000000001101000000000000110000000000000011110000000000001111000000000001111100000000001111011111111111111101111111111111110111111111111110"},
+	{"H", "1111111111111111111111111111111111111111111111111111111111111111111111111111111100000011110000000000001111000000000000111100000000000011110000000000001111000000000000111100000000000111110000000000000111000000000000011100000000000001110000001111111111111111"},
+	{"R", "0011111111111111001111111111111111111111111111101111111111111110111111111111111011111111111111101100000110000000110000011000000011000001100000001100011111000000111001111110000011111111111100000111111111111100011111100011111001111100000111100000100000011110"},
+	{"6", "0000000111111110000001111111111000000111111111110000111100110111000011110010011100001111001001110000110001100001000011000110000100001100011000010000111001110011000011110111111100000111001111110000011000111111110000000001110011000000000000001100000000000000"},
+	{"T", "1100000000000000110000000000000011000000000000001100000000000000111111111111000011111111111100001111111111110000111111111111000011111111111100001100000000000000110000000000000011000000000000001100000000000000110000000000000011000000000000001100000000000000"},
+	{"Q", "0000011111100000000011111111000000011111111110000011111111111100001111111111111000111000000111100111100000011111011100000000111101110000000011100110000000000111011000000000011101100000000001110110000000000111111100000000011111111000000011110111100000011111"},
+	{"V", "1011111111110111101111111111001100011111111001110001111111001101000111111110110100000111111111000000000111111110000000000111111100000000000111110000000000000111000000000001111100011000011111100001101111111000000110111111000000011111110000000001111100000000"},
+	{"F", "0001111111111111000111111111111100011111111111111001111111111111100110001111000010011000111100001001100111110000100110011111000010011001111100001001100111100000100110011110000010011001111000001000000111000000100000011100000010000000100000001000111111111110"},
+	{"U", "1001111111111110100111111111111010011111111111101000000001001110100000000100011010000000010001101000000000000110100000000000011010000000000001101000000000000110100000000000011000000000000001100000000000001110000111111111111000011111111111000001111111111100"},
+	{"R", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111100000111000001110000011100000111000001110000011100000111000001110000111100000111000011111000011111011111110001111111111111100011111111011111101111111101111110111111100001111"},
+	{"3", "0000000000000011000000000000000100000000000000000100000000001100110000111000111011000011100001101100001110000111110000111000011111000011100001111110001111000111111111111100111101111101111111000111110111111100011110011111110000000000001110000000000000000000"},
+	{"9", "1111111110001110111111111000111011111111100001101110001111000110110000011100011011100001110001101110000111000110111000011000111011100001100111101111001100111110111111111111110000111111111110000011111111110000000111111100000000000000000000000001111111111110"},
+	{"U", "1111111111111111111111111111111111111111111111110000000000001111000000000000011100000000000001110000000000000011000000000000001100000000000000110000000000000011000000000000011100000000000001110000000000001111111111111111111111111111111111111111111111111110"},
+	{"M", "1000000000000000100000000000000000010000000000000001111111111111000111111111111100011111111111110001111111000000000111111100000000011111111010000000111111111100000000111111111000000000111111110000000000111111000000000000011100000000000111110000000011111110"},
+	{"S", "0011111111000000001111100000110000111111111111111111111111111111111111111111111111111111111111111100011111000110110000111100011011000011110001101100001111001110110000011111110001100001111111000110000111111100000000000111110000000000000000000000000000000000"},
+	{"D", "0101111111111111000111111111111100011111111111110001111111111111000110100000001100011000000000010001110000000001000111000000000100011100000000010001110000100001000111100000001100011110001101110000111000111111111011111111111011101111111111101111111111111100"},
+	{"T", "1111000000000000111100000000000011100000000000001110000000000000111111111111111111111111111111111111111111111111111111111111111111111111111111111110000000000110111000000000000011100000000000001110000000000000111000000000000011100000000000001110000000000000"},
+	{"P", "1000000000000000100000000000000010001111111111110000111111111111000011111111111100001111111111110000111111111111000011000011000000001100001100000000110000110000000011000111000000001100011100000000111011110000000011111110000000000111111000000000011111100000"},
+	{"U", "1111111111111000111111111111111111111111111111111111111111111111111111111111111100000000000011110000000000000111000000000000001100000000000000110000000000000011000010000000001100001100000001110000000000000111001000010010111111111111111111111111111111111110"},
+	{"F", "1111111111111111111111111111111111111111111111111111111111111111111000011000000011100001100000001110000110000000111000011100000011100001110000001110000111000000111000011000000011100000000000000000000000000001000000000000000000000000000001110110000000001111"},
+	{"Z", "1110000001111111111000000111111111100000111111111110000111111111111000011111111111100011111001111110011111100111111001111110011111001111100001111111111110000111111111110000011111111110000001111111111000000111111110000000011111111000000001111111000000000111"},
+}

program/captcharecognition/main.go

@@ -0,0 +1,332 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"image"
+	"image/color"
+	"image/draw"
+	"image/jpeg"
+	_ "image/png"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/nfnt/resize"
+)
+
+/*
+1. 读取验证码
+2. 将验证码图片二值化
+3. 将二值化后的图片进行近似等分切割(可能有更好的方法,对于精度不高要求不高的话等分即可)
+4. 将切割后的图片进行识别(调用train()函数,手动标注验证码,生成指纹数组,也可以写入到一个文件中,然后读取)
+5. 将识别后的结果进行拼接
+*/
+var (
+	model               = flag.String("m", "train", "运行模式【train/predict】")
+	imageFile           = flag.String("img", "./demo.jpg", "要识别的图像文件")
+	threshold           = flag.Uint64("ct", 0xcccc, "颜色过滤阈值")
+	clearColorThreshold uint32
+)
+
+func init() {
+	flag.Parse()
+	clearColorThreshold = uint32(*threshold)
+	if _, err := os.Stat("./view"); err != nil {
+		os.Mkdir("./view", 0777)
+	}
+}
+
+// 判断r四周是否有任意两个连通白块，如果没有，则认为是噪点
+func imgConnect(ru, rd, rl, rr uint32) bool {
+	if (ru < clearColorThreshold &&
+		(rl < clearColorThreshold || rr < clearColorThreshold)) ||
+		(rd < clearColorThreshold && (rl < clearColorThreshold || rr < clearColorThreshold)) ||
+		(rl < clearColorThreshold && (ru < clearColorThreshold || rd < clearColorThreshold)) ||
+		(rr < clearColorThreshold && (ru < clearColorThreshold || rd < clearColorThreshold)) {
+		return true
+	}
+	return false
+}
+
+// 二值化
+func imgBarbarization(img image.Image) image.Image {
+	binImg := image.NewGray16(img.Bounds())
+	draw.Draw(binImg, binImg.Bounds(), img, img.Bounds().Min, draw.Over)
+	rect := binImg.Bounds()
+	// 遍历点像素点
+	for x := 0; x < rect.Dx(); x++ {
+		for y := 0; y < rect.Dy(); y++ {
+			// 获取颜色
+			r, _, _, _ := binImg.At(x, y).RGBA()
+			ru, _, _, _ := img.At(x, y+1).RGBA()
+			rd, _, _, _ := img.At(x, y-1).RGBA()
+			rl, _, _, _ := img.At(x-1, y).RGBA()
+			rr, _, _, _ := img.At(x+1, y).RGBA()
+			if r < clearColorThreshold && imgConnect(ru, rd, rl, rr) {
+				binImg.Set(x, y, color.White)
+			} else {
+				binImg.Set(x, y, color.Black)
+			}
+		}
+	}
+	return binImg
+}
+
+// 删除图片无用外部轮廓
+func imgCutSide(img image.Image) image.Image {
+	rect := img.Bounds()
+	// 开始x坐标
+	leftStartX := 0
+	for x := 0; x < rect.Dx(); x++ {
+		lxflag := false
+		for y := 0; y < rect.Dy(); y++ {
+			// 获取颜色
+			r, _, _, _ := img.At(x, y).RGBA()
+			if r == 0xFFFF {
+				lxflag = true
+				break
+			}
+		}
+		if lxflag {
+			leftStartX = x
+			break
+		}
+	}
+	// 开始y坐标
+	leftStartY := 0
+	for y := 0; y < rect.Dy(); y++ {
+		lyflag := false
+		for x := 0; x < rect.Dx(); x++ {
+			// 获取颜色
+			r, _, _, _ := img.At(x, y).RGBA()
+			if r == 0xFFFF {
+				lyflag = true
+				break
+			}
+		}
+		if lyflag {
+			leftStartY = y
+			break
+		}
+	}
+	// 结束x坐标
+	rightEndX := 0
+	for x := rect.Dx(); x > 0; x-- {
+		rxflag := false
+		for y := rect.Dy(); y > 0; y-- {
+			// 获取颜色
+			r, _, _, _ := img.At(x, y).RGBA()
+			if r == 0xFFFF {
+				rxflag = true
+				break
+			}
+		}
+		if rxflag {
+			rightEndX = x
+			break
+		}
+	}
+	// 结束y坐标
+	rightEndY := 0
+	for y := rect.Dy(); y > 0; y-- {
+		ryflag := false
+		for x := rect.Dy(); x > 0; x-- {
+			// 获取颜色
+			r, _, _, _ := img.At(x, y).RGBA()
+			if r == 0xFFFF {
+				ryflag = true
+				break
+			}
+		}
+		if ryflag {
+			rightEndY = y
+			break
+		}
+	}
+	// 创建新的图片
+	rectangle := image.Rectangle{
+		Min: image.Point{X: 0, Y: 0},
+		Max: image.Point{X: rightEndX - leftStartX, Y: rightEndY - leftStartY},
+	}
+	newSideImg := image.NewGray16(rectangle)
+	draw.Draw(newSideImg, newSideImg.Bounds(), img, image.Point{X: leftStartX, Y: leftStartY}, draw.Over)
+	return newSideImg
+}
+
+func cutImage(img image.Image) []image.Image {
+	rect := img.Bounds()
+	imgs := make([]image.Image, 0)
+	// 记录当前x的位置
+	nowCutStartX := 0
+	for x := 0; x <= rect.Dx(); x++ {
+		every := rect.Dx() / 4
+		if x%every == 0 && x != 0 {
+
+			// 创建新的图片
+			rectangle := image.Rectangle{
+				Min: image.Point{X: 0, Y: 0},
+				Max: image.Point{X: x + 3 - nowCutStartX, Y: rect.Dy()},
+			}
+			newImg := image.NewGray16(rectangle)
+			draw.Draw(newImg, newImg.Bounds(), img, image.Point{X: nowCutStartX, Y: 0}, draw.Over)
+			newResizeImg := imgCutSide(newImg)
+			newResizeImg = resize.Resize(16, 16, newResizeImg, resize.Lanczos3)
+			newResizeImg = imgCutSide(newImg)
+			newResizeImg = resize.Resize(16, 16, newResizeImg, resize.Lanczos3)
+			imgs = append(imgs, newResizeImg)
+			nowCutStartX = x + 3
+		}
+	}
+
+	return imgs
+}
+
+// 指纹验证
+func imgSign(imgBinary string) string {
+	if imgBinary == "" {
+		return imgBinary
+	}
+	imgBinarySign := strings.Split(imgBinary, "")
+	// 相似度
+	maxSimilarity := 0
+	num := ""
+	for _, sign := range numSign {
+		tmpSimilarity := 0
+		signArr := strings.Split(sign[1], "")
+		for k, s := range imgBinarySign {
+			if s == signArr[k] {
+				tmpSimilarity++
+			}
+		}
+		if maxSimilarity < tmpSimilarity {
+			maxSimilarity = tmpSimilarity
+			num = sign[0]
+		}
+	}
+	return num
+}
+
+// 将切割后的字符图片转换为二进制, 与指纹库进行比对
+func recognition(imgs []image.Image) (string, []string) {
+	signNum := make([]string, 0)
+	for _, img := range imgs {
+		tmpSignNum := make([]string, 0)
+		rect := img.Bounds()
+		for x := 0; x < rect.Dx(); x++ {
+			for y := 0; y < rect.Dy(); y++ {
+				// 获取颜色
+				r, _, _, _ := img.At(x, y).RGBA()
+				if r > 0x7788 {
+					tmpSignNum = append(tmpSignNum, "1")
+				} else {
+					tmpSignNum = append(tmpSignNum, "0")
+				}
+			}
+		}
+		signNum = append(signNum, strings.Join(tmpSignNum, ""))
+	}
+	res := make([]string, 0)
+	for _, v := range signNum {
+		res = append(res, imgSign(v))
+	}
+	return strings.Join(res, ""), signNum
+}
+
+func train() {
+	stringgen := ""
+	f, err := os.Create("train.txt")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	client := &http.Client{
+		Timeout: 2 * time.Second,
+	}
+	for {
+		resp, err := client.Get("https://beian.china-eia.com/servlet/validateCodeServlet?" + strconv.FormatInt(time.Now().UnixNano()/1e6, 10))
+		if err != nil {
+			fmt.Sprintf("打开图片失败:%+v", err.Error())
+			continue
+		}
+		fmt.Println(time.Now().UnixNano() / 1e6)
+
+		img, err := jpeg.Decode(resp.Body)
+		if err != nil {
+			fmt.Sprintf("解析图片失败:%+v", err.Error())
+			continue
+		}
+		resp.Body.Close()
+		dts, err := os.Create("./view/01.原始图片.jpeg")
+		jpeg.Encode(dts, img, nil)
+		// 图片二值化
+		binImg := imgBarbarization(img)
+		dts1, err := os.Create("./view/02.二值化后的图片.jpeg")
+		jpeg.Encode(dts1, binImg, nil)
+		// 图片切割
+		imgCutSilce := cutImage(binImg)
+		// 输出切割后的四张图片
+		dts3, err := os.Create("./view/03.第一个字符.jpeg")
+		jpeg.Encode(dts3, imgCutSilce[0], nil)
+		dts4, err := os.Create("./view/04.第二个字符.jpeg")
+		jpeg.Encode(dts4, imgCutSilce[1], nil)
+		dts5, err := os.Create("./view/05.第三个字符.jpeg")
+		jpeg.Encode(dts5, imgCutSilce[2], nil)
+		dts6, err := os.Create("./view/06.第四个字符.jpeg")
+		jpeg.Encode(dts6, imgCutSilce[3], nil)
+		// 验证码识别
+		result, resu := recognition(imgCutSilce)
+		fmt.Println("程序识别结果: ", result)
+		x := ""
+		fmt.Print("请输入./view/01.原始图片正确结果:  ")
+		fmt.Scan(&x)
+		if x == "q" {
+			return
+		} else if x == "k" {
+			continue
+		} else if len(x) != 4 {
+			fmt.Println("输入的格式不对")
+			continue
+		}
+
+		inputs := strings.Split(x, "")
+		stringgen = fmt.Sprintf("{\"%s\", \"%s\"},\n{\"%s\", \"%s\"},\n{\"%s\", \"%s\"},\n{\"%s\",  \"%s\"},\n", inputs[0], resu[0], inputs[1], resu[1], inputs[2], resu[2], inputs[3], resu[3])
+
+		_, err = f.Write([]byte(stringgen))
+		if err != nil {
+			fmt.Println(err)
+		}
+	}
+}
+
+// 识别预测
+func predict(imageFile string) (string, error) {
+	fi, err := os.Open(imageFile)
+	if err != nil {
+		return "", err
+	}
+	defer fi.Close()
+	img, err := jpeg.Decode(fi)
+	if err != nil {
+		return "", err
+	}
+	// 图片二值化
+	binImg := imgBarbarization(img)
+	// 图片切割
+	imgCutSilce := cutImage(binImg)
+	// 验证码识别
+	result, resu := recognition(imgCutSilce)
+	fmt.Println("程序识别结果: ", result, resu)
+	return result, nil
+}
+
+func main() {
+	switch *model {
+	case "train":
+		train()
+	default:
+		predict(*imageFile)
+	}
+}

program/captcharecognition/train.txt

@@ -0,0 +1,88 @@
+{"H", "1111111111111111111111111111111111111111111111111111111111111111111111111111111100000001110000000000000111000000000000011100000000000001111000000000000111100000000000011110000000000001111000000000000111100000000000011110000000000001111100001111111111111111"},
+{"5", "0011111111111111001111111111111100000001000000000011111111000111001111111100001100111111110000110011100111000011001111011100001100110001110000110011100111000011001110001110001100110000111111110011000001111111001100000111111000110000001111001000000000000000"},
+{"X", "1111000000000111111110000000111111111100001111100011111100111100001111111111000000001111111100000000011111110000000011111111110000011110111111100111110000011111111100000000111111110000000001111100000000000001000000000000000000000111111100000000111111111000"},
+{"Q",  "0111111111111100011110000011110001111000000111001110000000011110110000000000110011000000000011101100000000001100111000000000111011100000000011101110000000011111111100000011111101111000001111110111111111111011001111111111001100011111111000110000011111000001"},
+{"A", "0000000000000000000000000000001100000000000111110000000011111111000000011111111100000011111110000001111111111000011111110011100001111111001110001111000000111000111111100011100011111111001110001111111110111000111111111111100000011111111111000000001111111111"},
+{"F", "0000001100011111000000110000011111111111111001111111111111111110111111111111111011111111111111101111111111111110111001111000000011000011100000001100001110000000110000111000000011000011110000001100001111000000110000111100000011000011110000001100001111000000"},
+{"F", "1111111111111111111111111111111111111111111111111111111111111111111000011100000011100001110000001110000111000000111000011100000011100001110000001110000111000000111000011100000011100000000000000000000000000010000000000000001000000001111110000000011111111110"},
+{"Q",  "0011111111111110011111000101111011111100000011111111100000000111111110000000011111100000000000111110000000000011111000000000001111110000000001111111000000100111011110000010111101111100000111110011111111111100001111111111110000001111111110000000001111100000"},
+{"9", "0000000111000000000001111110001100000111111000110000011111100011000001111110000100001110011100010000110000110001100011000011000110001100001100011000110000110001000011000110001100001110011001110100111001101111010001110111111101000111111111100000011111111110"},
+{"4", "0000000001111111000000000111100000000000111110000000000111110000000011111111000000001111101100000011111000110000001111000011000001111000001100001111000000110000111111111111111111111111111111111111111111111111111111111111111100110000001100000011000000110000"},
+{"8", "0001000000011110000100110011111000011111111111110001111111111111000111111110001100011101110000010001110011100001000111001111000100011111111110110001111110111111000011110011111100000111001111100000000000001110000000000000000000000000000000001111111111111000"},
+{"P",  "1111111111111111111111111111111111111111111111111110000111100000111000011110000011100001111000001110000111100000111000011110000011100001111000001110000111100000111100111110000011111111110000001111111111000000011111111100000001111111100000000111111110000000"},
+{"Z", "0000000000000111111000000000111111100000000011111111000000011111111100000011111111110000011111111111000111111011111110111111001111100011111000111110011111100011111011111100001111111111000000111111111000000011111111100000001111111000000000111111100000000011"},
+{"C", "0000111111100000000111111111100000011111111110000011111111111100011111000011111001110000000111101111000000001111111000000000011111100000000001111110000000000111111000000000011111100000000001111110000000000111111000000000011111100000000001111110000000000111"},
+{"7", "0011000000000011111000000000001111100000000000111110000000001111111000010000111111100000011111111110000001111100111000011111000011100111110000001110111100001000111011100000110011111000000011001111000000000000111100000001000011110000000100001110000000010000"},
+{"T",  "1110000000000000111000000000000011100000000000001110000000000000111111111111111111111111111111111111111111111111111111111111111111111111111111111110000000000000111000000000000011100000000000001110000000000000111000000000000011100000000000001110000000000000"},
+{"F", "0000011000000000111111111111111111111111111111111111111111111111111111111111111111111111111111111110001111000000111000111100000011100011110000001110000111000000111000011100000011100001110000001110000111000000111000011100000011100001110000001110001110000000"},
+{"Y", "0000001100000000110000110010000011100000000000001111111000000000111111100000000011111110000000000011111110010000000011111111111100001111111111110000101111111111000110111111111100011111110010010001111100001000001111011000000011111001100000001110100011000000"},
+{"2", "0110001000111000011001100111100011000000011110001100011111111000110001111101100011000001100110001100001110011000111001111001100011111111000110000111111010011000011111001001100001111000010110000000000001000000000110000000000000001000011001110000111000000011"},
+{"X",  "1111110001011110001111110111110000011111111110000000111111110000000001111111000000000111111111000001111001111111001111100011111101111100000111111111000000011111111000000000111111000000000000010000000000000000000000000000000000000000000000000000000000000000"},
+{"A", "0000000000001110000000000011111100000000111111110000000111111100000011111111000000111111111001001111111001100001111111000110000111111000011010011110000001101001111111000111000111111110011100011111111111111001111111111111100100011111111110010000111111111101"},
+{"G", "0000000010111111000001111110111100011111111000110011111111111001011111111111100001111100111111000111000011011100111000001001111011000000100111101100000010001100110000001000110111000001100011011100010100001101110000011111110111000110111111100110011011111100"},
+{"Z", "1100000011111000110000001111100111000000111110001100010111111000110001111101110011000111100111011100111100011100110111110001100011111100000110001111110000111001111100000011100111110000001110011111000000111000000000000000000000001111000000000011111100011101"},
+{"9",  "0111111100011001111001110001100011100011000110001100001100011000110000110001100011000011000110001100001100011000110000110001100011000011001110001110001100111000011000101111100001110110111100000111111111110000011111111110000000111111111000000011111111000000"},
+{"7", "1100000000000000110000000001000011000000000100001100000001110000110000000111000011000000111100001100000111110000110000011110000011000011111000001100011100000000110111100000000011011110000000001101110000000110110110000010011011111000000001101111000000000110"},
+{"M", "1110000000000010000000000001000001111111111111000111111111111100011111111111110001111001000110000111111100011000011111111100000000011111111100100001001111111000000000001111100000000000001110000000000011111000000000111111100000011111111100110111111100010011"},
+{"J", "1111111111111110111111111111111011111111111111101111111111111110111111111111111011111111111110000000000000001001000000000000000000000000000000000000000000000001000001010000000000000000000000000000011100000000000000000000000000000000000100000001111111111110"},
+{"K",  "1111111111111111111111111111111111111111111111111000001110000000000000111100000000000011111000000000011111110000000111111111100000011111111110000011110110111100011111011011111011110001100111111111000010011111110000000000111111000000000001111100000010000011"},
+{"T", "0000000001000000000000000000000000000000000000000110000000000000011000000000000001100000000000000110000000000000011010000000000001101000000000000111111111111110011111111111111001111111111111100111111111111110011100000100000001100000010000000110000011000000"},
+{"Z", "0011100000110000001110000011011101111000000111111110000000011111111000000011111101100000011111110110000111111011011000111110001101100011111000110110111111100011011111110000011101111110011001110111110000000111011110000000011101110000000000110000000011000000"},
+{"Y", "1110001000001000111100000000000011111110000000001111111000000000001111111000000000001111111111110000011111111111000000011111111100000011111111110000111110000000000111110010000000111100000000001111100001100000111100000000000011000000011100000000000011110000"},
+{"4",  "0000011111110000000001111111000000001111101100000001111000110000001111100011000000111100001100000111100000110000111100000011000011110000001110001111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111110000000000110000"},
+{"A", "0001100000100111000000000001111100000000111111110000000011111111000001111111100000011111111100001111111100110000111111100011000011110100001100001111000000110000111111000011000011111111001100001111111111110000111111111111100000011111111111000000111111111111"},
+{"6", "0000000111111100000000111111110000001111111111000011111111111110001111111111111100111100110001110011100011000111011100001000011101100011100000110110000110000011011000011000001101110001110001111111000111111111001100001111111100110000111111110110000001111100"},
+{"7", "0000010000000000110000001000000011000000100001101100000000011110111000000011111011100000111111101100000011110000110001111100000011000111000000001100111000000000110111000000000011111000000000001111000000000000111100000000000000000000000000000011111111111111"},
+{"K",  "1111111111111111111111111111111111111111111111110000000110000000000000111100000000000011111000000000011111110000000111111111100000011110111110000011111001111100011111000011111011110000000111111111000000011111110000000000111111000000000001110000000000000011"},
+{"2", "0110000000011111011000000001111101100000000111111110000000011111111000000011111111100000011111111110000001111111111000001111011111100000111011111110001111001111111100111100111111111111100011110111111110000111011111110000011100111110000001110011110000000111"},
+{"9", "0000000000110000000111000011000001111110001110000111111100111000011111110001100011100011000110001100001100011010110000110001101011000011000110101100001100111010111000110011101001110110111110100111111111111010011111111110101000111111110000100001111110000001"},
+{"N", "1111111111111110111111111111111011111111111111101111100000000010011111000000001000111110000000100001111100001010000011111100101000000011111000000000000111110000000000001111100000000000011111001111111111111100111111111111110111111111111111011111111111111101"},
+{"P",  "1111111111111110111111111111110011111111111111011110000110000011110000011000001111000001100000111100000110000011110000011000001111000011100000111110001110000011111001111000001111111111100000111111111100000011011111110000001101111111000000000111111100000000"},
+{"B", "0000110100010000111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111000111110101111100101111000111110011111100011111001111110001111111111011001111111111101111111011111110111111101111111011111110011101001111111"},
+{"J", "1111111111111111111111111111111111111111111111111111111111111111111111111111111100000000000111000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000000000000000000001000000000000000100000000000000010000"},
+{"V", "0001111100001000000111111100100000001111111110000000011111111100000000011111111100000000011111110000000000001111000000000000001100000000000011110000000001111111000000001111110000000011111100000000111111010000000111110001000000011100000111001101000000111100"},
+{"Z",  "1110100000111111111010000111111111110000011111111111001111111011111100111111101111110011111010111111011111101011111111111100001111111111110000111111111100000011111111100001001111111110000100111111100000010011111110000001101111110000000110111111000000011011"},
+{"S", "1000000000000000100000000000000010000000000000001000000000000100000000000000010000011100000100110111111000000111011111110000001101111111100000111111011110000011111000111100001111100001110000111110000111100011111100011111111101110001111111110111000001111110"},
+{"K", "0011000000000000000000000000000000111111111111110011111111111111001111111111111100111111111111110000000011000000101000001110000010000001111100001011011111111000101101110111110010111111000111100011111000011111001110000000111100111000000001110011100000000011"},
+{"P", "1111111111111110111111111111110011111111111111000111111111111100011100111000000001110011100000000111000110000000011000111000000001110011100000000111111110000000011111111000000001111111100000000001111010000001000000001100000100010000110000010000000011000001"},
+{"J",  "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100000000001100000000000000110000000000000011000000000000001000000000000000000000000000000001000000000000000100000000000000010000000000000000000000000000000000000"},
+{"J", "0000000001000000111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111000000110001000000000000000100000000000000000000000000000000000000000000000000000000110000011000000011000001100000001100000111000000110000011000"},
+{"X", "1000000000000000100000000011000011100000011100001111000001110000111110001110000001111101110000000011111110000000000111110000000000001111100100000001111111010000001110111111100001111000111110001110000011111000111000000011100010000000000110001000000000011000"},
+{"C", "0001111111000000001111111111000001111111111100001111100011110000111000000111000011100000001110001110000000111000110000000001100011000000000110001100000000011000110000000011100011000000001110001100000000111000110000000011100001111111001100000111111110110000"},
+{"C",  "0011111111111110011111000011111101111000000011110111000000000111111100000000011111110000000001111110000000000011111000000000001111100000000000111110000000000011111000000000001111100000000000111110000000000011111000000000001111100000000000110111000000000111"},
+{"N", "0011111111111111001111111111111100111111111111110011111111111111001111110000000000111111000000000001111100000000000011111100000000000111110000000000011111110000000001111111000000000001111111000000000001111100100000110011111110000011000111111000000000011111"},
+{"X", "1111111111111111011111111111111101111000000001110111110000011111011111100001111000011111001111000000111111110000000001111110000000000111111100000000011111111100000111110111111000011111011111110111111000011111011111000000011101100100000000110000010000000000"},
+{"8", "0000110000011100000111100011111100111111001111110011111111111111001111111100001100110111110000110011001111000011001100011110001100111111111100110011111101111111001111100111111100011100001111100000000000011100000000000000000000111000000111000111110000011100"},
+{"S",  "0111111110000011011111111000001111110111100001111110011110000111111001111100011111100011110001111110001111000111111000011110011111100001111001111110000111100111111000011111111111100001111111111110000111111111011100011111111101110000011111100111000001111110"},
+{"Q", "0000011111000000000111111110000000011111111100000011111111111000011111111111110001111000001111000111000000011100111100000001111011110000000111001100000000001110110000000000110011000000000011001100000000001110111000000001111011110000000111100111000000011111"},
+{"P", "0011111111111001001111111111001111111111111111011111111111111101111111111111110111111111111111011100000110000000110010011000000011001001100000001100011110000000111001111000000011111111100000000111111100000000011111101100000000111100000000000001100110000000"},
+{"W", "1111111111110000111111111111111100111111111111110000011111111111000001100000111100000110111111110000011111111110001111111111000011111111100000001111100000000000111111111000000011111111111100000011111111111110000011111111111100001110011111110011111110001111"},
+{"4",  "0000011111111111000011111111111100001111111111110011111111100000001111111110000001111110111000000111100011100000111000001110000011100000111000001111111111111110111111111111111011111111111111001111111111111110111111111111111011111111111111000000000011100000"},
+{"3", "0110000000000110111000011000011111100001100001111110000110000111111000011000011111100001100001111110000110000111111000111000011111100011110001111110001111000111111111111100011111111111110011110111111011111110011111001111111001111100111111100011100011111100"},
+{"A", "0000000000011110000000000011111000000001111111100000011111110000000111111111000011111110011100001111110001110000111000000111000011100000011100001111110001100000111111100110000011111111111000000011111111110000000001111111110000000001111111110000000001111111"},
+{"D", "0001111111111111000111111111111100011111111111110001111111111111000111000100001100011100010000110001110001000011000111000000001100011100000000110001110000000011000111000000001100011110000001110000111100001111000011111111111000000111111111101100011111111100"},
+{"Z",  "1100000011111100110000011111110011000011111111001100001111111100110001111100111011001111100011111101111100001101111111100000111011111110000011101111111000001101111100000000110111110000000011001110000000001100111000000000100100000000000000010000000000000001"},
+{"N", "0001111111111111000111111111111100011111111111110001111111111111000011111000000000001111100000000000111111000000000010111110000000001001111000000000100011110000000010001111100010001000011111001000100000111110100000000001111110000000000011111000000000001111"},
+{"5", "0000111111111111000111111111111100011000000000000011111110000100001111111000011000111111100001100111000110000110011100011000011001110001100001100111000110000110011100011100111001110001111111000011000111111100001100011111110000110000011110001000000000001000"},
+{"X", "1110000010111000111100001111100011111000111000000111110111111000011111111000100000011111100000000000111110000000000111111110000000111101111000000111100011111000111100000111100011100000001110001000000000111000100111000000000000011100000000010000111000000011"},
+{"X",  "1111110100011110111111110011111010111111001111000001111111111000000111111111000000010111111100000001011111110000000001111111110000001111111111000101111001111110011111000011111101111100000111111111010000001111111100000000011111100000000000111100000000000001"},
+{"D", "0000001000000000111111111111110011111111111111101111111111111100111111111111111011111111111111101101000000001100110000000000110011000000000011001100000000001100110100000001111011011000000111101101100000011100111110000001111011111000000111001111100010011100"},
+{"R", "0111111111111100001111111111100011111111111111101111111111111110111111111111111011111111111111101100110110000000110010111000000011000111100000001100011111000000111001111110000011111111111110000111111111111100011111110011111001111111000111100000011100011110"},
+{"X", "1111011000000111111111100001111111111111001111000011111101111100001111111111000000001111111111000000111111111100000011111111110000111110111111000011110001111111111110000001111111100000000111111100000000011011000000000001110000000000000110000111111111111111"},
+{"P",  "0000111111111111000011111111111100001100001110000000110000111000000011000011100000001100011110000000110011111000000011111111000000001111111100000000011111110000000001111100000000000011100000000000000000000000000000000000000000000000000100000000000000000000"},
+{"N", "0001100000000000000001100010000011111111111111111111111111111111111111111111111111111111111111111111110000001000011111100000000000111111000000000001111111000000000001111110000000000011111100000000000111111001000000001111110100000000011111100001000001111111"},
+{"Y", "1111111111111111111111111111111111100000000000001111100000000000111111000000000011111110000000000011111111000000000011111111111100000111111111110000001111111111000000111111111100001111110000000001111100000000001111110000000011111100110000001111000011000000"},
+{"U", "1111111111110000111111111111000011111111111110001111111111111000000011000011111000001000001111100000000000001110000000000000111000000000000011000000000000111100000000000011110011111111111111001111111111110000111111111110011000010010000001110001111000000011"},
+{"X",  "1111110000011110011111110011111000111111001111100001111111111000000011111111000000000111111100000000011111110000001001111111110000111111111111000011111001111110001111000011111101111100000111111111000000011111111100000000011111100000000000111100000000000001"},
+{"Y", "0000000000000010000000000000000000000000000000100000000010000010000000000000100011100000000000111111100000000000111111100000000001111111001000010001111111111111000001111111111100000001111111110000001111111111000011111000000100011111000000010011111000110000"},
+{"S", "0111000000000000011110000011000001111000001100000111110000011000011111100001101011101110000110001110111000011000110001110001100011000111000110001100011110111010111000111111100001100011111100000110011111100000000101011110000000000000000000000000000000000000"},
+{"A", "0000000000111111000001100111111100000011111111100000111111111000001111111011111000111110001111000011100000111010001111100011101000111111101110000011111111111000000011111111110000000011111111110000000011111111000000000001111100000000000000111111111111110001"},
+{"U",  "1111111111111101111111111111110011111111111111100000000000011111000000000000111100000000000011110000000000000110000000000000011000000000000001100000000000000110000000000000111000000000000011100000000000011100111111111111110011111111111111001111111111111100"},
+{"Y", "1110000000000000111100000000000011111000000000001111110000000000111111100000000001111111100000000011111110000000000111111111111100000111111111110000011111111111000000111111111100000011111111110000111110000000000111110000000000111111000000000111110000000000"},
+{"B", "0011100000110000001100000011000011110111111111111111111111111111111111111111111111111111111111111111111111111111111000011100111111100001110001111110000111000111111000111100001111111111111001111111111011111111111111111111111101111111011111110011100000111111"},
+{"3", "0010000100000111111000011100011111100001110000111110000111000011111000001100001111100011110000111111001111100011011111111110001101111111111101110011111001111111001111100111111100011100011111100000000000011100000000000000000000000111111000000001111111111000"},
+{"Q",  "0111111111111000011100000111100001110000001111001110000000011100110000000001110011000000000011001100000000001100110000000000110011100000000111101110000000011110011100000011111101111000011111110111111111110011011111111111001100011111111000110000111110000001"},
+{"Z", "0000000000000111111000000000011111100000000011111110000000011111111000000011111111100000011111111110000111111011111000111111001111100011111000111110011111100011111011111110001111111111000000111111111000010011111111100001001111111000000000111111000000000111"},
+{"C", "0000000000000011000001111110001100011111111110110011111111111011001111111111111101111100001111100111000000001110111000000000111111100000000001111110000000000111111000000000011111100000000001111110000000000111111000000000011111100000000001111110000000000111"},
+{"4", "1100000011110000000000001111000000000011111100000000001110110000010011110011000000011110001100000011110000110000011110000011000001111111111111100011111111111110001111111111111000111111111111100000000000110000000000000011000000000000000000000101111111111111"},
+{"D",  "1000111111111111111011111111111101001110000000110100110000000001000011000000000100001100000000010000110000000001000011000000000100001110000000110000111000000011000011110000001100000111001001110000011111101111000001111111111000000011111111100000001111111100"},

program/config.go

@@ -0,0 +1,17 @@
+package jycdp
+
+import (
+	"io/ioutil"
+
+	"gopkg.in/yaml.v2"
+)
+
+// LoadConfig
+func LoadConfig(cf string, target interface{}) error {
+	bs, err := ioutil.ReadFile(cf)
+	if err != nil {
+		return err
+	}
+
+	return yaml.Unmarshal(bs, target)
+}

program/go.mod

@@ -0,0 +1,41 @@
+module jycdp
+
+go 1.19
+
+require (
+	github.com/cespare/xxhash/v2 v2.2.0
+	github.com/chromedp/cdproto v0.0.0-20240102194822-c006b26f21c7
+	github.com/chromedp/chromedp v0.9.3
+	github.com/gabriel-vasile/mimetype v1.4.3
+	github.com/mattn/go-sqlite3 v1.14.22
+	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
+	github.com/studio-b12/gowebdav v0.9.0
+	github.com/yuin/gopher-lua v1.1.1
+	go.mongodb.org/mongo-driver v1.13.1
+	golang.org/x/net v0.17.0
+	gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/gobwas/httphead v0.1.0 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.3.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+)

program/go.sum

@@ -0,0 +1,107 @@
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chromedp/cdproto v0.0.0-20231011050154-1d073bb38998/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
+github.com/chromedp/cdproto v0.0.0-20240102194822-c006b26f21c7 h1:XDMhsjCzDu+sTkUz2VJxBINfDhbcoHHzJWWVqBt9WpA=
+github.com/chromedp/cdproto v0.0.0-20240102194822-c006b26f21c7/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
+github.com/chromedp/chromedp v0.9.3 h1:Wq58e0dZOdHsxaj9Owmfcf+ibtpYN1N0FWVbaxa/esg=
+github.com/chromedp/chromedp v0.9.3/go.mod h1:NipeUkUcuzIdFbBP8eNNvl9upcceOfWzoJn6cRe4ksA=
+github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
+github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
+github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
+github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
+github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.3.0 h1:sbeU3Y4Qzlb+MOzIe6mQGf7QR4Hkv6ZD0qhGkBFL2O0=
+github.com/gobwas/ws v1.3.0/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
+github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
+github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
+github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
+github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
+github.com/studio-b12/gowebdav v0.9.0 h1:1j1sc9gQnNxbXXM4M/CebPOX4aXYtr7MojAVcN4dHjU=
+github.com/studio-b12/gowebdav v0.9.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
+github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk=
+go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

program/spider/action.go

@@ -0,0 +1,356 @@
+/**
+ * 浏览器行为封装
+ * 基础动作
+ */
+package main
+
+import (
+	"context"
+	"errors"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/chromedp/chromedp"
+	"github.com/yuin/gopher-lua"
+)
+
+const (
+	selector_type_id        = 0
+	selector_type_query     = 1
+	selector_type_search    = 2
+	selector_type_jspath    = 3
+	selector_type_query_all = 4
+
+	execute_return_type_string = 0
+	execute_return_type_list   = 1
+	execute_return_type_table  = 2
+)
+
+// findTab 根据标题、url找tab
+func (b *Browser) findTabContext(tabTitle, tabUrl string, timeoutInt64 int64) (ctx context.Context, err error) {
+	if timeoutInt64 == 0 {
+		timeoutInt64 = 5000
+	}
+	timeout := time.Duration(timeoutInt64) * time.Millisecond
+	if tabTitle == "*" && tabUrl == "*" {
+		return b.Ctx, nil
+	} else if tabTitle == "" && tabUrl == "" {
+		ctx, _ = context.WithTimeout(b.Ctx, timeout)
+		return ctx, nil
+	} else {
+		ts, err := chromedp.Targets(b.Ctx)
+		if err != nil {
+			return nil, err
+		}
+		for _, t := range ts {
+			if (tabTitle != "" && strings.Contains(t.Title, tabTitle)) ||
+				(tabUrl != "" && strings.Contains(t.URL, tabUrl)) {
+				// log.Printf("find tab param<title,url>: %s %s found %s %s", tabTitle, tabUrl,
+				// 	t.Title, t.URL)
+				newCtx, _ := chromedp.NewContext(b.Ctx, chromedp.WithTargetID(t.TargetID))
+				ctx, _ = context.WithTimeout(newCtx, timeout)
+				return ctx, nil
+			}
+		}
+	}
+	return nil, errors.New("can't find tab")
+}
+
+// BindLuaState
+func (b *Browser) BindLuaState(state *lua.LState) {
+	//执行暂停
+	state.SetGlobal("browser_sleep", state.NewFunction(func(l *lua.LState) int {
+		timeout := l.ToInt64(-1)
+		if timeout == 0 {
+			timeout = 5
+		}
+		time.Sleep(time.Duration(timeout) * time.Millisecond)
+		return 0
+	}))
+	//关闭tabl页
+	state.SetGlobal("browser_closetabs", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-3)
+		tabUrl := l.ToString(-2)
+		timeout := l.ToInt64(-1)
+		if timeout == 0 {
+			timeout = 5
+		}
+		b.CloseTabs(tabTitle, tabUrl, timeout)
+		return 0
+	}))
+	//注册打开地址
+	state.SetGlobal("browser_navagite", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-5)
+		tabUrl := l.ToString(-4)
+		isNewTab := l.ToBool(-3)
+		timeout := l.ToInt64(-2)
+		targetUrl := l.ToString(-1)
+		if err := b.Navigate(tabTitle, tabUrl, isNewTab, targetUrl, timeout); err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+	//执行浏览器端js
+	state.SetGlobal("browser_executejs", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-5)
+		tabUrl := l.ToString(-4)
+		timeout := l.ToInt64(-3)
+		returnType := l.ToInt(-2)
+		script := l.ToString(-1)
+
+		switch returnType {
+		case execute_return_type_string: //返回string
+			var ret string
+			if err := b.ExecuteJS(tabTitle, tabUrl, script, &ret, timeout); err == nil {
+				l.Push(lua.LString("ok"))
+				l.Push(lua.LString(ret))
+			} else {
+				l.Push(lua.LString("err"))
+				l.Push(lua.LString(err.Error()))
+			}
+		case execute_return_type_list: //返回list
+			var ret = make([]interface{}, 0, 0)
+			var tmp = make(map[string]interface{})
+			if err := b.ExecuteJS(tabTitle, tabUrl, script, &ret, timeout); err == nil {
+				for i, v := range ret {
+					tmp[strconv.Itoa(i)] = v
+				}
+				l.Push(lua.LString("ok"))
+				l.Push(MapToTable(tmp))
+			} else {
+				l.Push(lua.LString("err"))
+				l.Push(lua.LString(err.Error()))
+			}
+		case execute_return_type_table: //返回table
+			var ret = make(map[string]interface{})
+			if err := b.ExecuteJS(tabTitle, tabUrl, script, &ret, timeout); err == nil {
+				l.Push(lua.LString("ok"))
+				l.Push(MapToTable(ret))
+			} else {
+				l.Push(lua.LString("err"))
+				l.Push(lua.LString(err.Error()))
+			}
+		}
+		return 2
+	}))
+	//按键
+	state.SetGlobal("browser_keysend", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-6)
+		tabUrl := l.ToString(-5)
+		timeout := l.ToInt64(-4)
+
+		selectorType := l.ToInt(-3)
+		selector := l.ToString(-2)
+		words := l.ToString(-1)
+
+		err := b.KeySend(tabTitle, tabUrl, selector, words, selectorType, timeout)
+		if err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+	//点击
+	state.SetGlobal("browser_click", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-5)
+		tabUrl := l.ToString(-4)
+		timeout := l.ToInt64(-3)
+		selectorType := l.ToInt(-2)
+		selector := l.ToString(-1)
+
+		err := b.Click(tabTitle, tabUrl, selector, selectorType, timeout)
+		if err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+	//browser_history_back
+	state.SetGlobal("browser_history_back", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-3)
+		tabUrl := l.ToString(-2)
+		timeout := l.ToInt64(-1)
+
+		err := b.GoHistoryBack(tabTitle, tabUrl, timeout)
+		if err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+	state.SetGlobal("browser_waitvisible", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-5)
+		tabUrl := l.ToString(-4)
+		timeout := l.ToInt64(-3)
+		selectorType := l.ToInt(-2)
+		selector := l.ToString(-1)
+
+		err := b.WaitVisible(tabTitle, tabUrl, selector, selectorType, timeout)
+		if err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+	//点击
+	state.SetGlobal("browser_downloadfile", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-7)
+		tabUrl := l.ToString(-6)
+		timeout := l.ToInt64(-5)
+		selectorType := l.ToInt(-4)
+		selector := l.ToString(-3)
+		filename := l.ToString(-2)
+		save2dir := l.ToString(-1)
+
+		err := b.DownloadFile(tabTitle, tabUrl, timeout, selector, selectorType, filename, save2dir)
+		if err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+
+	//关闭tabl页
+	state.SetGlobal("browser_closetabs_without", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-3)
+		tabUrl := l.ToString(-2)
+		timeout := l.ToInt64(-1)
+		if timeout == 0 {
+			timeout = 5
+		}
+		b.CloseTabsWithout(tabTitle, tabUrl, timeout)
+		return 0
+	}))
+	//browser_screenshot 网页局部截图
+	state.SetGlobal("browser_screenshot", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-6)
+		tabUrl := l.ToString(-5)
+		timeout := l.ToInt64(-4)
+		selectorType := l.ToInt(-3)
+		selector := l.ToString(-2)
+		filename := l.ToString(-1)
+		if timeout == 0 {
+			timeout = 5
+		}
+
+		if err := b.Screenshot(tabTitle, tabUrl, timeout, selectorType, selector, filename); err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+	//browser_print2pdf 整个网页生成pdf
+	state.SetGlobal("browser_print2pdf", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-4)
+		tabUrl := l.ToString(-3)
+		timeout := l.ToInt64(-2)
+		filename := l.ToString(-1)
+		if timeout == 0 {
+			timeout = 5
+		}
+
+		if err := b.PrintToPDF(tabTitle, tabUrl, timeout, filename); err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+	state.SetGlobal("browser_tabs", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-3)
+		tabUrl := l.ToString(-2)
+		timeout := l.ToInt64(-1)
+		if timeout == 0 {
+			timeout = 500
+		}
+		var tmp = make(map[string]interface{})
+		tabs, err := b.GetBrowserTabs(tabTitle, tabUrl, timeout)
+		//fmt.Println(err, tabs)
+		if err == nil {
+			for i, v := range tabs {
+				tmp[strconv.Itoa(i)] = v
+			}
+			l.Push(lua.LString("ok"))
+			l.Push(MapToTable(tmp))
+		} else {
+			l.Push(lua.LString("err"))
+			l.Push(lua.LString(err.Error()))
+		}
+		return 2
+	}))
+
+	state.SetGlobal("browser_send_img_chatbot", state.NewFunction(func(l *lua.LState) int {
+		mentioned := l.ToString(-3)
+		uri := l.ToString(-2)
+		img := l.ToString(-1)
+		err := SendImage2ChatBot(uri, img, mentioned)
+		if err != nil {
+			l.Push(lua.LString("err"))
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+			l.Push(lua.LString("ok"))
+		}
+		return 2
+	}))
+	state.SetGlobal("browser_send_text_chatbot", state.NewFunction(func(l *lua.LState) int {
+		mentioned := l.ToString(-3)
+		uri := l.ToString(-2)
+		text := l.ToString(-1)
+		err := SendImage2ChatBot(uri, text, mentioned)
+		if err != nil {
+			l.Push(lua.LString("err"))
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+			l.Push(lua.LString("ok"))
+		}
+		return 2
+	}))
+
+	state.SetGlobal("browser_reload", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-3)
+		tabUrl := l.ToString(-2)
+		timeout := l.ToInt64(-1)
+		err := b.Reload(tabTitle, tabUrl, timeout)
+		if err != nil {
+			l.Push(lua.LString("err"))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+	state.SetGlobal("browser_back", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-3)
+		tabUrl := l.ToString(-2)
+		timeout := l.ToInt64(-1)
+		err := b.Back(tabTitle, tabUrl, timeout)
+		if err != nil {
+			l.Push(lua.LString("err"))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+	state.SetGlobal("browser_forward", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-3)
+		tabUrl := l.ToString(-2)
+		timeout := l.ToInt64(-1)
+		err := b.Forward(tabTitle, tabUrl, timeout)
+		if err != nil {
+			l.Push(lua.LString("err"))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+}

program/spider/baseaction.go

@@ -0,0 +1,240 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/chromedp/cdproto/input"
+
+	"github.com/chromedp/cdproto/page"
+	"github.com/chromedp/chromedp"
+)
+
+// CloseTabs关闭页面
+func (b *Browser) CloseTabs(tabTitle, tabUrl string, timeoutInt64 int64) (err error) {
+	if timeoutInt64 == 0 {
+		timeoutInt64 = 5
+	}
+	timeout := time.Duration(timeoutInt64) * time.Millisecond
+
+	ts, err := chromedp.Targets(b.Ctx)
+	if err != nil {
+		return err
+	}
+	for _, t := range ts {
+		if (tabTitle != "" && strings.Contains(t.Title, tabTitle)) ||
+			(tabUrl != "" && strings.Contains(t.URL, tabUrl)) {
+			newCtx, _ := chromedp.NewContext(b.Ctx, chromedp.WithTargetID(t.TargetID))
+			ctx, _ := context.WithTimeout(newCtx, timeout)
+			chromedp.Run(
+				ctx,
+				page.Close(),
+			)
+		}
+	}
+	return nil
+}
+
+// CloseTabs关闭页面
+func (b *Browser) CloseTabsWithout(tabTitle, tabUrl string, timeoutInt64 int64) (err error) {
+	if timeoutInt64 == 0 {
+		timeoutInt64 = 5
+	}
+	timeout := time.Duration(timeoutInt64) * time.Millisecond
+
+	ts, err := chromedp.Targets(b.Ctx)
+	if err != nil {
+		return err
+	}
+	for _, t := range ts {
+		if (tabTitle != "" && !strings.Contains(t.Title, tabTitle)) ||
+			(tabUrl != "" && !strings.Contains(t.URL, tabUrl)) {
+			newCtx, _ := chromedp.NewContext(b.Ctx, chromedp.WithTargetID(t.TargetID))
+			ctx, _ := context.WithTimeout(newCtx, timeout)
+			chromedp.Run(
+				ctx,
+				page.Close(),
+			)
+		}
+	}
+	return nil
+}
+
+// Navigate 导航到指定网址
+func (b *Browser) Navigate(tabTitle string,
+	tabUrl string, isNewTab bool,
+	targetUrl string, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	//新标签页
+	if isNewTab {
+		ctx, _ = chromedp.NewContext(ctx)
+	}
+	//
+	return chromedp.Run(ctx,
+		chromedp.Navigate(targetUrl))
+}
+
+// Reload 重新加载tab页地址
+func (b *Browser) Reload(tabTitle string,
+	tabUrl string, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	//
+	return chromedp.Run(ctx,
+		chromedp.Reload())
+}
+
+// Back 回退
+func (b *Browser) Back(tabTitle string,
+	tabUrl string, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	//
+	return chromedp.Run(ctx,
+		chromedp.NavigateBack())
+}
+
+// Forward 向前
+func (b *Browser) Forward(tabTitle string,
+	tabUrl string, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	//
+	return chromedp.Run(ctx,
+		chromedp.NavigateForward())
+}
+
+// ExecuteJS 执行脚本
+func (b *Browser) ExecuteJS(tabTitle, tabUrl, script string, ret interface{}, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	return chromedp.Run(ctx,
+		chromedp.Evaluate(script, ret))
+}
+
+// Click 点击
+func (b *Browser) Click(tabTitle, tabUrl, selector string, selectorType int, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	var act chromedp.QueryAction
+	switch selectorType {
+	case selector_type_id:
+		act = chromedp.Click(selector, chromedp.ByID)
+	case selector_type_query:
+		act = chromedp.Click(selector, chromedp.ByQuery)
+	case selector_type_search:
+		act = chromedp.Click(selector, chromedp.BySearch)
+	case selector_type_jspath:
+		act = chromedp.Click(selector, chromedp.ByJSPath)
+	default:
+		act = chromedp.Click(selector, chromedp.ByQueryAll)
+	}
+	return chromedp.Run(ctx,
+		act)
+}
+
+// keysend 键盘输入
+func (b *Browser) KeySend(tabTitle, tabUrl, selector, sendStr string,
+	selectorType int, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	var act chromedp.QueryAction
+	switch selectorType {
+	case selector_type_id:
+		act = chromedp.SendKeys(selector, sendStr, chromedp.ByID)
+	case selector_type_query:
+		act = chromedp.SendKeys(selector, sendStr, chromedp.ByQuery)
+	case selector_type_search:
+		act = chromedp.SendKeys(selector, sendStr, chromedp.BySearch)
+	case selector_type_jspath:
+		act = chromedp.SendKeys(selector, sendStr, chromedp.ByJSPath)
+	default:
+		act = chromedp.SendKeys(selector, sendStr, chromedp.ByQueryAll)
+	}
+	return chromedp.Run(ctx,
+		act)
+}
+
+// WaitVisible 等待元素可见
+func (b *Browser) WaitVisible(tabTitle, tabUrl, selector string,
+	selectorType int, timeout int64) error {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	var act chromedp.QueryAction
+	switch selectorType {
+	case selector_type_id:
+		act = chromedp.WaitVisible(selector, chromedp.ByID)
+	case selector_type_query:
+		act = chromedp.WaitVisible(selector, chromedp.ByQuery)
+	case selector_type_search:
+		act = chromedp.WaitVisible(selector, chromedp.BySearch)
+	case selector_type_jspath:
+		act = chromedp.WaitVisible(selector, chromedp.ByJSPath)
+	default:
+		act = chromedp.WaitVisible(selector, chromedp.ByQueryAll)
+	}
+	return chromedp.Run(ctx,
+		act)
+}
+
+// 鼠标点击 双击
+func (b *Browser) DoubleClick(tabTitle, tabUrl, selector string, selectorType int, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	var act chromedp.QueryAction
+	switch selectorType {
+	case selector_type_id:
+		act = chromedp.DoubleClick(selector, chromedp.ByID)
+	case selector_type_query:
+		act = chromedp.DoubleClick(selector, chromedp.ByQuery)
+	case selector_type_search:
+		act = chromedp.DoubleClick(selector, chromedp.BySearch)
+	case selector_type_jspath:
+		act = chromedp.DoubleClick(selector, chromedp.ByJSPath)
+	default:
+		act = chromedp.DoubleClick(selector, chromedp.ByQueryAll)
+	}
+	return chromedp.Run(ctx,
+		act)
+}
+
+// 鼠标拖拽
+func (b *Browser) MouseDrag(tabTitle, tabUrl string, points [][2]float64, delay []int64, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	//鼠标拖拽
+	actionsLen := len(points)*2 + 2
+	pointsLen := len(points)
+	var acts = make([]chromedp.Action, actionsLen, actionsLen)
+	//第一个动作
+	acts[0] = chromedp.MouseEvent(input.MousePressed, points[0][0], points[0][1], chromedp.ButtonLeft)
+	for i := 1; i <= len(points); i++ {
+		acts[i*2] = chromedp.MouseEvent(input.MouseMoved, points[i-1][0], points[i-1][1], chromedp.ButtonLeft)
+		acts[i*2+1] = chromedp.Sleep(time.Duration(delay[i]) * time.Millisecond)
+	}
+	acts[actionsLen-1] = chromedp.MouseEvent(input.MouseReleased, points[pointsLen-1][0], points[pointsLen-1][1], chromedp.ButtonLeft)
+	return chromedp.Run(ctx,
+		acts...)
+}

program/spider/cli/batchrename/main.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+var (
+	dir            = flag.String("d", "./attaches", "目录")
+	mode           = flag.String("m", "scan", "模式:scan扫描目录 rename更名")
+	sourceDatafile = flag.String("sdf", "./source.txt", "原始数据文件（不能换顺序）")
+	targetDatafile = flag.String("tdf", "./target.txt", "目标数据文件（不能换顺序）")
+)
+
+// init
+func init() {
+	flag.Parse()
+}
+
+// scan
+func scan() {
+	fo1, err := os.Create(*sourceDatafile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fo2, err := os.Create(*targetDatafile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fo1.Close()
+	defer fo2.Close()
+
+	fs, err := ioutil.ReadDir(*dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, f := range fs {
+		if strings.HasPrefix(f.Name(), ".") {
+			continue
+		}
+		fo1.WriteString(f.Name() + "\n")
+		fo2.WriteString(f.Name() + "\n")
+	}
+}
+
+// rename
+func rename() {
+	fi1, err := os.Open(*sourceDatafile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fi2, err := os.Open(*targetDatafile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fi1.Close()
+	defer fi2.Close()
+	source, target := make([]string, 0, 0), make([]string, 0, 0)
+	br1 := bufio.NewReader(fi1)
+	for {
+		bytes, _, err := br1.ReadLine()
+		if err != nil {
+			break
+		}
+		source = append(source, string(bytes))
+	}
+	br2 := bufio.NewReader(fi2)
+	for {
+		bytes, _, err := br2.ReadLine()
+		if err != nil {
+			break
+		}
+		target = append(target, string(bytes))
+	}
+	if len(source) != len(target) {
+		log.Println("数据文件，源 目标 长度不一致")
+		return
+	}
+	for index, s := range source {
+		oldName, newName := s, target[index]
+		log.Println(oldName, ">>", newName)
+		os.Rename(filepath.Join(*dir, oldName), filepath.Join(*dir, newName))
+	}
+}
+
+// main
+func main() {
+	switch *mode {
+	case "scan":
+		scan()
+	case "rename":
+		rename()
+	}
+}

program/spider/cli/counter/main.go

@@ -0,0 +1,122 @@
+// 数据统计
+package main
+
+import (
+	"encoding/csv"
+	"flag"
+	"os"
+	"sort"
+	"strconv"
+
+	"database/sql"
+	"log"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+type (
+	Folder struct {
+		Id               int
+		Name             string
+		ParentName       string
+		ParentParentName string
+		ParentId         int
+		Count            int
+	}
+	Cities []*Folder
+)
+
+var (
+	dbfile  = flag.String("db", "/Users/taozhang/Downloads/cloudreve.db", "数据文件")
+	outfile = flag.String("of", "./结果.csv", "结果输出文件")
+	db      *sql.DB
+)
+
+// init
+func init() {
+	flag.Parse()
+	var err error
+	db, err = sql.Open("sqlite3", *dbfile)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func main() {
+	defer db.Close()
+	// 执行查询，加载目录
+	rows, err := db.Query("SELECT id, name,parent_id FROM folders")
+	if err != nil {
+		log.Fatal(err)
+	}
+	allCities := map[int]*Folder{}
+	// 先拿到所有目录
+	for rows.Next() {
+		var id, _pid int
+		var parentId sql.NullInt32
+		var name string
+		err := rows.Scan(&id, &name, &parentId)
+		if err != nil {
+			log.Fatal(err)
+		}
+		_pid = int(parentId.Int32)
+		allCities[id] = &Folder{
+			Id:       id,
+			Name:     name,
+			ParentId: _pid,
+		}
+	}
+	rows.Close()
+	//TODO 需要筛选的目录，只筛选政府类的市，但是要补上父级，爷爷级节点名称，便于统计
+	selectCities := map[int]*Folder{}
+	for _, v := range allCities {
+		if v.Name == "政府" {
+			if pv, ok := allCities[v.ParentId]; ok {
+				//给pv也补上父级关系
+				if ppv, ok := allCities[pv.ParentId]; ok {
+					pv.ParentName = ppv.Name
+					//补上爷爷级节点
+					if pppv, ok := allCities[ppv.ParentId]; ok {
+						pv.ParentParentName = pppv.Name
+					}
+				}
+				selectCities[v.Id] = pv
+			}
+		}
+	}
+	//TODO 计算所有目录的文件数量
+	rows, err = db.Query("select folder_id,count(id) as fs from files group by folder_id;")
+	if err != nil {
+		log.Fatal(err)
+	}
+	selectCitiesArray := make(Cities, 0, 0)
+	for rows.Next() {
+		var id int
+		var filesCount int
+		err := rows.Scan(&id, &filesCount)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if v, ok := selectCities[id]; ok {
+			v.Count = filesCount
+			selectCitiesArray = append(selectCitiesArray, v)
+		}
+	}
+	rows.Close()
+	//TODO 正向排序
+	sort.Slice(selectCitiesArray, func(i, j int) bool { return selectCitiesArray[i].Count < selectCitiesArray[j].Count })
+	//TODO 结果输出到CSV文件
+	fo, err := os.Create(*outfile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	writer := csv.NewWriter(fo)
+	writer.Write([]string{"年", "省", "城市", "组合筛选KEY", "文件数"})
+	for _, v := range selectCitiesArray {
+		writer.Write([]string{v.ParentParentName, v.ParentName, v.Name,
+			v.ParentParentName + v.ParentName + v.Name, strconv.Itoa(v.Count)})
+		log.Println(v.Id, v.ParentParentName, v.ParentName, v.Name, v.Count)
+	}
+	writer.Flush()
+	fo.Close()
+}

program/spider/cli/counter/结果.csv

@@ -0,0 +1,1168 @@
+年,省,城市,组合筛选KEY,文件数
+2023年,湖南,湖南,2023年湖南湖南,1
+2022年,重庆,江北区,2022年重庆江北区,1
+2024年,陕西,陕西,2024年陕西陕西,1
+2024年,重庆,开州区,2024年重庆开州区,1
+2024年,西藏,山南市,2024年西藏山南市,1
+2023年,西藏,山南市,2023年西藏山南市,1
+2024年,青海,玉树藏族自治州,2024年青海玉树藏族自治州,1
+2024年,青海,果洛藏族自治州,2024年青海果洛藏族自治州,1
+2024年," 山西",忻州市,2024年 山西忻州市,1
+2022年,北京,顺义区,2022年北京顺义区,1
+2024年,安徽,安庆市,2024年安徽安庆市,1
+2022年,吉林,通化市,2022年吉林通化市,1
+2023年,吉林,长春市,2023年吉林长春市,1
+2023年,山东,山东,2023年山东山东,1
+2023年,湖北,湖北,2023年湖北湖北,1
+2022年,江苏,扬州市,2022年江苏扬州市,2
+2022年,吉林,白城市,2022年吉林白城市,2
+2023年,广西,广西壮族自治区,2023年广西广西壮族自治区,2
+2023年,陕西,汉中市,2023年陕西汉中市,2
+2022年,江苏,镇江市,2022年江苏镇江市,2
+2022年,江苏,泰州市,2022年江苏泰州市,2
+2023年,山西,朔州市,2023年山西朔州市,2
+2022年,江苏,宿迁市,2022年江苏宿迁市,2
+2023,湖南,张家界市,2023湖南张家界市,2
+2022年,江苏,淮安市,2022年江苏淮安市,2
+2022年,海南,文昌市,2022年海南文昌市,2
+2022,湖南,张家界市,2022湖南张家界市,2
+2022年,江苏,盐城市,2022年江苏盐城市,2
+2024年,重庆,江北区,2024年重庆江北区,2
+2024年,广西,桂林市,2024年广西桂林市,2
+2022年,江苏,连云港市,2022年江苏连云港市,2
+2024年,广西,钦州市,2024年广西钦州市,2
+2024年,陕西,渭南市,2024年陕西渭南市,2
+2024年,福建,福州市,2024年福建福州市,2
+2023年,福建,福州市,2023年福建福州市,2
+2024年,天津,津南区,2024年天津津南区,2
+2024,湖南省,张家界市,2024湖南省张家界市,2
+2022,湖南,岳阳市,2022湖南岳阳市,3
+2024年,天津,宁河区,2024年天津宁河区,3
+2022年,云南,德宏傣族景颇族自治州,2022年云南德宏傣族景颇族自治州,3
+2024年,广东,汕头市,2024年广东汕头市,3
+2022年,山西,临汾市,2022年山西临汾市,3
+2024年,河南,周口市,2024年河南周口市,3
+2022年,安徽,安庆市,2022年安徽安庆市,3
+2024年,广西,南宁市,2024年广西南宁市,3
+2024年,广东,珠海市,2024年广东珠海市,3
+2022年,安徽,合肥市,2022年安徽合肥市,3
+2022年,陕西,汉中市,2022年陕西汉中市,3
+2024年,吉林,白山市,2024年吉林白山市,3
+2024年,新疆,吐鲁番市,2024年新疆吐鲁番市,3
+2022年,福建,福州市,2022年福建福州市,4
+2022年,吉林,吉林市,2022年吉林吉林市,4
+2024年,陕西,宝鸡市,2024年陕西宝鸡市,4
+2022年,江苏,江苏,2022年江苏江苏,4
+2023年,吉林,白城市,2023年吉林白城市,4
+2022年,广东,深圳市,2022年广东深圳市,4
+2022年,海南,定安县,2022年海南定安县,4
+2024年,海南,定安县,2024年海南定安县,4
+2022,湖南,常德市,2022湖南常德市,4
+2024年," 山西",长治市,2024年 山西长治市,4
+2023年,新疆,新疆,2023年新疆新疆,4
+2022年,安徽,合肥,2022年安徽合肥,4
+2024年,河南,濮阳市,2024年河南濮阳市,5
+2023年,广东,广东,2023年广东广东,6
+2022年,西藏,山南市,2022年西藏山南市,6
+2024年,新疆,克拉玛依市,2024年新疆克拉玛依市,6
+2024年,宁夏,宁夏,2024年宁夏宁夏,6
+2022年,青海,果洛藏族自治州,2022年青海果洛藏族自治州,6
+2024年,天津,红桥区,2024年天津红桥区,7
+2023年,宁夏,宁夏,2023年宁夏宁夏,7
+2023年,吉林,延边朝鲜族自治州,2023年吉林延边朝鲜族自治州,7
+2022年,重庆,涪陵区,2022年重庆涪陵区,7
+2023年,贵州,贵州省,2023年贵州贵州省,7
+2023年,西藏,阿里地区,2023年西藏阿里地区,7
+2024年,浙江,衢州市,2024年浙江衢州市,7
+2024年,河南,南阳市,2024年河南南阳市,8
+2024年,河南,焦作市,2024年河南焦作市,8
+2024年,吉林,长春,2024年吉林长春,9
+2022年,吉林,长春,2022年吉林长春,9
+2024年,海南,海南,2024年海南海南,10
+2024年,广西,玉林市,2024年广西玉林市,10
+2024年," 山西",山西,2024年 山西山西,10
+2023年,海南,琼中黎族苗族自治县,2023年海南琼中黎族苗族自治县,10
+2022年,山西,山西,2022年山西山西,10
+2023年,山西,山西省,2023年山西山西省,11
+2022年,吉林,松原市,2022年吉林松原市,11
+2024年,江苏,南京,2024年江苏南京,11
+2022年,海南,琼中黎族苗族自治县,2022年海南琼中黎族苗族自治县,12
+2022年,江苏,常州市,2022年江苏常州市,12
+2024年,海南,陵水黎族自治县,2024年海南陵水黎族自治县,12
+2022年,浙江,嘉兴市bm,2022年浙江嘉兴市bm,13
+2022年,广东,广东,2022年广东广东,13
+2024年,广东,广东省,2024年广东广东省,14
+2023年,广东,深圳市,2023年广东深圳市,14
+2024年,安徽,马鞍山市,2024年安徽马鞍山市,14
+2023年,浙江,衢州市,2023年浙江衢州市,14
+2022年,浙江,衢州市,2022年浙江衢州市,14
+2022年,广东,广东省,2022年广东广东省,15
+2022年,宁夏,宁夏,2022年宁夏宁夏,15
+2024年,河南,商丘市,2024年河南商丘市,15
+2024年,宁夏,银川市,2024年宁夏银川市,15
+2023年,浙江,丽水市,2023年浙江丽水市,16
+2024年," 山西",太原市,2024年 山西太原市,17
+2024年,辽宁,本溪市,2024年辽宁本溪市,17
+2022年,安徽,马鞍山市,2022年安徽马鞍山市,17
+2022年,吉林,吉林,2022年吉林吉林,17
+2023年,重庆,江北区,2023年重庆江北区,17
+2023年,广东,广东省,2023年广东广东省,17
+2023年,吉林,吉林,2023年吉林吉林,18
+2023年,甘肃,酒泉市,2023年甘肃酒泉市,18
+2023年,重庆,武隆区,2023年重庆武隆区,18
+2022年,山西,太原市,2022年山西太原市,18
+2023年,安徽,马鞍山市,2023年安徽马鞍山市,18
+2022年,宁夏,银川市,2022年宁夏银川市,18
+2023,湖南,湖南省,2023湖南湖南省,19
+2024年,吉林,吉林,2024年吉林吉林,19
+2024年,云南,曲靖市,2024年云南曲靖市,19
+2022年,浙江,丽水市,2022年浙江丽水市,19
+2024年,上海,金山区,2024年上海金山区,20
+2022年,西藏,拉萨市,2022年西藏拉萨市,20
+2022年,辽宁,葫芦岛市,2022年辽宁葫芦岛市,21
+2023年,西藏,日喀则市,2023年西藏日喀则市,21
+2023年,宁夏,银川市,2023年宁夏银川市,21
+2024年,海南,屯昌县,2024年海南屯昌县,22
+2022年,云南,云南,2022年云南云南,22
+2023年,吉林,松原市,2023年吉林松原市,22
+2024年,浙江,丽水市,2024年浙江丽水市,22
+2023年,浙江,嘉兴市,2023年浙江嘉兴市,22
+2024年,吉林,松原市,2024年吉林松原市,23
+2024,湖南省,常德市,2024湖南省常德市,23
+2022年,浙江,嘉兴市,2022年浙江嘉兴市,24
+2024年,吉林,吉林市,2024年吉林吉林市,25
+2022年,海南,琼海市,2022年海南琼海市,25
+2022年,甘肃,酒泉市,2022年甘肃酒泉市,26
+2022年,湖北,黄石市,2022年湖北黄石市,27
+2022年,浙江,舟山市,2022年浙江舟山市,28
+2022年,福建,泉州市,2022年福建泉州市,29
+2022年,海南,白沙黎族自治县,2022年海南白沙黎族自治县,29
+2022年,吉林,延边朝鲜族自治州,2022年吉林延边朝鲜族自治州,30
+2022年,西藏,日喀则市,2022年西藏日喀则市,30
+2023年,海南,屯昌县,2023年海南屯昌县,31
+2022年,重庆,大渡口区,2022年重庆大渡口区,31
+2023年,江西,萍乡市,2023年江西萍乡市,31
+2022年,重庆,武隆区,2022年重庆武隆区,32
+2022年,海南,东方市,2022年海南东方市,32
+2023年,安徽,合肥市,2023年安徽合肥市,33
+2022年,海南,屯昌县,2022年海南屯昌县,33
+2023年,山西,阳泉市,2023年山西阳泉市,33
+2023年,河南,商丘市,2023年河南商丘市,34
+2022年,宁夏,吴忠市,2022年宁夏吴忠市,34
+2024年,四川,巴中市,2024年四川巴中市,35
+2023年,上海,金山区,2023年上海金山区,35
+2023年,山西,太原市,2023年山西太原市,35
+2022年,江苏,南京市,2022年江苏南京市,35
+2024年,安徽,合肥市,2024年安徽合肥市,36
+2022年,河南,新乡市,2022年河南新乡市,36
+2023年,海南,三亚市,2023年海南三亚市,36
+2024年,江西,赣州市,2024年江西赣州市,37
+2023年,吉林,吉林市,2023年吉林吉林市,37
+2024年,广西,崇左市,2024年广西崇左市,37
+2023年,甘肃,甘肃,2023年甘肃甘肃,38
+2024年,广东,云浮市,2024年广东云浮市,39
+2024年,海南,海口市,2024年海南海口市,40
+2023年,广东,广州市,2023年广东广州市,41
+2022年,江苏,南通市,2022年江苏南通市,42
+2022年,海南,万宁市,2022年海南万宁市,42
+2024年,湖北,黄冈市,2024年湖北黄冈市,43
+2024年,云南,昆明市,2024年云南昆明市,44
+2024年,西藏,昌都市,2024年西藏昌都市,44
+2023年,西藏,拉萨市,2023年西藏拉萨市,44
+2022年,宁夏,固原市,2022年宁夏固原市,46
+2024年,四川,眉山市,2024年四川眉山市,49
+2022年,广东,广州市,2022年广东广州市,49
+2023年,湖北,黄石市,2023年湖北黄石市,50
+2022年,辽宁,朝阳市,2022年辽宁朝阳市,51
+2024年,湖北,黄石市,2024年湖北黄石市,51
+2022年,安徽,蚌埠市,2022年安徽蚌埠市,51
+2023年,江西,江西省,2023年江西江西省,51
+2024年,四川,广安市,2024年四川广安市,52
+2022年,上海,金山区,2022年上海金山区,52
+2024年,四川,自贡市,2024年四川自贡市,53
+2024年,福建,宁德市,2024年福建宁德市,53
+2022年,新疆,克拉玛依市,2022年新疆克拉玛依市,54
+2023年,河南,新乡市,2023年河南新乡市,55
+2022年,广东,汕头市,2022年广东汕头市,56
+2024年,上海,崇明区,2024年上海崇明区,57
+2024年,广西,防城港市,2024年广西防城港市,58
+2023年,辽宁,大连市,2023年辽宁大连市,59
+2022年,广东,东莞市,2022年广东东莞市,60
+2022年,甘肃,天水市,2022年甘肃天水市,60
+2022年,河南,漯河市,2022年河南漯河市,60
+2022年,重庆,巴南区,2022年重庆巴南区,61
+2022年,云南,昆明市,2022年云南昆明市,61
+2022年,云南,迪庆藏族自治州,2022年云南迪庆藏族自治州,61
+2023年,云南,昆明市,2023年云南昆明市,61
+2022年,海南,海南,2022年海南海南,62
+2024年,云南,楚雄彝族自治州,2024年云南楚雄彝族自治州,62
+2022年,西藏,阿里地区,2022年西藏阿里地区,64
+2022年,安徽,芜湖市,2022年安徽芜湖市,64
+2022年,重庆,合川区,2022年重庆合川区,66
+2023年,辽宁,朝阳市,2023年辽宁朝阳市,67
+2024年,云南,丽江市,2024年云南丽江市,68
+2022年,云南,文山壮族苗族自治州,2022年云南文山壮族苗族自治州,69
+2023年,云南,德宏傣族景颇族自治州,2023年云南德宏傣族景颇族自治州,69
+2024年,云南,德宏傣族景颇族自治州,2024年云南德宏傣族景颇族自治州,69
+2023年,重庆,合川区,2023年重庆合川区,70
+2022年,重庆,潼南区,2022年重庆潼南区,71
+2023年,云南,云南,2023年云南云南,72
+2022,湖南,湘潭市,2022湖南湘潭市,72
+2023年,广东,茂名市,2023年广东茂名市,72
+2023年,广西,防城港市,2023年广西防城港市,73
+2023年,安徽,芜湖市,2023年安徽芜湖市,73
+2022年,江西,九江市,2022年江西九江市,73
+2023年,新疆,克拉玛依市,2023年新疆克拉玛依市,73
+2024年,新疆,昌吉回族自治州,2024年新疆昌吉回族自治州,74
+2023年,湖北,荆门市,2023年湖北荆门市,74
+2022年,上海,静安区,2022年上海静安区,75
+2022年,广东,梅州市,2022年广东梅州市,75
+2022年,广东,茂名市,2022年广东茂名市,75
+2023年,云南,文山壮族苗族自治州,2023年云南文山壮族苗族自治州,75
+2023年,重庆,潼南区,2023年重庆潼南区,76
+2022年,湖北,荆门市,2022年湖北荆门市,76
+2023年,江西,赣州市,2023年江西赣州市,77
+2023年,重庆,涪陵区,2023年重庆涪陵区,77
+2023年,河南,漯河市,2023年河南漯河市,78
+2024年,江西,抚州市,2024年江西抚州市,78
+2024年,云南,临沧市,2024年云南临沧市,79
+2022年,山西,阳泉市,2022年山西阳泉市,79
+2023年,辽宁,葫芦岛市,2023年辽宁葫芦岛市,79
+2022年,河南,商丘市,2022年河南商丘市,79
+2024年,甘肃,临夏回族自治州,2024年甘肃临夏回族自治州,79
+2022年,四川,吉安市,2022年四川吉安市,79
+2024年,青海,海东市,2024年青海海东市,79
+2022年,重庆,九龙坡区,2022年重庆九龙坡区,81
+2023年,重庆,巴南区,2023年重庆巴南区,81
+2024年,河南,河南,2024年河南河南,82
+2022年,上海,虹口区,2022年上海虹口区,82
+2023年,海南,临高县,2023年海南临高县,82
+2023,湖南,永州市,2023湖南永州市,83
+2024年,上海,普陀区,2024年上海普陀区,84
+2023年,青海,海东市,2023年青海海东市,85
+2023年,山西,忻州市,2023年山西忻州市,85
+2022年,重庆,荣昌区,2022年重庆荣昌区,85
+2024年,四川,吉安市,2024年四川吉安市,85
+2023,湖南,湘潭市,2023湖南湘潭市,86
+2024年,黑龙江,双鸭山市,2024年黑龙江双鸭山市,86
+2023年,天津,滨海新区,2023年天津滨海新区,86
+2024年,重庆,潼南区,2024年重庆潼南区,86
+2023年,四川,吉安市,2023年四川吉安市,86
+2023年,广西,钦州市,2023年广西钦州市,86
+2023年,安徽,淮南市,2023年安徽淮南市,86
+2023年,青海,果洛藏族自治州,2023年青海果洛藏族自治州,86
+2024年,云南,怒江傈僳族自治州,2024年云南怒江傈僳族自治州,86
+2022年,广西,柳州市,2022年广西柳州市,86
+2022年,青海,海东市,2022年青海海东市,87
+2022年,陕西,商洛市,2022年陕西商洛市,87
+2023年,上海,宝山区,2023年上海宝山区,88
+2022年,河北,衡水市,2022年河北衡水市,88
+2022年,云南,曲靖市,2022年云南曲靖市,88
+2023年,宁夏,中卫市,2023年宁夏中卫市,89
+2022年,辽宁,大连市,2022年辽宁大连市,89
+2024,湖南省,湘潭市,2024湖南省湘潭市,89
+2022年,河北,石家庄市,2022年河北石家庄市,89
+2024年,云南,普洱市,2024年云南普洱市,90
+2024年,湖北,荆门市,2024年湖北荆门市,90
+2022年,黑龙江,双鸭山市,2022年黑龙江双鸭山市,90
+2024年,广东,惠州市,2024年广东惠州市,91
+2023年,广西,崇左市,2023年广西崇左市,91
+2024年,北京,东城区,2024年北京东城区,91
+2022年,广东,惠州市,2022年广东惠州市,91
+2022年,广西,防城港市,2022年广西防城港市,91
+2022年,湖北,仙桃市,2022年湖北仙桃市,92
+2023年,福建,三明市,2023年福建三明市,92
+2024年,河北,邢台市,2024年河北邢台市,92
+2023年,四川,内江市,2023年四川内江市,92
+2022年,辽宁,阜新市,2022年辽宁阜新市,92
+2022年,吉林,四平市,2022年吉林四平市,93
+2023年,广东,惠州市,2023年广东惠州市,93
+2024年,河南,平顶山市,2024年河南平顶山市,93
+2023年,北京,顺义区,2023年北京顺义区,94
+2022,湖南,永州市,2022湖南永州市,94
+2023年,江西,江西,2023年江西江西,94
+2024年,广东,茂名市,2024年广东茂名市,95
+2023年,云南,临沧市,2023年云南临沧市,95
+2024年,四川,广元市,2024年四川广元市,95
+2022年,河南,驻马店市,2022年河南驻马店市,95
+2024年,辽宁,葫芦岛市,2024年辽宁葫芦岛市,95
+2023年,黑龙江,双鸭山市,2023年黑龙江双鸭山市,96
+2023年,重庆,九龙坡区,2023年重庆九龙坡区,96
+2023年,陕西,铜川市,2023年陕西铜川市,96
+2023年,安徽,黄山市,2023年安徽黄山市,96
+2022年,黑龙江,伊春市,2022年黑龙江伊春市,96
+2022年,西藏,林芝市,2022年西藏林芝市,97
+2022年,新疆,昌吉回族自治州,2022年新疆昌吉回族自治州,97
+2022年,江西,景德镇市,2022年江西景德镇市,97
+2022年,辽宁,鞍山市,2022年辽宁鞍山市,98
+2024年,云南,文山壮族苗族自治州,2024年云南文山壮族苗族自治州,99
+2023年,新疆,昌吉回族自治州,2023年新疆昌吉回族自治州,99
+2023年,河南,平顶山市,2023年河南平顶山市,99
+2022年,湖北,鄂州市,2022年湖北鄂州市,99
+2023年,吉林,通化市,2023年吉林通化市,99
+2024年,重庆,合川区,2024年重庆合川区,100
+2022年,重庆,大足区,2022年重庆大足区,100
+2022年,浙江,温州市,2022年浙江温州市,100
+2022年,广西,广西,2022年广西广西,100
+2022年,福建,三明市,2022年福建三明市,102
+2023年,北京,石景山区,2023年北京石景山区,102
+2024年,海南,琼中黎族苗族自治县,2024年海南琼中黎族苗族自治县,102
+2023年,浙江,温州市,2023年浙江温州市,102
+2024年,安徽,芜湖市,2024年安徽芜湖市,103
+2024年,辽宁,大连市,2024年辽宁大连市,103
+2022年,广西,崇左市,2022年广西崇左市,103
+2024年,浙江,温州市,2024年浙江温州市,103
+2024年,黑龙江,黑龙江,2024年黑龙江黑龙江,104
+2023年,云南,普洱市,2023年云南普洱市,104
+2023年,新疆,和田地区,2023年新疆和田地区,104
+2022年,湖北,潜江市,2022年湖北潜江市,104
+2022年,广西,玉林市,2022年广西玉林市,104
+2024年,福建,泉州市,2024年福建泉州市,105
+2024年,河北,秦皇岛,2024年河北秦皇岛,105
+2024年,江苏,江苏,2024年江苏江苏,105
+2023年,江苏,江苏,2023年江苏江苏,106
+2022年,陕西,咸阳市,2022年陕西咸阳市,106
+2024年,广西,柳州市,2024年广西柳州市,106
+2023年,云南,怒江傈僳族自治州,2023年云南怒江傈僳族自治州,106
+2023年,吉林,四平市,2023年吉林四平市,107
+2023年,上海,普陀区,2023年上海普陀区,107
+2022年,云南,临沧市,2022年云南临沧市,107
+2023年,江西,九江市,2023年江西九江市,108
+2023年,广西,南宁市,2023年广西南宁市,108
+2023年,青海,青海,2023年青海青海,108
+2024年,河北,衡水市,2024年河北衡水市,108
+2022年,海南,保亭黎族苗族自治县,2022年海南保亭黎族苗族自治县,108
+2023年,青海,海北藏族自治州,2023年青海海北藏族自治州,108
+2024年,江西,上饶市,2024年江西上饶市,109
+2024年,辽宁,鞍山市,2024年辽宁鞍山市,109
+2023年,安徽,蚌埠市,2023年安徽蚌埠市,109
+2022年,安徽,淮南市,2022年安徽淮南市,109
+2024年,江西,南昌市,2024年江西南昌市,110
+2022年,四川,广元市,2022年四川广元市,110
+2023年,云南,曲靖市,2023年云南曲靖市,110
+2024年,河北,邯郸市,2024年河北邯郸市,110
+2024年,辽宁,铁岭市,2024年辽宁铁岭市,111
+2023年,海南,陵水黎族自治县,2023年海南陵水黎族自治县,111
+2023年,辽宁,鞍山市,2023年辽宁鞍山市,111
+2023年,江西,上饶市,2023年江西上饶市,111
+2024年,江西,景德镇市,2024年江西景德镇市,112
+2024年,青海,海南藏族自治州,2024年青海海南藏族自治州,112
+2022年,宁夏,中卫市,2022年宁夏中卫市,112
+2024年,河南,洛阳市,2024年河南洛阳市,113
+2024年,福建,三明市,2024年福建三明市,113
+2022年,青海,海南藏族自治州,2022年青海海南藏族自治州,113
+2024年,广东,梅州市,2024年广东梅州市,113
+2023年,河北,秦皇岛市,2023年河北秦皇岛市,114
+2024年,青海,海北藏族自治州,2024年青海海北藏族自治州,114
+2024年,四川,德阳市,2024年四川德阳市,114
+2024年,安徽,黄山市,2024年安徽黄山市,114
+2023年,广东,云浮市,2023年广东云浮市,114
+2022年,新疆,哈密市,2022年新疆哈密市,115
+2022年,青海,青海,2022年青海青海,115
+2023年,江苏,扬州市,2023年江苏扬州市,115
+2022年,辽宁,沈阳市,2022年辽宁沈阳市,115
+2023年,广东,梅州市,2023年广东梅州市,115
+2024年,河北,保定市,2024年河北保定市,116
+2023年,云南,迪庆藏族自治州,2023年云南迪庆藏族自治州,117
+2022年,上海,宝山区,2022年上海宝山区,117
+2022年,上海,普陀区,2022年上海普陀区,117
+2023年,四川,德阳市,2023年四川德阳市,117
+2022年,青海,玉树藏族自治州,2022年青海玉树藏族自治州,118
+2022年,湖北,恩施市,2022年湖北恩施市,118
+2022年,河北,秦皇岛,2022年河北秦皇岛,118
+2024年,吉林,四平市,2024年吉林四平市,118
+2024年,重庆,武隆区,2024年重庆武隆区,118
+2023年,陕西,陕西,2023年陕西陕西,118
+2022年,云南,普洱市,2022年云南普洱市,119
+2023年,重庆,大渡口区,2023年重庆大渡口区,119
+2022年,重庆,永川区,2022年重庆永川区,119
+2024年,香港,香港特别行政区人民政府,2024年香港香港特别行政区人民政府,120
+2022年,天津,和平区,2022年天津和平区,121
+2022年,青海,海北藏族自治州,2022年青海海北藏族自治州,121
+2023年,河北,邯郸市,2023年河北邯郸市,121
+2024年,甘肃,陇南市,2024年甘肃陇南市,121
+2022年,重庆,江津区,2022年重庆江津区,122
+2023年,陕西,咸阳市,2023年陕西咸阳市,122
+2023年,福建,漳州市,2023年福建漳州市,122
+2023年,新疆,阿勒泰地区,2023年新疆阿勒泰地区,122
+2024年,宁夏,中卫市,2024年宁夏中卫市,123
+2023年,青海,玉树藏族自治州,2023年青海玉树藏族自治州,123
+2023年,山西,运城市,2023年山西运城市,123
+2024年,北京,海淀区,2024年北京海淀区,123
+2024年,北京,石景山区,2024年北京石景山区,123
+2023年,香港特别行政区人民政府,香港特别行政区人民政府,2023年香港特别行政区人民政府香港特别行政区人民政府,124
+2023年,河北,衡水市,2023年河北衡水市,124
+2024年,天津,南开区,2024年天津南开区,124
+2023年,广西,柳州市,2023年广西柳州市,125
+2024年,黑龙江,大兴安岭地区,2024年黑龙江大兴安岭地区,125
+2022年,辽宁,铁岭市,2022年辽宁铁岭市,126
+2022年,广西,南宁市,2022年广西南宁市,126
+2022年,新疆,塔城区,2022年新疆塔城区,126
+2024年,北京,房山区,2024年北京房山区,126
+2024年,青海,青海,2024年青海青海,127
+2023年,新疆,哈密市,2023年新疆哈密市,127
+2024年,宁夏,固原市,2024年宁夏固原市,128
+2023年,重庆,荣昌区,2023年重庆荣昌区,129
+2023年,黑龙江,大兴安岭地区,2023年黑龙江大兴安岭地区,129
+2023年,河北,石家庄市,2023年河北石家庄市,129
+2022年,辽宁,抚顺市,2022年辽宁抚顺市,129
+2023年,河南,安阳市,2023年河南安阳市,130
+2022年,重庆,渝中区,2022年重庆渝中区,130
+2022年,新疆,和田地区,2022年新疆和田地区,131
+2024年,河北,廊坊市,2024年河北廊坊市,131
+2022年,新疆,阿勒泰地区,2022年新疆阿勒泰地区,131
+2023年,河南,三门峡市,2023年河南三门峡市,131
+2024年,四川,四川,2024年四川四川,131
+2022年,中央,中央,2022年中央中央,132
+2023年,上海,虹口区,2023年上海虹口区,132
+2024年,西藏,林芝市,2024年西藏林芝市,132
+2023年,河南,驻马店市,2023年河南驻马店市,133
+2022年,江苏,无锡市,2022年江苏无锡市,134
+2023年,云南,丽江市,2023年云南丽江市,134
+2023年,中央,中央,2023年中央中央,134
+2024年,北京,大兴区,2024年北京大兴区,135
+2024年,云南,西双版纳傣族自治州,2024年云南西双版纳傣族自治州,135
+2023年,青海,海南藏族自治州,2023年青海海南藏族自治州,135
+2022年,黑龙江,大兴安岭地区,2022年黑龙江大兴安岭地区,135
+2024年,辽宁,盘锦市,2024年辽宁盘锦市,136
+2024年,辽宁,辽阳市,2024年辽宁辽阳市,136
+2022年,北京,石景山区,2022年北京石景山区,136
+2022年,广东,佛山市,2022年广东佛山市,136
+2022年,甘肃,陇南市,2022年甘肃陇南市,137
+2023年,黑龙江,七台河市,2023年黑龙江七台河市,137
+2024年,江西,新余市,2024年江西新余市,137
+2024年,云南,迪庆藏族自治州,2024年云南迪庆藏族自治州,138
+2024年,甘肃,天水市,2024年甘肃天水市,138
+2023年,北京,大兴区,2023年北京大兴区,138
+2023年,安徽,铜陵市,2023年安徽铜陵市,139
+2022年,上海,徐汇区,2022年上海徐汇区,139
+2022年,山西,忻州市,2022年山西忻州市,139
+2023年,新疆,吐鲁番市,2023年新疆吐鲁番市,139
+2022年,广东,云浮市,2022年广东云浮市,140
+2022年,云南,怒江傈僳族自治州,2022年云南怒江傈僳族自治州,140
+2023年,河北,邢台市,2023年河北邢台市,140
+2023年,甘肃,陇南市,2023年甘肃陇南市,140
+2024年,上海,宝山区,2024年上海宝山区,140
+2022年,安徽,宣城市,2022年安徽宣城市,140
+2022年,北京,大兴区,2022年北京大兴区,140
+2024年,青海,海西蒙古族藏族自治州,2024年青海海西蒙古族藏族自治州,140
+2022年,西藏,昌都市,2022年西藏昌都市,140
+2024年,黑龙江,七台河市,2024年黑龙江七台河市,141
+2023年,宁夏,固原市,2023年宁夏固原市,142
+2024年,广西,河池市,2024年广西河池市,142
+2024年,河北,沧州市,2024年河北沧州市,143
+2022年,辽宁,盘锦市,2022年辽宁盘锦市,143
+2023年,广东,佛山市,2023年广东佛山市,143
+2024年,黑龙江,伊春市,2024年黑龙江伊春市,144
+2022年,云南,保山市,2022年云南保山市,144
+2023年,湖北,黄冈市,2023年湖北黄冈市,144
+2024年,黑龙江,黑河市,2024年黑龙江黑河市,144
+2023年,湖北,恩施市,2023年湖北恩施市,144
+2022年,北京,门头沟区,2022年北京门头沟区,145
+2024年,安徽,蚌埠市,2024年安徽蚌埠市,145
+2022年,海南,五指山市,2022年海南五指山市,145
+2023年,陕西,商洛市,2023年陕西商洛市,145
+2024年,重庆,巴南区,2024年重庆巴南区,145
+2023年,天津,津南区,2023年天津津南区,146
+2023年,河北,沧州市,2023年河北沧州市,146
+2023年,辽宁,盘锦市,2023年辽宁盘锦市,146
+2024年,广西,贵港市,2024年广西贵港市,146
+2024年,海南,保亭黎族苗族自治县,2024年海南保亭黎族苗族自治县,146
+2022年,新疆,克孜勒苏柯尔克孜自治州,2022年新疆克孜勒苏柯尔克孜自治州,147
+2022年,黑龙江,黑龙江,2022年黑龙江黑龙江,147
+2023年,新疆,克孜勒苏柯尔克孜自治州,2023年新疆克孜勒苏柯尔克孜自治州,148
+2023年,海南,海南,2023年海南海南,148
+2023年,甘肃,张掖市,2023年甘肃张掖市,148
+2023年,黑龙江,黑龙江,2023年黑龙江黑龙江,149
+2022年,四川,德阳市,2022年四川德阳市,150
+2022年,广东,潮州市,2022年广东潮州市,151
+2023年,北京,海淀区,2023年北京海淀区,151
+2022年,河北,邢台市,2022年河北邢台市,151
+2024年,云南,保山市,2024年云南保山市,152
+2023年,云南,保山市,2023年云南保山市,152
+2024年,湖北,鄂州市,2024年湖北鄂州市,153
+2023年,重庆,铜梁区,2023年重庆铜梁区,153
+2023年,甘肃,金昌市,2023年甘肃金昌市,153
+2022年,辽宁,辽阳市,2022年辽宁辽阳市,153
+2022年,重庆,南岸区,2022年重庆南岸区,154
+2024年,甘肃,定西市,2024年甘肃定西市,154
+2022年,四川,雅安市,2022年四川雅安市,155
+2023,湖南,岳阳市,2023湖南岳阳市,155
+2024年,安徽,宿州市,2024年安徽宿州市,155
+2023年,甘肃,临夏回族自治州,2023年甘肃临夏回族自治州,155
+2023年,黑龙江,伊春市,2023年黑龙江伊春市,156
+2022年,河北,沧州市,2022年河北沧州市,157
+2023年,新疆,塔城区,2023年新疆塔城区,157
+2024年,广东,佛山市,2024年广东佛山市,158
+2023年,广东,河源市,2023年广东河源市,158
+2024年,重庆,荣昌区,2024年重庆荣昌区,159
+2023年,四川,雅安市,2023年四川雅安市,159
+2022年,江苏,徐州市,2022年江苏徐州市,159
+2022年,天津,武清区,2022年天津武清区,160
+2022年,河北,保定市,2022年河北保定市,160
+2022年,甘肃,嘉峪关市,2022年甘肃嘉峪关市,160
+2023年,重庆,大足区,2023年重庆大足区,162
+2024年,甘肃,白银市,2024年甘肃白银市,162
+2022年,重庆,长寿区,2022年重庆长寿区,162
+2024年,广东,肇庆市,2024年广东肇庆市,162
+2024年,甘肃,武威市,2024年甘肃武威市,163
+2023年,广西,河池市,2023年广西河池市,163
+2022年,重庆,铜梁区,2022年重庆铜梁区,163
+2022年,安徽,铜陵市,2022年安徽铜陵市,164
+2024年,广西,梧州市,2024年广西梧州市,164
+2023年,湖北,仙桃市,2023年湖北仙桃市,164
+2023年,海南,昌江黎族自治县,2023年海南昌江黎族自治县,164
+2022年,黑龙江,黑河市,2022年黑龙江黑河市,165
+2023年,上海,徐汇区,2023年上海徐汇区,165
+2022年,黑龙江,七台河市,2022年黑龙江七台河市,166
+2023年,四川,广元市,2023年四川广元市,166
+2022年,江西,南昌市,2022年江西南昌市,167
+2023年,江西,新余市,2023年江西新余市,167
+2022年,河南,安阳市,2022年河南安阳市,167
+2024年,辽宁,抚顺市,2024年辽宁抚顺市,167
+2023年,西藏,昌都市,2023年西藏昌都市,168
+2023,湖南,常德市,2023湖南常德市,168
+2023年,海南,万宁市,2023年海南万宁市,169
+2023年,云南,楚雄彝族自治州,2023年云南楚雄彝族自治州,169
+2022年,山西,朔州市,2022年山西朔州市,169
+2023年,北京,门头沟区,2023年北京门头沟区,169
+2022,湖南,湖南省,2022湖南湖南省,169
+2023年,甘肃,嘉峪关市,2023年甘肃嘉峪关市,170
+2024年,浙江,嘉兴市,2024年浙江嘉兴市,170
+2024年,北京,门头沟区,2024年北京门头沟区,170
+2024年,海南,五指山市,2024年海南五指山市,171
+2024年,新疆,巴音郭楞蒙古自治州,2024年新疆巴音郭楞蒙古自治州,171
+2024年,四川,雅安市,2024年四川雅安市,171
+2023年,新疆,巴音郭楞蒙古自治州,2023年新疆巴音郭楞蒙古自治州,172
+2023年,重庆,渝中区,2023年重庆渝中区,172
+2023年,辽宁,沈阳市,2023年辽宁沈阳市,172
+2024年,湖北,咸宁市,2024年湖北咸宁市,173
+2023年,天津,西青区,2023年天津西青区,173
+2022年,辽宁,锦州市,2022年辽宁锦州市,175
+2024年,浙江,舟山市,2024年浙江舟山市,175
+2022年,云南,大理白族自治州,2022年云南大理白族自治州,176
+2023年,辽宁,锦州市,2023年辽宁锦州市,177
+2024,湖南省,岳阳市,2024湖南省岳阳市,177
+2023年,四川,四川,2023年四川四川,177
+2022年,天津,津南区,2022年天津津南区,177
+2024年,湖北,恩施市,2024年湖北恩施市,177
+2022年,广东,珠海市,2022年广东珠海市,178
+2022年,湖北,咸宁市,2022年湖北咸宁市,179
+2023年,海南,保亭黎族苗族自治县,2023年海南保亭黎族苗族自治县,179
+2023年,宁夏,吴忠市,2023年宁夏吴忠市,179
+2023年,安徽,宣城市,2023年安徽宣城市,179
+2024年,安徽,铜陵市,2024年安徽铜陵市,180
+2024年,辽宁,锦州市,2024年辽宁锦州市,181
+2022年,新疆,巴音郭楞蒙古自治州,2022年新疆巴音郭楞蒙古自治州,181
+2024年,上海,虹口区,2024年上海虹口区,182
+2024,湖南省,湖南省,2024湖南省湖南省,182
+2023年,重庆,长寿区,2023年重庆长寿区,182
+2022年,海南,陵水黎族自治县,2022年海南陵水黎族自治县,182
+2024年," 山西",临汾市,2024年 山西临汾市,183
+2023年,湖北,潜江市,2023年湖北潜江市,183
+2024年,重庆,铜梁区,2024年重庆铜梁区,185
+2023年,福建,泉州市,2023年福建泉州市,185
+2024年,甘肃,平凉市,2024年甘肃平凉市,185
+2023年,广东,珠海市,2023年广东珠海市,185
+2024年,海南,万宁市,2024年海南万宁市,186
+2023年,青海,西宁市,2023年青海西宁市,187
+2024年,广西,百色市,2024年广西百色市,187
+2022年,广东,中山市,2022年广东中山市,188
+2024年,甘肃,张掖市,2024年甘肃张掖市,189
+2024年,四川,资阳市,2024年四川资阳市,189
+2023年,黑龙江,绥化市,2023年黑龙江绥化市,189
+2024年," 山西",晋中市,2024年 山西晋中市,189
+2023,湖南,怀化市,2023湖南怀化市,189
+2023年,天津,红桥区,2023年天津红桥区,190
+2023年,河北,保定市,2023年河北保定市,190
+2023年,安徽,宿州市,2023年安徽宿州市,190
+2024年,上海,徐汇区,2024年上海徐汇区,193
+2023年,宁夏,石嘴山市,2023年宁夏石嘴山市,193
+2024年,北京,平谷区,2024年北京平谷区,194
+2022年,云南,楚雄彝族自治州,2022年云南楚雄彝族自治州,194
+2022年,新疆,喀什地区,2022年新疆喀什地区,195
+2023年,广东,中山市,2023年广东中山市,195
+2023年,广西,广西,2023年广西广西,196
+2023年,海南,五指山市,2023年海南五指山市,196
+2024年,云南,大理白族自治州,2024年云南大理白族自治州,196
+2023年,重庆,黔江区,2023年重庆黔江区,197
+2023年,辽宁,抚顺市,2023年辽宁抚顺市,197
+2022年,浙江,金华市,2022年浙江金华市,198
+2022年,甘肃,张掖市,2022年甘肃张掖市,199
+2022年,云南,玉溪市,2022年云南玉溪市,199
+2024年,广西,广西,2024年广西广西,199
+2022年,重庆,万州区,2022年重庆万州区,199
+2023年,北京,北京市,2023年北京北京市,200
+2022年,河北,邯郸市,2022年河北邯郸市,201
+2022年,上海,崇明区,2022年上海崇明区,201
+2022年,北京,北京市,2022年北京北京市,203
+2024年,北京,北京市,2024年北京北京市,205
+2024年,河北,河北省,2024年河北河北省,205
+2022年,湖北,随州市,2022年湖北随州市,205
+2022年,河南,三门峡市,2022年河南三门峡市,207
+2022年,重庆,綦江区,2022年重庆綦江区,207
+2022年,甘肃,金昌市,2022年甘肃金昌市,209
+2022年,天津,西青区,2022年天津西青区,209
+2023年,浙江,金华市,2023年浙江金华市,209
+2022年,广东,汕尾市,2022年广东汕尾市,209
+2024年,广东,汕尾市,2024年广东汕尾市,210
+2023年,云南,大理白族自治州,2023年云南大理白族自治州,210
+2023年,青海,海西蒙古族藏族自治州,2023年青海海西蒙古族藏族自治州,210
+2022年,四川,巴中市,2022年四川巴中市,211
+2023年,广东,汕尾市,2023年广东汕尾市,212
+2024年,甘肃,嘉峪关市,2024年甘肃嘉峪关市,212
+2024年,湖北,随州市,2024年湖北随州市,213
+2022年,广东,阳江市,2022年广东阳江市,213
+2024年,江西,九江市,2024年江西九江市,214
+2024年,广东,阳江市,2024年广东阳江市,214
+2023年,甘肃,平凉市,2023年甘肃平凉市,214
+2022年,河南,鹤壁市,2022年河南鹤壁市,215
+2022年,江西,萍乡市,2022年江西萍乡市,215
+2022年,海南,昌江黎族自治县,2022年海南昌江黎族自治县,215
+2023年,北京,延庆区,2023年北京延庆区,216
+2023年,四川,遂宁市,2023年四川遂宁市,218
+2022年,重庆,黔江区,2022年重庆黔江区,219
+2024年,海南,文昌市,2024年海南文昌市,220
+2023年,湖北,随州市,2023年湖北随州市,220
+2022年,广东,肇庆市,2022年广东肇庆市,220
+2024年,湖北,十堰市,2024年湖北十堰市,221
+2023年,安徽,安庆市,2023年安徽安庆市,222
+2024年,天津,武清区,2024年天津武清区,222
+2024年,云南,红河哈尼族彝族自治州,2024年云南红河哈尼族彝族自治州,223
+2023年,广东,揭阳市,2023年广东揭阳市,223
+2024年,北京,延庆区,2024年北京延庆区,224
+2023年,河南,鹤壁市,2023年河南鹤壁市,224
+2023年,新疆,喀什地区,2023年新疆喀什地区,224
+2024年,黑龙江,绥化市,2024年黑龙江绥化市,225
+2023年,广东,江门市,2023年广东江门市,225
+2023年,广东,阳江市,2023年广东阳江市,225
+2024年,安徽,淮南市,2024年安徽淮南市,225
+2024年,广东,潮州市,2024年广东潮州市,225
+2022年,宁夏,石嘴山市,2022年宁夏石嘴山市,226
+2022年,四川,宜宾市,2022年四川宜宾市,227
+2024年,重庆,大足区,2024年重庆大足区,227
+2022年,甘肃,平凉市,2022年甘肃平凉市,227
+2022年,江西,上饶市,2022年江西上饶市,228
+2022年,海南,临高县,2022年海南临高县,228
+2023年,北京,房山区,2023年北京房山区,229
+2024年,广东,中山市,2024年广东中山市,229
+2023年,云南,红河哈尼族彝族自治州,2023年云南红河哈尼族彝族自治州,230
+2023年,重庆,永川区,2023年重庆永川区,230
+2023年,北京,平谷区,2023年北京平谷区,231
+2022年,安徽,黄山市,2022年安徽黄山市,232
+2023年,广西,来宾市,2023年广西来宾市,232
+2024年,黑龙江,鹤岗市,2024年黑龙江鹤岗市,232
+2023年,河北,河北,2023年河北河北,232
+2022年,江西,赣州市,2022年江西赣州市,232
+2022年,河北,河北,2022年河北河北,233
+2022年,甘肃,临夏回族自治州,2022年甘肃临夏回族自治州,234
+2022年,江西,抚州市,2022年江西抚州市,234
+2024年,甘肃,金昌市,2024年甘肃金昌市,235
+2024年,江西,萍乡市,2024年江西萍乡市,235
+2022年,吉林,白山市,2022年吉林白山市,235
+2022年,山西,吕梁市,2022年山西吕梁市,235
+2023年,广东,肇庆市,2023年广东肇庆市,236
+2024年,广东,揭阳市,2024年广东揭阳市,236
+2023年,山西,吕梁市,2023年山西吕梁市,236
+2022年,山西,晋城市,2022年山西晋城市,237
+2022年,黑龙江,鹤岗市,2022年黑龙江鹤岗市,237
+2023年,北京,西城区,2023年北京西城区,238
+2024年,天津,河东区,2024年天津河东区,240
+2023年,新疆,阿克苏地区,2023年新疆阿克苏地区,240
+2023年,甘肃,天水市,2023年甘肃天水市,241
+2022年,北京,平谷区,2022年北京平谷区,242
+2023年,安徽,滁州市,2023年安徽滁州市,244
+2022年,黑龙江,鸡西市,2022年黑龙江鸡西市,245
+2024年,海南,昌江黎族自治县,2024年海南昌江黎族自治县,245
+2022年,云南,西双版纳傣族自治州,2022年云南西双版纳傣族自治州,245
+2023年,广东,潮州市,2023年广东潮州市,245
+2022年,新疆,博尔塔拉蒙古自治州,2022年新疆博尔塔拉蒙古自治州,245
+2022年,广西,来宾市,2022年广西来宾市,246
+2023年,湖北,咸宁市,2023年湖北咸宁市,246
+2023年,广东,汕头市,2023年广东汕头市,246
+2022年,黑龙江,大庆市,2022年黑龙江大庆市,246
+2024年,北京,密云区,2024年北京密云区,247
+2022年,江西,鹰潭市,2022年江西鹰潭市,247
+2023年,广西,玉林市,2023年广西玉林市,247
+2022年,辽宁,丹东市,2022年辽宁丹东市,247
+2024年,广西,来宾市,2024年广西来宾市,249
+2024年,河北,唐山市,2024年河北唐山市,249
+2024年,辽宁,辽宁,2024年辽宁辽宁,249
+2024年,黑龙江,大庆市,2024年黑龙江大庆市,249
+2024年,浙江,嘉兴市bm,2024年浙江嘉兴市bm,250
+2023年,新疆,博尔塔拉蒙古自治州,2023年新疆博尔塔拉蒙古自治州,250
+2024年,安徽,宣城市,2024年安徽宣城市,250
+2023年,黑龙江,黑河市,2023年黑龙江黑河市,252
+2023年,云南,西双版纳傣族自治州,2023年云南西双版纳傣族自治州,252
+2022年,北京,西城区,2022年北京西城区,252
+2022年,广西,河池市,2022年广西河池市,252
+2023年,浙江,舟山市,2023年浙江舟山市,252
+2024年,四川,达州市,2024年四川达州市,253
+2022年,安徽,淮北市,2022年安徽淮北市,254
+2022年,广西,钦州市,2022年广西钦州市,254
+2023年,黑龙江,大庆市,2023年黑龙江大庆市,255
+2024年,广东,江门市,2024年广东江门市,255
+2024年,辽宁,沈阳市,2024年辽宁沈阳市,255
+2024年,北京,西城区,2024年北京西城区,255
+2024年,重庆,黔江区,2024年重庆黔江区,255
+2022年,广东,江门市,2022年广东江门市,256
+2023年,重庆,南岸区,2023年重庆南岸区,256
+2024,湖南省,邵阳市,2024湖南省邵阳市,256
+2022年,天津,蓟州区,2022年天津蓟州区,257
+2023年,江西,鹰潭市,2023年江西鹰潭市,257
+2022年,福建,南平市,2022年福建南平市,257
+2023年,湖北,十堰市,2023年湖北十堰市,258
+2022年,湖北,十堰市,2022年湖北十堰市,258
+2024年,福建,南平市,2024年福建南平市,258
+2023年,广西,贵港市,2023年广西贵港市,258
+2024年,海南,澄迈县,2024年海南澄迈县,258
+2023,湖南,邵阳市,2023湖南邵阳市,258
+2022年,安徽,宿州市,2022年安徽宿州市,259
+2023年,江西,抚州市,2023年江西抚州市,259
+2024年,新疆,乌鲁木齐市,2024年新疆乌鲁木齐市,260
+2024年,河南,开封市,2024年河南开封市,260
+2023年,云南,玉溪市,2023年云南玉溪市,260
+2024年,黑龙江,鸡西市,2024年黑龙江鸡西市,261
+2023年,山西,晋城市,2023年山西晋城市,261
+2024年,天津,滨海新区,2024年天津滨海新区,262
+2024年,重庆,渝中区,2024年重庆渝中区,263
+2022年,青海,海西蒙古族藏族自治州,2022年青海海西蒙古族藏族自治州,264
+2023年,辽宁,辽宁,2023年辽宁辽宁,265
+2022年,重庆,沙坪坝区,2022年重庆沙坪坝区,266
+2023年,上海,崇明区,2023年上海崇明区,266
+2023年,海南,文昌市,2023年海南文昌市,266
+2023年,安徽,淮北市,2023年安徽淮北市,266
+2023年,重庆,万州区,2023年重庆万州区,267
+2023年,吉林,白山市,2023年吉林白山市,267
+2024年,福建,漳州市,2024年福建漳州市,268
+2023年,陕西,渭南市,2023年陕西渭南市,268
+2024年,安徽,亳州市,2024年安徽亳州市,268
+2024年,河南,郑州市,2024年河南郑州市,268
+2024年,河北,张家口市,2024年河北张家口市,269
+2023年,辽宁,辽阳市,2023年辽宁辽阳市,269
+2024年,天津,河北区,2024年天津河北区,272
+2023年,福建,南平市,2023年福建南平市,273
+2024年,北京,丰台区,2024年北京丰台区,273
+2023年,江西,景德镇市,2023年江西景德镇市,274
+2022年,吉林,辽源市,2022年吉林辽源市,274
+2022年,海南,儋州市,2022年海南儋州市,274
+2024年,青海,西宁市,2024年青海西宁市,274
+2022年,广西,贵港市,2022年广西贵港市,275
+2022年,四川,内江市,2022年四川内江市,276
+2023年,湖北,襄阳市,2023年湖北襄阳市,276
+2023年,辽宁,阜新市,2023年辽宁阜新市,276
+2022年,新疆,吐鲁番市,2022年新疆吐鲁番市,276
+2022年,安徽,滁州市,2022年安徽滁州市,278
+2022年,北京,延庆区,2022年北京延庆区,278
+2022年,广东,清远市,2022年广东清远市,280
+2024年,重庆,南川区,2024年重庆南川区,280
+2023年,湖北,孝感市,2023年湖北孝感市,280
+2023年,吉林,辽源市,2023年吉林辽源市,280
+2024年,黑龙江,哈尔滨市,2024年黑龙江哈尔滨市,281
+2024年,广东,湛江市,2024年广东湛江市,281
+2024年,海南,儋州市,2024年海南儋州市,282
+2024年,辽宁,丹东市,2024年辽宁丹东市,282
+2021年,陕西省,渭南市,2021年陕西省渭南市,282
+2022年,云南,丽江市,2022年云南丽江市,285
+2023年,四川,巴中市,2023年四川巴中市,286
+2024年,浙江,金华市,2024年浙江金华市,286
+2023年,四川,攀枝花市,2023年四川攀枝花市,286
+2024年,吉林,辽源市,2024年吉林辽源市,286
+2023年,湖北,鄂州市,2023年湖北鄂州市,287
+2024年,广东,韶关市,2024年广东韶关市,287
+2023年,湖北,神农架林区,2023年湖北神农架林区,288
+2022年,陕西,宝鸡市,2022年陕西宝鸡市,288
+2023年,福建,宁德市,2023年福建宁德市,288
+2022年,河南,濮阳市,2022年河南濮阳市,288
+2022,湖南,益阳市,2022湖南益阳市,288
+2023年,四川,南充市,2023年四川南充市,289
+2023年,西藏,林芝市,2023年西藏林芝市,289
+2022年,重庆,南川区,2022年重庆南川区,289
+2022年,新疆,乌鲁木齐市,2022年新疆乌鲁木齐市,289
+2024,湖南省,益阳市,2024湖南省益阳市,290
+2023,湖南,益阳市,2023湖南益阳市,291
+2022年,河南,开封市,2022年河南开封市,292
+2023年,浙江,嘉兴市bm,2023年浙江嘉兴市bm,293
+2023年,河南,濮阳市,2023年河南濮阳市,293
+2023年,四川,达州市,2023年四川达州市,294
+2024年,辽宁,阜新市,2024年辽宁阜新市,295
+2023年,黑龙江,鹤岗市,2023年黑龙江鹤岗市,295
+2023年,重庆,南川区,2023年重庆南川区,296
+2022年,四川,达州市,2022年四川达州市,298
+2024年,云南,玉溪市,2024年云南玉溪市,298
+2023年,陕西,宝鸡市,2023年陕西宝鸡市,298
+2022年,陕西,西安,2022年陕西西安,299
+2023年,北京,密云区,2023年北京密云区,299
+2023年,西藏,西藏,2023年西藏西藏,299
+2022年,河南,河南,2022年河南河南,300
+2024年,海南,白沙黎族自治县,2024年海南白沙黎族自治县,300
+2023年,黑龙江,鸡西市,2023年黑龙江鸡西市,300
+2023年,广东,湛江市,2023年广东湛江市,301
+2022年,西藏,西藏,2022年西藏西藏,302
+2024年,河北,承德市,2024年河北承德市,302
+2022年,河南,信阳市,2022年河南信阳市,303
+2024年,天津,蓟州区,2024年天津蓟州区,303
+2022年,河北,张家口市,2022年河北张家口市,303
+2023年,河南,开封市,2023年河南开封市,304
+2022年,湖北,襄阳市,2022年湖北襄阳市,305
+2022年,河南,周口市,2022年河南周口市,306
+2023年,河南,许昌市,2023年河南许昌市,306
+2022年,辽宁,辽宁,2022年辽宁辽宁,309
+2023年,河南,周口市,2023年河南周口市,309
+2023年,上海,青浦区,2023年上海青浦区,309
+2024年,海南,琼海市,2024年海南琼海市,309
+2024年,江苏,泰州市,2024年江苏泰州市,310
+2024年,湖北,神农架林区,2024年湖北神农架林区,311
+2023年,河南,焦作市,2023年河南焦作市,312
+2024年,重庆,沙坪坝区,2024年重庆沙坪坝区,312
+2022年,四川,攀枝花市,2022年四川攀枝花市,313
+2023年,甘肃,武威市,2023年甘肃武威市,315
+2024年,北京,通州区,2024年北京通州区,316
+2023年,上海,静安区,2023年上海静安区,317
+2023年,四川,宜宾市,2023年四川宜宾市,317
+2024年,北京,昌平区,2024年北京昌平区,319
+2024年,上海,闵行区,2024年上海闵行区,319
+2022年,北京,房山区,2022年北京房山区,320
+2022年,四川,南充市,2022年四川南充市,320
+2022年,四川,眉山市,2022年四川眉山市,320
+2022年,河南,南阳市,2022年河南南阳市,320
+2024年,西藏,拉萨市,2024年西藏拉萨市,320
+2023年,浙江,绍兴市,2023年浙江绍兴市,321
+2024年,重庆,重庆,2024年重庆重庆,321
+2024年,河北,石家庄市,2024年河北石家庄市,324
+2023年,辽宁,丹东市,2023年辽宁丹东市,324
+2023年,山西,大同市,2023年山西大同市,324
+2022年,河南,许昌市,2022年河南许昌市,326
+2022年,新疆,阿克苏地区,2022年新疆阿克苏地区,326
+2022年,广西,梧州市,2022年广西梧州市,327
+2024年,江西,鹰潭市,2024年江西鹰潭市,327
+2022年,北京,密云区,2022年北京密云区,327
+2023年,海南,定安县,2023年海南定安县,328
+2022年,广东,揭阳市,2022年广东揭阳市,329
+2023年,北京,昌平区,2023年北京昌平区,329
+2024年,西藏,西藏,2024年西藏西藏,329
+2024年,湖北,仙桃市,2024年湖北仙桃市,329
+2024年,天津,和平区,2024年天津和平区,331
+2023年,浙江,湖州市,2023年浙江湖州市,331
+2024年,天津,东丽区,2024年天津东丽区,333
+2022年,河南,焦作市,2022年河南焦作市,334
+2024年,湖北,宜昌市,2024年湖北宜昌市,334
+2023年,安徽,亳州市,2023年安徽亳州市,334
+2023年,安徽,安徽,2023年安徽安徽,334
+2023年,江西,南昌市,2023年江西南昌市,335
+2024年,浙江,湖州市,2024年浙江湖州市,337
+2022年,山西,运城市,2022年山西运城市,337
+2022年,北京,东城区,2022年北京东城区,338
+2022年,甘肃,定西市,2022年甘肃定西市,339
+2024年,天津,静海区,2024年天津静海区,341
+2022年,上海,松江区,2022年上海松江区,342
+2022年,安徽,亳州市,2022年安徽亳州市,342
+2022年,江西,新余市,2022年江西新余市,342
+2022年,浙江,绍兴市,2022年浙江绍兴市,343
+2022年,广东,河源市,2022年广东河源市,343
+2024年,广东,河源市,2024年广东河源市,343
+2023年,广西,贺州市,2023年广西贺州市,345
+2024年,北京,顺义区,2024年北京顺义区,346
+2023年,河北,张家口市,2023年河北张家口市,346
+2023年,四川,眉山市,2023年四川眉山市,347
+2023年,河北,唐山市,2023年河北唐山市,347
+2023年,广西,梧州市,2023年广西梧州市,347
+2023年,河南,郑州市,2023年河南郑州市,348
+2022年,青海,西宁市,2022年青海西宁市,348
+2023年,西藏,西藏自治区,2023年西藏西藏自治区,348
+2024年,重庆,南岸区,2024年重庆南岸区,349
+2023年,河南,洛阳市,2023年河南洛阳市,349
+2023年,广东,东莞市,2023年广东东莞市,350
+2024年,广东,清远市,2024年广东清远市,352
+2022年,重庆,梁平区,2022年重庆梁平区,353
+2023年,海南,白沙黎族自治县,2023年海南白沙黎族自治县,353
+2023年,新疆,乌鲁木齐市,2023年新疆乌鲁木齐市,355
+2023年,海南,澄迈县,2023年海南澄迈县,356
+2024年,江苏,连云港市,2024年江苏连云港市,356
+2024年,广东,东莞市,2024年广东东莞市,356
+2022年,浙江,湖州市,2022年浙江湖州市,356
+2022年,北京,昌平区,2022年北京昌平区,357
+2023年,湖北,宜昌市,2023年湖北宜昌市,358
+2023年,甘肃,定西市,2023年甘肃定西市,359
+2022年,海南,澄迈县,2022年海南澄迈县,359
+2022年,湖北,宜昌市,2022年湖北宜昌市,360
+2024年,安徽,安徽,2024年安徽安徽,360
+2024年,福建,福建,2024年福建福建,361
+2023年,重庆,沙坪坝区,2023年重庆沙坪坝区,362
+2022年,福建,福建,2022年福建福建,362
+2023年,河南,信阳市,2023年河南信阳市,363
+2022年,广东,湛江市,2022年广东湛江市,365
+2022年,广西,贺州市,2022年广西贺州市,366
+2023年,福建,福建,2023年福建福建,367
+2024年,上海,杨浦区,2024年上海杨浦区,368
+2024年,四川,内江市,2024年四川内江市,368
+2022年,福建,漳州市,2022年福建漳州市,368
+2024年,云南,云南,2024年云南云南,369
+2022年,河南,郑州市,2022年河南郑州市,369
+2023年,广东,清远市,2023年广东清远市,369
+2023年,天津,蓟州区,2023年天津蓟州区,370
+2024年,上海,青浦区,2024年上海青浦区,372
+2024,湖南省,怀化市,2024湖南省怀化市,373
+2024年,江苏,宿迁市,2024年江苏宿迁市,374
+2022年,河北,唐山市,2022年河北唐山市,376
+2024年,安徽,池州市,2024年安徽池州市,377
+2024年,四川,攀枝花市,2024年四川攀枝花市,378
+2022年,广西,桂林市,2022年广西桂林市,378
+2022年,黑龙江,牡丹江市,2022年黑龙江牡丹江市,379
+2022年,广西,百色市,2022年广西百色市,381
+2023年,山西,临汾市,2023年山西临汾市,382
+2024年,上海,静安区,2024年上海静安区,382
+2024年,宁夏,吴忠市,2024年宁夏吴忠市,382
+2023年,天津,南开区,2023年天津南开区,385
+2022年,四川,资阳市,2022年四川资阳市,388
+2023年,上海,闵行区,2023年上海闵行区,389
+2024年,四川,宜宾市,2024年四川宜宾市,390
+2023年,广东,韶关市,2023年广东韶关市,391
+2022年,北京,通州区,2022年北京通州区,391
+2023年,海南,东方市,2023年海南东方市,391
+2024年,上海,奉贤区,2024年上海奉贤区,393
+2023年,黑龙江,牡丹江市,2023年黑龙江牡丹江市,395
+2024年,浙江,绍兴市,2024年浙江绍兴市,396
+2023年,河北,廊坊市,2023年河北廊坊市,397
+2022年,黑龙江,绥化市,2022年黑龙江绥化市,399
+2022年,河南,平顶山市,2022年河南平顶山市,400
+2022年,河北,廊坊市,2022年河北廊坊市,402
+2022,湖南,怀化市,2022湖南怀化市,402
+2023年,北京,东城区,2023年北京东城区,403
+2024年,黑龙江,牡丹江市,2024年黑龙江牡丹江市,403
+2024年,甘肃,酒泉市,2024年甘肃酒泉市,405
+2022年,湖北,孝感市,2022年湖北孝感市,405
+2024,湖南省,永州市,2024湖南省永州市,406
+2023年,天津,武清区,2023年天津武清区,406
+2023年,上海,杨浦区,2023年上海杨浦区,406
+2022年,安徽,阜阳市,2022年安徽阜阳市,408
+2023年,山西,晋中市,2023年山西晋中市,411
+2024年,天津,天津,2024年天津天津,411
+2023年,山西,长治市,2023年山西长治市,412
+2022年,上海,奉贤区,2022年上海奉贤区,412
+2023年,四川,成都市,2023年四川成都市,414
+2022年,云南,红河哈尼族彝族自治州,2022年云南红河哈尼族彝族自治州,415
+2022年,河南,济源市,2022年河南济源市,415
+2023年,广西,百色市,2023年广西百色市,415
+2022年,上海,杨浦区,2022年上海杨浦区,419
+2022年,上海,闵行区,2022年上海闵行区,420
+2024年,上海,松江区,2024年上海松江区,420
+2022年,天津,南开区,2022年天津南开区,421
+2024年,湖北,武汉市,2024年湖北武汉市,421
+2023年,河南,济源市,2023年河南济源市,421
+2023年,四川,资阳市,2023年四川资阳市,422
+2024年,江苏,常州市,2024年江苏常州市,422
+2023年,湖北,荆州市,2023年湖北荆州市,424
+2023年,江苏,宿迁市,2023年江苏宿迁市,424
+2023年,江苏,常州市,2023年江苏常州市,425
+2023年,河北,承德市,2023年河北承德市,425
+2022年,湖北,神农架林区,2022年湖北神农架林区,426
+2024年,广东,广州市,2024年广东广州市,426
+2022年,黑龙江,哈尔滨市,2022年黑龙江哈尔滨市,427
+2022年,福建,龙岩市,2022年福建龙岩市,428
+2024年,重庆,涪陵区,2024年重庆涪陵区,429
+2024年,福建,龙岩市,2024年福建龙岩市,429
+2022年,重庆,渝北区,2022年重庆渝北区,429
+2023年,四川,广安市,2023年四川广安市,429
+2022年,重庆,开州区,2022年重庆开州区,430
+2023年,江苏,淮安市,2023年江苏淮安市,431
+2023年,浙江,宁波市,2023年浙江宁波市,433
+2022年,浙江,宁波市,2022年浙江宁波市,434
+2024年,浙江,宁波市,2024年浙江宁波市,434
+2022年,河南,洛阳市,2022年河南洛阳市,435
+2023年,河南,南阳市,2023年河南南阳市,437
+2024年,江苏,扬州市,2024年江苏扬州市,437
+2022年,四川,成都市,2022年四川成都市,439
+2024年,安徽,淮北市,2024年安徽淮北市,440
+2023年,天津,宁河区,2023年天津宁河区,442
+2023年,黑龙江,佳木斯市,2023年黑龙江佳木斯市,443
+2023年,福建,龙岩市,2023年福建龙岩市,443
+2023年,黑龙江,哈尔滨市,2023年黑龙江哈尔滨市,443
+2023年,北京,朝阳区,2023年北京朝阳区,444
+2023年,天津,天津市,2023年天津天津市,445
+2022年,上海,青浦区,2022年上海青浦区,446
+2023年,黑龙江,齐齐哈尔市,2023年黑龙江齐齐哈尔市,446
+2024年,四川,遂宁市,2024年四川遂宁市,447
+2022年,新疆,伊犁哈萨克自治州,2022年新疆伊犁哈萨克自治州,450
+2022年,广东,韶关市,2022年广东韶关市,452
+2023年,湖北,武汉市,2023年湖北武汉市,452
+2023年,重庆,梁平区,2023年重庆梁平区,455
+2024年,黑龙江,佳木斯市,2024年黑龙江佳木斯市,456
+2024年,安徽,阜阳市,2024年安徽阜阳市,465
+2022年,湖北,武汉市,2022年湖北武汉市,467
+2022年,天津,天津市,2022年天津天津市,472
+2023年,重庆,綦江区,2023年重庆綦江区,473
+2022年,甘肃,武威市,2022年甘肃武威市,474
+2022年,浙江,杭州市,2022年浙江杭州市,474
+2022年,福建,宁德市,2022年福建宁德市,475
+2023年,天津,北辰区,2023年天津北辰区,479
+2023年,浙江,杭州市,2023年浙江杭州市,479
+2024年,四川,成都市,2024年四川成都市,481
+2022年,黑龙江,佳木斯市,2022年黑龙江佳木斯市,483
+2023年,安徽,阜阳市,2023年安徽阜阳市,491
+2022年,四川,广安市,2022年四川广安市,494
+2024年,重庆,梁平区,2024年重庆梁平区,496
+2024年,四川,泸州市,2024年四川泸州市,498
+2022年,河北,承德市,2022年河北承德市,500
+2024年,海南,东方市,2024年海南东方市,501
+2023年,四川,泸州市,2023年四川泸州市,501
+2022年,天津,宁河区,2022年天津宁河区,502
+2022年,四川,泸州市,2022年四川泸州市,505
+2024年,安徽,滁州市,2024年安徽滁州市,507
+2023年,江苏,连云港市,2023年江苏连云港市,508
+2024年,甘肃,兰州市,2024年甘肃兰州市,515
+2024年,重庆,九龙坡区,2024年重庆九龙坡区,518
+2022年,天津,静海区,2022年天津静海区,519
+2023年,新疆,伊犁哈萨克自治州,2023年新疆伊犁哈萨克自治州,519
+2024年,重庆,江津区,2024年重庆江津区,519
+2022年,广西,北海市,2022年广西北海市,521
+2023年,江苏,镇江市,2023年江苏镇江市,526
+2022年,甘肃,白银市,2022年甘肃白银市,527
+2024年,江苏,镇江市,2024年江苏镇江市,528
+2024年,江苏,南通市,2024年江苏南通市,532
+2024年,上海,长宁区,2024年上海长宁区,532
+2023年,海南,琼海市,2023年海南琼海市,534
+2022年,上海,嘉定区,2022年上海嘉定区,534
+2022年,安徽,安徽,2022年安徽安徽,535
+2023年,重庆,江津区,2023年重庆江津区,536
+2024年,天津,北辰区,2024年天津北辰区,545
+2024年,重庆,万州区,2024年重庆万州区,550
+2024年,重庆,长寿区,2024年重庆长寿区,551
+2024年,江苏,盐城市,2024年江苏盐城市,553
+2022年,浙江,浙江,2022年浙江浙江,553
+2024年,福建,莆田市,2024年福建莆田市,555
+2023年,浙江,浙江,2023年浙江浙江,559
+2022年,上海,黄埔区,2022年上海黄埔区,564
+2023年,天津,河东区,2023年天津河东区,568
+2024年,江苏,无锡市,2024年江苏无锡市,569
+2023年,天津,河西区,2023年天津河西区,571
+2024年,广西,北海市,2024年广西北海市,573
+2022年,天津,河东区,2022年天津河东区,574
+2024年,天津,河西区,2024年天津河西区,582
+2024年,浙江,浙江,2024年浙江浙江,583
+2023年,重庆,开州区,2023年重庆开州区,583
+2022年,天津,滨海新区,2022年天津滨海新区,583
+2023年,甘肃,兰州市,2023年甘肃兰州市,587
+2024年,江苏,淮安市,2024年江苏淮安市,587
+2023年,上海,奉贤区,2023年上海奉贤区,587
+2022年,四川,四川,2022年四川四川,590
+2023年,天津,静海区,2023年天津静海区,591
+2023年,江苏,南通市,2023年江苏南通市,591
+2022年,天津,红桥区,2022年天津红桥区,610
+2024年,浙江,杭州市,2024年浙江杭州市,616
+2024年,重庆,永川区,2024年重庆永川区,626
+2024年,北京,朝阳区,2024年北京朝阳区,630
+2023年,江苏,泰州市,2023年江苏泰州市,645
+2024年,上海,黄埔区,2024年上海黄埔区,649
+2023年,江苏,无锡市,2023年江苏无锡市,650
+2023年,北京,通州区,2023年北京通州区,654
+2023年,陕西,西安市,2023年陕西西安市,658
+2022年,甘肃,兰州市,2022年甘肃兰州市,665
+2023年,上海,黄埔区,2023年上海黄埔区,667
+2023年,河南,河南,2023年河南河南,669
+2022年,安徽,池州市,2022年安徽池州市,672
+2022年,黑龙江,齐齐哈尔市,2022年黑龙江齐齐哈尔市,682
+2024年,黑龙江,齐齐哈尔市,2024年黑龙江齐齐哈尔市,698
+2024年,天津,宝坻区,2024年天津宝坻区,702
+2022年,山西,大同市,2022年山西大同市,702
+2023年,安徽,池州市,2023年安徽池州市,706
+2022年,天津,北辰区,2022年天津北辰区,708
+2023年,江苏,盐城市,2023年江苏盐城市,713
+2024年,重庆,綦江区,2024年重庆綦江区,723
+2024年,福建,厦门市,2024年福建厦门市,740
+2023年,上海,松江区,2023年上海松江区,743
+2024年,江苏,徐州市,2024年江苏徐州市,746
+2024年,北京,怀柔区,2024年北京怀柔区,747
+2023年,广西,桂林市,2023年广西桂林市,750
+2023年,上海,嘉定区,2023年上海嘉定区,762
+2023年,广西,北海市,2023年广西北海市,765
+2023年,北京,怀柔区,2023年北京怀柔区,770
+2023年,江苏,徐州市,2023年江苏徐州市,771
+2024年,江苏,南京市,2024年江苏南京市,777
+2022年,北京,怀柔区,2022年北京怀柔区,795
+2023年,江苏,南京市,2023年江苏南京市,796
+2024年,重庆,渝北区,2024年重庆渝北区,825
+2022年,上海,浦东新区,2022年上海浦东新区,826
+2023年,上海,长宁区,2023年上海长宁区,843
+2024年,湖北,襄阳市,2024年湖北襄阳市,845
+2022年,上海,长宁区,2022年上海长宁区,859
+2022年,福建,厦门市,2022年福建厦门市,877
+2022年,天津,东丽区,2022年天津东丽区,884
+2022年,天津,河西区,2022年天津河西区,889
+2023年,天津,河北区,2023年天津河北区,892
+2023年,福建,厦门市,2023年福建厦门市,892
+2024年,湖北,潜江市,2024年湖北潜江市,905
+2023年,上海,浦东新区,2023年上海浦东新区,909
+2024年,上海,嘉定区,2024年上海嘉定区,914
+2023年,天津,宝坻区,2023年天津宝坻区,915
+2023年,甘肃,白银市,2023年甘肃白银市,926
+2024年,内蒙古,二连浩特市,2024年内蒙古二连浩特市,950
+2022年,江西,江西,2022年江西江西,961
+2022年,北京,朝阳区,2022年北京朝阳区,988
+2022年,北京,海淀区,2022年北京海淀区,988
+2022年,四川,遂宁市,2022年四川遂宁市,1016
+2023年,内蒙古,二连浩特市,2023年内蒙古二连浩特市,1037
+2022年,重庆,重庆,2022年重庆重庆,1075
+2023年,福建,莆田市,2023年福建莆田市,1088
+2022年,上海,上海,2022年上海上海,1120
+2022年,山西,长治市,2022年山西长治市,1123
+2022年,福建,莆田市,2022年福建莆田市,1153
+2023年,重庆,重庆,2023年重庆重庆,1168
+2024年,广东,深圳市,2024年广东深圳市,1177
+2024年,海南,三亚市,2024年海南三亚市,1208
+2022年,天津,河北区,2022年天津河北区,1251
+2023年,天津,东丽区,2023年天津东丽区,1305
+2023年,重庆,渝北区,2023年重庆渝北区,1334
+2024年,上海,上海,2024年上海上海,1420
+2024年,上海,浦东新区,2024年上海浦东新区,1426
+2022年,山西,晋中市,2022年山西晋中市,1479
+2023年,天津,和平区,2023年天津和平区,1483
+2024年,江苏,苏州市,2024年江苏苏州市,1499
+2023年,上海,上海,2023年上海上海,1503
+2022年,天津,宝坻区,2022年天津宝坻区,1785
+2024年,内蒙古,满洲里市,2024年内蒙古满洲里市,1787
+2023年,内蒙古,满洲里市,2023年内蒙古满洲里市,1794
+2022年,内蒙古,满洲里市,2022年内蒙古满洲里市,1938
+2023年,江苏,苏州市,2023年江苏苏州市,1993
+2022年,内蒙古,兴安盟,2022年内蒙古兴安盟,2046
+2024年,湖北,荆州市,2024年湖北荆州市,2067
+2022年,安徽,六安市,2022年安徽六安市,2082
+2023年,内蒙古,兴安盟,2023年内蒙古兴安盟,2110
+2024年,安徽,六安市,2024年安徽六安市,2312
+2023年,安徽,六安市,2023年安徽六安市,2331
+2022年,湖北,荆州市,2022年湖北荆州市,2335
+2023年,内蒙古,赤峰市,2023年内蒙古赤峰市,2426
+2024年,内蒙古,赤峰市,2024年内蒙古赤峰市,2467
+2022年,内蒙古,赤峰市,2022年内蒙古赤峰市,2594
+2024年,内蒙古,呼伦贝尔市,2024年内蒙古呼伦贝尔市,2600
+2024年,内蒙古,乌海市,2024年内蒙古乌海市,2638
+2023年,内蒙古,乌海市,2023年内蒙古乌海市,2679
+2024年,内蒙古,巴彦淖尔市,2024年内蒙古巴彦淖尔市,2801
+2023年,内蒙古,巴彦淖尔市,2023年内蒙古巴彦淖尔市,2812
+2022年,内蒙古,乌海市,2022年内蒙古乌海市,2848
+2023年,内蒙古,呼伦贝尔市,2023年内蒙古呼伦贝尔市,2916
+2023年,内蒙古,阿拉善盟,2023年内蒙古阿拉善盟,2991
+2022年,内蒙古,阿拉善盟,2022年内蒙古阿拉善盟,3052
+2022年,内蒙古,锡林郭勒盟,2022年内蒙古锡林郭勒盟,3259
+2023年,内蒙古,锡林郭勒盟,2023年内蒙古锡林郭勒盟,3288
+2022年,内蒙古,乌兰察布市,2022年内蒙古乌兰察布市,3368
+2023年,内蒙古,乌兰察布市,2023年内蒙古乌兰察布市,3594
+2022年,内蒙古,巴彦淖尔市,2022年内蒙古巴彦淖尔市,3669
+2022年,内蒙古,呼伦贝尔市,2022年内蒙古呼伦贝尔市,3720
+2022年,内蒙古,通辽市,2022年内蒙古通辽市,3953
+2024年,内蒙古,鄂尔多斯市,2024年内蒙古鄂尔多斯市,4221
+2022年,内蒙古,鄂尔多斯市,2022年内蒙古鄂尔多斯市,4271
+2024年,内蒙古,通辽市,2024年内蒙古通辽市,4285
+2023年,内蒙古,鄂尔多斯市,2023年内蒙古鄂尔多斯市,4293
+2022年,内蒙古,二连浩特市,2022年内蒙古二连浩特市,4369
+2024年,内蒙古,乌兰察布市,2024年内蒙古乌兰察布市,4987
+2023年,内蒙古,包头市,2023年内蒙古包头市,5411
+2024年,内蒙古,包头市,2024年内蒙古包头市,5487
+2022年,内蒙古,包头市,2022年内蒙古包头市,5535
+2023年,内蒙古,通辽市,2023年内蒙古通辽市,5610
+2023年,内蒙古,呼和浩特市,2023年内蒙古呼和浩特市,5716
+2022年,内蒙古,呼和浩特市,2022年内蒙古呼和浩特市,5744
+2024年,内蒙古,呼和浩特市,2024年内蒙古呼和浩特市,5756
+2024年,内蒙古,内蒙古,2024年内蒙古内蒙古,11399
+2022年,内蒙古,内蒙古,2022年内蒙古内蒙古,12681
+2023年,内蒙古,内蒙古自治区政府,2023年内蒙古内蒙古自治区政府,12785

program/spider/cli/data2html/main.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	_ "embed"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type (
+	//Attach 附件
+	Attach struct {
+		Id         int
+		Department string `json:"department"`
+		Href       string `json:"attach_href"`
+		Title      string `json:"info_title"`
+		Raw        string
+	}
+	Attaches []*Attach
+)
+
+var (
+	df  = flag.String("df", "./data.dat", "datafile 数据文件")
+	of  = flag.String("of", "./data.html", "输出的html文件")
+	tpl *template.Template
+	//go:embed tpl.html
+	html    string
+	reg, _  = regexp.Compile("(\\d{4}年)")
+	repeart = map[string]int{}
+)
+
+// readDataFile
+func readDataFile() Attaches {
+	ret := make(Attaches, 0, 0)
+	bs, err := ioutil.ReadFile(*df)
+	if err != nil {
+		log.Fatal(err)
+	}
+	content := string(bs)
+	index := 1
+	for _, s := range strings.Split(content, "\n") {
+		if len(s) == 0 {
+			continue
+		}
+		var data string = s
+		if strings.HasPrefix(s, ",") {
+			data = s[1:]
+		}
+		var attach = new(Attach)
+		err = json.Unmarshal([]byte(data), attach)
+		if err == nil {
+			ret = append(ret, attach)
+			attach.Raw = s
+			attach.Id = index
+			index += 1
+			//TODO 修改标题
+			attach.Title = rename(attach.Title)
+		}
+	}
+	return ret
+}
+
+// rename
+func rename(text string) string {
+	if reg.MatchString(text) {
+		m := reg.FindAllString(text, -1)
+		year := findMaxYear(m)
+		text = reg.ReplaceAllString(text, "")
+		ext := ""
+		if strings.Contains(text, ".") {
+			tmp := strings.Split(text, ".")
+			ext = strings.ToLower(tmp[len(tmp)-1])
+			if ext == "pdf" || ext == "doc" || ext == "docx" || ext == "xls" || ext == "xlsx" || ext == "zip" || ext == "rar" {
+				text = text[:len(text)-4]
+			}
+		}
+		key := fmt.Sprintf("%d_%s", year, text)
+		if v, ok := repeart[key]; ok {
+			repeart[key] = v + 1
+			return fmt.Sprintf("%s_%d.%s", key, v+1, ext)
+		} else {
+			repeart[key] = 1
+			return fmt.Sprintf("%s_%d.%s", key, 1, ext)
+		}
+	}
+	return text
+}
+
+// findMaxYear
+func findMaxYear(src []string) int {
+	year := 2022
+	for _, v := range src {
+		y, err := strconv.Atoi(v[:4])
+		if err == nil && y > year {
+			year = y
+		}
+	}
+	return year
+}
+
+// init
+func init() {
+	flag.Parse()
+	var err error
+	tpl, err = template.New("tpl").Parse(html)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+// main
+func main() {
+	rs := readDataFile()
+	fo, err := os.Create(*of)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = tpl.Execute(fo, rs)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fo.Close()
+}

program/spider/cli/data2html/tpl.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html><html><head><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta charset="utf-8">
+<style>
+body{
+  font-size:12pt;
+  line-height:1.5em;
+}
+li{
+  padding:10px 15px;
+}
+</style>
+</head>
+<body>
+<script>
+function c(text){
+  let type = "text/plain";
+  let blob = new Blob([text], { type });
+  let data = [new ClipboardItem({ [type]: blob })];
+  navigator.clipboard.write(data)
+}
+</script>
+<ol>
+{{range .}}
+<li><a href="{{.Href}}" class="data" target="_blank" id="{{.Id}}">{{.Title}}</a> <a href="javascript:void(0)" onclick="c('{{.Title}}')">复制</a></li>
+{{end}}
+</ol>
+</body>
+</html>

program/spider/cli/downloader/batch_download.go

@@ -0,0 +1,345 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/cespare/xxhash/v2"
+	"golang.org/x/net/proxy"
+)
+
+type (
+	//Attach 附件
+	Attach struct {
+		Department string `json:"department"`
+		Href       string `json:"attach_href"`
+		Title      string `json:"info_title"`
+		Raw        string
+	}
+	Attaches []*Attach
+)
+
+var (
+	df             = flag.String("df", "./data.dat", "datafile 数据文件")
+	save2dir       = flag.String("d", "./attaches", "save to dir 保存到目录")
+	areadir        = flag.String("ad", "河南/郑州市/政府", "地域目录")
+	defaultYear    = flag.String("y", "", "year 默认年份，注意：只有整个下载清单是同一个年份才可以指定")
+	threads        = flag.Int("ts", 4, "threads 线程数")
+	sleep          = flag.Int64("s", 0, "sleep 下载间歇")
+	filter         = flag.String("ft", "true", "filter title 是否过滤标题，标题必须包含年份、预算或者预决算")
+	defaultExt     = flag.String("de", "pdf", "default ext默认扩展名")
+	headFile       = flag.String("hf", "", "headfile 下载请求头")
+	logFile        = flag.String("lf", "./error.dat", "log file日志文件")
+	proxyStr       = flag.String("p", "", "proxy 代理，使用代理时，只能开一个线程")
+	connectTimeout = flag.Int64("cto", 30, "超时设定")
+	defaultYearInt int
+	fileIndex      = map[string]int{}
+	allowFileExt   = map[string]bool{
+		"doc":  true,
+		"docx": true,
+		"xls":  true,
+		"xlsx": true,
+		"pdf":  true,
+		"zip":  true,
+		"rar":  true,
+		"txt":  true,
+		"png":  true,
+		"jpg":  true,
+		"jpeg": true,
+		"ppt":  true,
+		"pptx": true,
+		"wps":  true,
+	}
+	reg, _        = regexp.Compile("(\\d{4})")
+	reg4Title1, _ = regexp.Compile("(202\\d+)年度?(.*?)((部门)*(预|决)算)")
+	reg4Title2, _ = regexp.Compile("(.*?)(202\\d+)年度?((部门)*(预|决)算)")
+	downloadLog   *SpiderLog
+	header        = map[string]string{}
+	fnLock        = new(sync.RWMutex)
+	proxyUri      []string
+)
+
+// init
+func init() {
+	flag.Parse()
+	var err error
+	defaultYearInt, _ = strconv.Atoi(*defaultYear)
+	//TODO 目录检查并创建目录 3年的
+	for _, year := range []int{2022, 2023, 2024} {
+		path := fmt.Sprintf("%s/%d年/%s", *save2dir, year, *areadir)
+		if _, err = os.Stat(path); err != nil {
+			os.MkdirAll(path, os.ModeAppend|os.ModePerm)
+		}
+	}
+	if *headFile != "" {
+		bs, err := ioutil.ReadFile(*headFile)
+		if err == nil {
+			content := string(bs)
+			for _, l := range strings.Split(content, "\n") {
+				if strings.HasPrefix(l, "#") {
+					continue
+				}
+				pos := strings.Index(l, ":")
+				if pos > 0 {
+					key := l[:pos]
+					value := l[pos+1:]
+					header[key] = value
+				}
+			}
+		}
+	}
+	log.Println(header)
+	if *logFile != "" {
+		downloadLog, err = NewSpiderLog(*logFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+	proxyUri = strings.Split(*proxyStr, ";")
+}
+
+// id
+func id(text string) uint64 {
+	has := xxhash.New()
+	has.WriteString(text)
+	return has.Sum64()
+}
+
+// changeIp
+func changeIp(client *http.Client) {
+	resp, err := client.Get("http://pleasechangemyip.com")
+	if err == nil {
+		ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+	}
+	//time.Sleep(1 * time.Second)
+}
+
+// httpClient
+func httpClient(index int) *http.Client {
+	if *proxyStr != "" {
+		pos := index % len(proxyUri)
+		dialer, err := proxy.SOCKS5("tcp", proxyUri[pos], nil, proxy.Direct)
+		if err != nil {
+			log.Println(err.Error())
+		}
+		// setup a http client
+		httpTransport := &http.Transport{}
+		httpTransport.Dial = dialer.Dial
+		httpClient := &http.Client{Transport: httpTransport, Timeout: time.Duration(*connectTimeout) * time.Second}
+		changeIp(httpClient)
+		return httpClient
+	} else {
+		return &http.Client{Timeout: time.Duration(*connectTimeout) * time.Second}
+	}
+}
+
+// readDataFile
+func readDataFile() Attaches {
+	ret := make(Attaches, 0, 0)
+	bs, err := ioutil.ReadFile(*df)
+	if err != nil {
+		log.Fatal(err)
+	}
+	content := string(bs)
+	for _, s := range strings.Split(content, "\n") {
+		if len(s) == 0 {
+			continue
+		}
+		var data string = s
+		if strings.HasPrefix(s, ",") {
+			data = s[1:]
+		}
+		var attach = new(Attach)
+		err = json.Unmarshal([]byte(data), attach)
+		if err == nil {
+			ret = append(ret, attach)
+			attach.Raw = s
+		}
+	}
+	return ret
+}
+
+// getExt 取得后缀，扩展名
+func getExt(text string) string {
+	if strings.Contains(text, ".") {
+		tmp := strings.Split(text, ".")
+		return tmp[len(tmp)-1]
+	} else {
+		return ""
+	}
+}
+
+// 分解title
+func extractTitle(organ, title, href string) (year int, department, ext string) {
+	if v := strings.ToLower(getExt(title)); allowFileExt[v] {
+		ext = v
+	} else if v := getExt(href); allowFileExt[v] {
+		ext = v
+	}
+	if ext == "" {
+		ext = *defaultExt
+	}
+
+	if *filter == "true" {
+		check := strings.Contains(organ, "预算") || strings.Contains(organ, "预决算")
+		if !check && !strings.Contains(title, "预算") && !strings.Contains(title, "预决算") {
+			log.Print("失败(文件不包含预算｜决算)", organ, title)
+			return
+		}
+	}
+
+	var matchObj []string
+	if reg.MatchString(title) {
+		matchObj = reg.FindAllString(title, -1)
+	} else if reg.MatchString(organ) {
+		matchObj = reg.FindAllString(organ, -1)
+	}
+	//年排序
+	if matchObj != nil && len(matchObj) > 0 {
+		yearArr := make([]int, len(matchObj))
+		for i, v := range matchObj {
+			yearArr[i], _ = strconv.Atoi(v)
+		}
+		sort.Slice(yearArr, func(i, j int) bool { return yearArr[i] > yearArr[j] })
+		year = yearArr[0]
+	}
+
+	if department == "" {
+		if len(organ) > len(title) {
+			department = organ
+		} else {
+			department = title
+		}
+	}
+	return
+}
+
+// 拼装文件名
+func createFileName(organ, title, href string) (year int, filename string) {
+	fnLock.Lock()
+	defer fnLock.Unlock()
+
+	year, department, ext := extractTitle(organ, title, href)
+
+	if year < 2022 || year > 2024 {
+		log.Println("无法生成有效的文件名", organ, title, href)
+		return 0, ""
+	}
+	//修正文件名，去掉特殊符号
+	department = strings.Map(func(r rune) rune {
+		switch r {
+		case '\n', '.', '-', '?', '=', '&', '*', '#', '$', '%', '^', '(', ')', '|':
+			return -1
+		default:
+			return r
+		}
+	}, department)
+	key := fmt.Sprintf("%d_%s", year, department)
+	if v, ok := fileIndex[key]; ok {
+		fileIndex[key] = v + 1
+	} else {
+		fileIndex[key] = 1
+	}
+	filename = fmt.Sprintf("%d_%s_%d.%s",
+		year, department, fileIndex[key], ext)
+	return year, filename
+}
+
+// download
+func download(attach *Attach, index int) {
+	//生成本地文件
+	year, newFileName := createFileName(attach.Department, attach.Title, attach.Href)
+	if newFileName == "" {
+		log.Println("失败(无法生成文件名)", attach.Title, attach.Href)
+		downloadLog.Log(attach)
+
+		return
+	}
+	log.Println("dowload and save to ", attach.Title, attach.Department, newFileName)
+
+	//TODO 下载文件
+	req, err := http.NewRequest("GET", attach.Href, nil)
+	if err != nil {
+		log.Println("失败(请求头)", attach.Title, attach.Href)
+		downloadLog.Log(attach)
+		return
+	}
+	for k, v := range header {
+		req.Header.Set(k, v)
+	}
+	client := httpClient(index)
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Println("失败(网络问题)", attach.Title, attach.Href)
+		downloadLog.Log(attach)
+
+		return
+	}
+	buf := bytes.NewBuffer(nil)
+	_, err = io.Copy(buf, resp.Body)
+	if err != nil {
+		log.Println("失败(网络问题)", attach.Title, attach.Href)
+		downloadLog.Log(attach)
+
+		return
+	}
+	resp.Body.Close()
+	//TODO 保存到目录
+	outputPaht := fmt.Sprintf("%s/%d年/%s/%s", *save2dir, year, *areadir, newFileName)
+	//log.Println(outputPaht)
+	err = os.WriteFile(outputPaht, buf.Bytes(), os.ModeAppend|os.ModePerm)
+	if err != nil {
+		log.Println("失败(写入文件)", attach.Title, attach.Href)
+		downloadLog.Log(attach)
+		return
+	}
+	if *sleep > 0 {
+		time.Sleep(time.Duration(*sleep) * time.Second)
+	}
+}
+
+// main
+func main() {
+	checkHrefRepeat := map[uint64]bool{}
+	lock := make(chan bool, *threads)
+	wg := new(sync.WaitGroup)
+	fn := func(attach *Attach, index int) {
+		defer func() {
+			<-lock
+			wg.Done()
+		}()
+		download(attach, index)
+	}
+	attches := readDataFile()
+	attchesLen := len(attches)
+	for index, attach := range attches {
+		_id := id(attach.Href)
+		if _, ok := checkHrefRepeat[_id]; ok {
+			log.Println("失败(URL地址重复)", attach.Title, attach.Href)
+			continue
+		}
+		checkHrefRepeat[_id] = true
+		lock <- true
+		wg.Add(1)
+		go fn(attach, index)
+		log.Printf("当前下载进度%.2f%%\n", float32(index)/float32(attchesLen)*100)
+	}
+	wg.Wait()
+	log.Println("下载完成，请关闭本窗口")
+	close(lock)
+	time.Sleep(20 * time.Second)
+}

program/spider/cli/downloader/log.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"os"
+)
+
+type (
+	SpiderLog struct {
+		F *os.File
+	}
+)
+
+var (
+	sl *SpiderLog
+)
+
+// NewSpiderLog
+func NewSpiderLog(sf string) (*SpiderLog, error) {
+	fo, err := os.Create(sf)
+	if err != nil {
+		return nil, err
+	}
+	return &SpiderLog{
+		fo,
+	}, nil
+}
+
+// Close
+func (sl *SpiderLog) Close() {
+	sl.F.Close()
+}
+
+// Log
+func (sl *SpiderLog) Log(attach *Attach) {
+	sl.F.WriteString(attach.Raw)
+	sl.F.WriteString("\n")
+}

program/spider/cli/rnext/README.md

@@ -0,0 +1 @@
+重新修改扩展名

program/spider/cli/rnext/main.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gabriel-vasile/mimetype"
+)
+
+// FileSignature 定义了文件签名的结构
+type FileSignature struct {
+	Signature []byte
+	Extension string
+}
+
+var (
+	dir = flag.String("dir", "/Users/taozhang/Downloads/docs2", "要处理的文件目录")
+)
+
+// detectFileType 通过文件的签名来识别文件类型
+func detectFileType(filePath string) (string, string, error) {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return "", "", err
+	}
+	defer f.Close()
+	// 读取文件的前16字节，这应该足以检测大多数文件的类型
+	signature := make([]byte, 64)
+	if _, err := io.ReadFull(f, signature); err != nil {
+		return "", "", err
+	}
+	// 查找匹配的签名
+	mtype := mimetype.Detect(signature)
+	// 如果没有匹配的签名，返回未知类型
+	extension := mtype.Extension()
+	mime := mtype.String()
+	if extension == "" {
+		return "", "", errors.New("unknow extension")
+	} else {
+		return extension, mime, nil
+	}
+}
+
+// init
+func init() {
+	flag.Parse()
+}
+
+// main
+func main() {
+	fs, err := ioutil.ReadDir(*dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, f := range fs {
+		ext := strings.ToLower(filepath.Ext(f.Name()))
+		newExt, mime, err := detectFileType(filepath.Join(*dir, f.Name()))
+		if err == nil && ext != newExt {
+			newFileName := f.Name()[:len(f.Name())-len(ext)] + newExt
+			log.Println(f.Name(), mime, ext, newExt, newFileName)
+			//os.Rename(filepath.Join(*dir, f.Name()), filepath.Join(*dir, newFileName))
+		}
+	}
+}

program/spider/cli/test/main.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"io/ioutil"
+	"strings"
+)
+
+var (
+	file = "/Users/taozhang/Downloads/jchdp/bin/target.txt"
+)
+
+func main() {
+	bs, _ := ioutil.ReadFile(file)
+	sb := new(strings.Builder)
+	for _, l := range strings.Split(string(bs), "\n") {
+		var newFileName string
+		if strings.Contains(l, "2022年") {
+			newFileName = "2022" + l[4:]
+		} else if strings.Contains(l, "2023年") {
+			newFileName = "2023" + l[4:]
+		} else {
+			newFileName = l
+		}
+		sb.WriteString(newFileName + "\n")
+	}
+	ioutil.WriteFile(file, []byte(sb.String()), 0777)
+}

program/spider/cli/uploader/main.go

@@ -0,0 +1,89 @@
+/*
+*
+webdav上传
+*/
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strconv"
+
+	"github.com/studio-b12/gowebdav"
+)
+
+var (
+	davurl            = flag.String("dav", "https://cloudreve.jydev.jianyu360.com/dav", "dav地址")
+	user              = flag.String("user", "jichunling@topnet.net.cn", "用户")
+	password          = flag.String("password", "imq6aZfwvWdu9XRsd368uywaalD5Jgwf", "密码")
+	localdir          = flag.String("local", "./attaches", "本地目录")
+	remotedir         = flag.String("remote", "/%s/北京/东城区/政府", "远程目录")
+	limitMiniFileSize = flag.Int64("minisize", 1024, "最小文件大小,不小于1Kb")
+	client            *gowebdav.Client
+)
+
+// init
+func init() {
+	flag.Parse()
+	client = gowebdav.NewAuthClient(*davurl, gowebdav.NewAutoAuth(*user, *password))
+	err := client.Connect()
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+// main
+func main() {
+	//TODO 遍历目录
+	years := []string{"2022", "2023", "2024"}
+	for _, y := range years {
+		//建立远程目录
+		rdir := fmt.Sprintf(*remotedir, y+"年")
+		log.Println("上传到远端目录:", rdir)
+		if _, err := client.Stat(rdir); err != nil {
+			if err = client.MkdirAll(rdir, 0644); err != nil {
+				log.Fatal(err)
+			}
+		}
+		//遍历目录
+		fs, err := ioutil.ReadDir(*localdir)
+		if err != nil {
+			log.Fatal(err)
+		}
+		for _, f := range fs {
+			prefix := f.Name()[:4]
+			if _, err := strconv.Atoi(prefix); err == nil &&
+				prefix == y {
+				log.Println(f.Name())
+				//上传文件
+				localFilePath := fmt.Sprintf("%s/%s", *localdir, f.Name())
+				remoteFilePath := fmt.Sprintf("%s/%s", rdir, f.Name())
+				//TODO 检查文件大小
+				if fi, err := os.Stat(localFilePath); err == nil && fi.Size() <= *limitMiniFileSize {
+					log.Println("\t 文件太小：", fi.Size())
+					os.Remove(localFilePath)
+				} else if err != nil {
+					log.Println("\t error： ", err.Error())
+					os.Remove(localFilePath)
+					continue
+				}
+
+				fi, err := os.Open(localFilePath)
+				if err != nil {
+					log.Println(err.Error())
+					continue
+				}
+				if err = client.WriteStream(remoteFilePath, fi, 0644); err != nil {
+					log.Println(err.Error())
+					continue
+				} else {
+					os.Remove(localFilePath)
+				}
+			}
+		}
+	}
+	log.Println("所有文件上传完毕")
+}

program/spider/cli/webserver/README.md

@@ -0,0 +1,15 @@
+# 导出证书
+# 替换为你实际的证书文件路径
+openssl x509 -in certificate.pem -outform der -out certificate.der
+
+# 打开钥匙链
+security unlock-keychain -p "your_password" ~/Library/Keychains/login.keychain
+
+# 导入证书
+security import certificate.der -k ~/Library/Keychains/login.keychain -T /usr/bin/codesign -x -A -t "legacy"
+
+# 信任证书
+security add-trusted-cert -k ~/Library/Keychains/login.keychain certificate.der
+
+# 重新锁定钥匙链
+security lock-keychain ~/Library/Keychains/login.keychain

program/spider/cli/webserver/client.crt

@@ -0,0 +1,124 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 1 (0x1)
+    Signature Algorithm: sha256WithRSAEncryption
+        Issuer: C=cn, ST=ha, L=zz, O=topnet, OU=software, CN=127.0.0.1/emailAddress=394922814@qq.com
+        Validity
+            Not Before: Apr  5 08:00:00 2024 GMT
+            Not After : Apr  5 08:00:00 2025 GMT
+        Subject: C=cn, ST=ha, O=topnet, OU=software, CN=127.0.0.1/emailAddress=394922814@qq.com
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                Public-Key: (4096 bit)
+                Modulus:
+                    00:d4:01:f5:29:b3:c8:a8:4e:cf:3a:9f:4a:bd:8a:
+                    af:94:89:87:34:dd:64:03:d5:76:5e:6e:1a:da:6f:
+                    94:a9:6f:96:0b:2c:5e:4e:2f:5a:94:34:9b:0a:f1:
+                    dc:a0:59:0b:23:6e:ab:e1:88:15:4d:a4:83:fe:ba:
+                    31:23:f3:4a:ae:2b:a2:dc:50:04:64:13:19:7d:44:
+                    d3:24:8e:2e:de:6e:6e:e6:34:4b:84:01:3f:5f:c6:
+                    5a:82:e3:03:b6:eb:28:6d:91:8d:92:62:84:59:ee:
+                    cd:df:b1:6c:bd:46:24:4a:ad:ea:87:00:72:54:b4:
+                    07:86:2f:e8:d9:75:d8:07:f8:64:15:25:67:d5:93:
+                    42:eb:0d:e0:4b:4b:de:69:1e:03:72:30:ed:49:63:
+                    97:b4:f6:1b:83:1c:ad:6f:e3:ff:f9:21:c0:c1:f3:
+                    61:aa:be:ed:48:fd:61:e2:6a:4e:4b:8c:95:6d:ab:
+                    93:92:bf:df:b1:2c:56:57:80:45:ee:8f:99:2e:ef:
+                    4c:34:1c:22:5d:87:0b:21:b9:d2:9a:c9:32:8a:20:
+                    aa:93:9b:11:2d:74:2e:78:20:43:19:ec:0c:39:5d:
+                    9d:b1:d2:34:89:14:66:01:43:e9:0e:7a:28:03:d4:
+                    cd:13:45:11:23:41:16:75:db:42:c0:4a:3e:96:c5:
+                    63:1f:7e:28:34:a6:4d:c5:40:90:20:da:e7:2c:7b:
+                    62:49:18:de:48:84:82:64:f9:e4:63:d0:3f:11:93:
+                    bb:81:50:21:de:21:d5:0e:06:80:e0:77:c0:1e:8e:
+                    24:22:5b:a0:dc:56:3f:39:31:51:73:72:6b:0e:bb:
+                    45:98:34:c6:76:81:05:31:72:4c:c0:69:d6:a4:f6:
+                    ac:be:6c:f7:ae:be:2d:2c:bb:68:03:fb:12:1c:14:
+                    32:39:dc:3b:05:83:5e:5d:91:f7:50:06:12:bc:61:
+                    a1:0d:e0:60:d5:f3:ce:f8:35:73:f1:c3:c3:f6:1b:
+                    f5:e6:f4:d9:a3:82:50:ea:9f:dd:c1:82:49:9f:a6:
+                    cb:6f:f2:be:a2:08:31:39:43:18:69:3f:39:12:b2:
+                    ec:cd:00:0c:56:67:e3:d1:1f:31:92:bb:30:b6:b5:
+                    d5:9a:42:64:f8:da:a9:c3:6a:f4:cc:a7:45:96:3f:
+                    a9:54:da:86:51:ef:9a:00:e7:6e:b5:ed:d8:79:44:
+                    35:c7:79:f2:30:42:7a:3e:bc:82:ee:0f:b8:70:5e:
+                    c9:4f:7b:9d:65:65:20:3f:b2:70:72:97:20:0e:bf:
+                    a0:90:dd:8d:bc:09:4f:c7:55:19:26:ba:81:e8:c6:
+                    40:e1:e7:ea:52:4c:2d:9a:cc:05:90:d9:61:df:d9:
+                    16:a9:d7
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            Netscape Comment: 
+                OpenSSL Generated Certificate
+            X509v3 Subject Key Identifier: 
+                AE:B4:CE:0D:4E:DE:1B:CD:FC:AF:25:2E:30:7D:EC:EF:93:D8:A6:4F
+            X509v3 Authority Key Identifier: 
+                keyid:22:8B:24:79:F7:16:D5:B7:EA:5E:4C:F8:B1:F5:0E:F8:F4:24:67:C1
+
+    Signature Algorithm: sha256WithRSAEncryption
+         52:8a:12:88:19:da:8c:4e:76:f5:da:fb:27:92:f5:15:d6:1a:
+         ff:8b:40:07:c8:6e:95:9c:61:38:5a:4b:6f:db:9f:82:d6:89:
+         e0:35:54:72:75:3c:af:77:85:8d:e9:51:9d:84:4d:e7:44:c8:
+         16:60:91:1c:5e:44:67:ba:b0:58:51:f0:6f:07:e7:2f:9d:ff:
+         86:62:b9:6e:ba:2f:4d:ae:de:62:4a:25:2f:da:42:ba:76:68:
+         52:92:02:a4:24:5f:33:63:95:0c:65:c5:4e:5c:2f:d7:da:24:
+         57:fe:9b:b8:ae:29:89:3e:74:db:56:61:60:9a:2b:96:e1:6a:
+         17:7e:72:40:09:8a:87:7c:60:54:b7:8a:ef:e6:1c:4b:85:4e:
+         7e:bf:c2:dc:53:64:8e:08:c7:36:47:a4:d9:c0:4f:42:20:0d:
+         55:f6:59:67:19:79:1c:a0:ff:10:a1:a0:16:6d:ca:eb:e1:85:
+         f2:b3:d6:99:2c:1e:e5:0b:a0:9c:fd:b3:55:63:eb:ce:61:7c:
+         ea:09:f1:fd:0b:fd:01:b3:bc:b2:98:5a:12:6d:c8:dd:cb:d5:
+         3f:9f:7f:03:04:f5:81:2c:63:05:fc:6e:d1:84:ff:cd:de:bf:
+         b8:3c:d8:88:11:62:34:d4:b3:fb:cb:b9:ec:b5:1e:ba:bd:5d:
+         d3:3d:c8:cc:ef:31:4b:ee:ee:6f:9f:d3:06:31:7c:d4:5c:25:
+         e0:0d:70:9b:83:61:2d:49:ee:27:78:89:5e:d4:47:67:2e:78:
+         44:d7:9b:14:f2:1f:7f:39:ce:05:cc:0b:79:6c:e8:b9:46:48:
+         44:9f:68:dc:00:b0:cf:4e:70:ff:a0:ce:19:0e:8a:db:09:e2:
+         05:d3:ea:6b:b1:b5:64:99:fa:b9:35:de:6c:e1:af:c9:f0:72:
+         54:4d:d9:06:dd:c2:84:e1:e7:9b:fa:33:00:5f:75:26:25:5e:
+         46:a2:a3:d4:95:7e:62:d8:7a:03:2b:7c:1b:6f:d5:e0:39:0d:
+         3a:71:72:df:b3:47:f1:f0:c9:28:bb:38:2f:14:99:10:bc:86:
+         d0:38:9a:d6:85:da:5f:31:b5:7c:61:aa:5b:21:e2:ab:d0:19:
+         a4:a8:a5:ab:17:61:f7:5e:61:9c:3b:a0:da:1b:da:e6:33:ad:
+         89:9d:29:08:b2:d8:45:f7:b3:50:f5:0c:b1:54:f1:2a:eb:56:
+         3d:14:ed:27:c0:eb:da:03:8f:1d:af:2c:33:8f:c3:b4:f6:3b:
+         a1:8d:bc:7e:74:27:b5:56:02:e6:63:b9:dd:87:67:eb:64:aa:
+         14:70:c6:80:ed:21:37:56:43:fb:b0:92:af:98:61:cb:e6:fa:
+         f6:1a:54:48:6a:cb:30:db
+-----BEGIN CERTIFICATE-----
+MIIF6jCCA9KgAwIBAgIBATANBgkqhkiG9w0BAQsFADCBgDELMAkGA1UEBhMCY24x
+CzAJBgNVBAgMAmhhMQswCQYDVQQHDAJ6ejEPMA0GA1UECgwGdG9wbmV0MREwDwYD
+VQQLDAhzb2Z0d2FyZTESMBAGA1UEAwwJMTI3LjAuMC4xMR8wHQYJKoZIhvcNAQkB
+FhAzOTQ5MjI4MTRAcXEuY29tMB4XDTI0MDQwNTA4MDAwMFoXDTI1MDQwNTA4MDAw
+MFowczELMAkGA1UEBhMCY24xCzAJBgNVBAgMAmhhMQ8wDQYDVQQKDAZ0b3BuZXQx
+ETAPBgNVBAsMCHNvZnR3YXJlMRIwEAYDVQQDDAkxMjcuMC4wLjExHzAdBgkqhkiG
+9w0BCQEWEDM5NDkyMjgxNEBxcS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw
+ggIKAoICAQDUAfUps8ioTs86n0q9iq+UiYc03WQD1XZebhrab5Spb5YLLF5OL1qU
+NJsK8dygWQsjbqvhiBVNpIP+ujEj80quK6LcUARkExl9RNMkji7ebm7mNEuEAT9f
+xlqC4wO26yhtkY2SYoRZ7s3fsWy9RiRKreqHAHJUtAeGL+jZddgH+GQVJWfVk0Lr
+DeBLS95pHgNyMO1JY5e09huDHK1v4//5IcDB82Gqvu1I/WHiak5LjJVtq5OSv9+x
+LFZXgEXuj5ku70w0HCJdhwshudKayTKKIKqTmxEtdC54IEMZ7Aw5XZ2x0jSJFGYB
+Q+kOeigD1M0TRREjQRZ120LASj6WxWMffig0pk3FQJAg2ucse2JJGN5IhIJk+eRj
+0D8Rk7uBUCHeIdUOBoDgd8AejiQiW6DcVj85MVFzcmsOu0WYNMZ2gQUxckzAadak
+9qy+bPeuvi0su2gD+xIcFDI53DsFg15dkfdQBhK8YaEN4GDV8874NXPxw8P2G/Xm
+9NmjglDqn93Bgkmfpstv8r6iCDE5QxhpPzkSsuzNAAxWZ+PRHzGSuzC2tdWaQmT4
+2qnDavTMp0WWP6lU2oZR75oA52617dh5RDXHefIwQno+vILuD7hwXslPe51lZSA/
+snBylyAOv6CQ3Y28CU/HVRkmuoHoxkDh5+pSTC2azAWQ2WHf2Rap1wIDAQABo3sw
+eTAJBgNVHRMEAjAAMCwGCWCGSAGG+EIBDQQfFh1PcGVuU1NMIEdlbmVyYXRlZCBD
+ZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUrrTODU7eG838ryUuMH3s75PYpk8wHwYDVR0j
+BBgwFoAUIoskefcW1bfqXkz4sfUO+PQkZ8EwDQYJKoZIhvcNAQELBQADggIBAFKK
+EogZ2oxOdvXa+yeS9RXWGv+LQAfIbpWcYThaS2/bn4LWieA1VHJ1PK93hY3pUZ2E
+TedEyBZgkRxeRGe6sFhR8G8H5y+d/4ZiuW66L02u3mJKJS/aQrp2aFKSAqQkXzNj
+lQxlxU5cL9faJFf+m7iuKYk+dNtWYWCaK5bhahd+ckAJiod8YFS3iu/mHEuFTn6/
+wtxTZI4IxzZHpNnAT0IgDVX2WWcZeRyg/xChoBZtyuvhhfKz1pksHuULoJz9s1Vj
+685hfOoJ8f0L/QGzvLKYWhJtyN3L1T+ffwME9YEsYwX8btGE/83ev7g82IgRYjTU
+s/vLuey1Hrq9XdM9yMzvMUvu7m+f0wYxfNRcJeANcJuDYS1J7id4iV7UR2cueETX
+mxTyH385zgXMC3ls6LlGSESfaNwAsM9OcP+gzhkOitsJ4gXT6muxtWSZ+rk13mzh
+r8nwclRN2QbdwoTh55v6MwBfdSYlXkaio9SVfmLYegMrfBtv1eA5DTpxct+zR/Hw
+ySi7OC8UmRC8htA4mtaF2l8xtXxhqlsh4qvQGaSopasXYfdeYZw7oNob2uYzrYmd
+KQiy2EX3s1D1DLFU8SrrVj0U7SfA69oDjx2vLDOPw7T2O6GNvH50J7VWAuZjud2H
+Z+tkqhRwxoDtITdWQ/uwkq+YYcvm+vYaVEhqyzDb
+-----END CERTIFICATE-----

program/spider/cli/webserver/client.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKgIBAAKCAgEA1AH1KbPIqE7POp9KvYqvlImHNN1kA9V2Xm4a2m+UqW+WCyxe
+Ti9alDSbCvHcoFkLI26r4YgVTaSD/roxI/NKriui3FAEZBMZfUTTJI4u3m5u5jRL
+hAE/X8ZaguMDtusobZGNkmKEWe7N37FsvUYkSq3qhwByVLQHhi/o2XXYB/hkFSVn
+1ZNC6w3gS0veaR4DcjDtSWOXtPYbgxytb+P/+SHAwfNhqr7tSP1h4mpOS4yVbauT
+kr/fsSxWV4BF7o+ZLu9MNBwiXYcLIbnSmskyiiCqk5sRLXQueCBDGewMOV2dsdI0
+iRRmAUPpDnooA9TNE0URI0EWddtCwEo+lsVjH34oNKZNxUCQINrnLHtiSRjeSISC
+ZPnkY9A/EZO7gVAh3iHVDgaA4HfAHo4kIlug3FY/OTFRc3JrDrtFmDTGdoEFMXJM
+wGnWpPasvmz3rr4tLLtoA/sSHBQyOdw7BYNeXZH3UAYSvGGhDeBg1fPO+DVz8cPD
+9hv15vTZo4JQ6p/dwYJJn6bLb/K+oggxOUMYaT85ErLszQAMVmfj0R8xkrswtrXV
+mkJk+Nqpw2r0zKdFlj+pVNqGUe+aAOdute3YeUQ1x3nyMEJ6PryC7g+4cF7JT3ud
+ZWUgP7JwcpcgDr+gkN2NvAlPx1UZJrqB6MZA4efqUkwtmswFkNlh39kWqdcCAwEA
+AQKCAgEAowfiXLBJJMx95UmrcvhcLvBXTdrV13DmdpVbBxD5FBagHnV/C8a0c/XE
+KHV3QMNYomWvvV4la7CQZGkP8uJbUnZx5dOOnNpnJoKvj5PbDOeFVnduftNqxZ7F
+6ZLjP120mBVu9dvWhUXzgQpyYVa3/8mf6TL5uJM8mf7ro5ZQi0iCxhQetglpU1I3
+Yg+0t8cBxWVEg8QXXGxJGdkKsll1dUJ0dXFQcLojxJY9GJzBGq3AFzby482zGg7L
+UrxsiphYptQUGJpt/tgjQksgXjFMMRYPtjRv0DFgrrW4hA2WFOMdqinIhoVuL5jm
+dmCPtuqEzUW2eRLLT9RNj4tIeC7QGnheluBUhphCvV3q/v1Vl5IyCclTpJaO0cX8
+9VNbqYYdscAOWuMnbJCdXCy31nTnC5joGx9Xm5xL1qQRuwqtBBVBW7KuPL73x/v6
+KJgTbDHArPqCp9CIWTDXx9uy2IGGshLQ6CK6RrhffeDgX1voAsHiIpkFIHqDNyjP
++m76CFIqPXGetI5mFUeSgOUB+LzPHRFufiDug+PraOf0Qh3qO+chHSMi6ecxPmT+
+ixgB2HlzdblTAZ/HpzAYXrLMp0y7Bc59F9PQrCqaW7WRh9GpKA7IjxT3ocKpIDhY
+79LwZDW7Vormz1As1nyeQCkJW3iXZT9CymIpE6qqoD8/SfYIpiECggEBAOk382Kt
+FZAm0hJAI3kNHGabFtruwOHciRc5lnQLofLYPc5EEmgIN+/FkXYWtDeqft/3lnaD
+qucURaGFjGowPuQ+kVXaFI3DD+EH4TC11hDR8UmBux5KE5yFEdl7o4MNZB6tkya/
+mXRF4Xbul28kp+zsduib1lyw3RUggFHvGTQvQMzH9chCJLyptJjZX9peTNTT/dy2
+6ycN3z4Mo1W1QQuFWF+LN4Ig8Wl2kCm8wU2bySy4lnGgeHBWpEJDKjSNKmPEa6QO
+NFZWlHgzl063IMhqsRm2h1qY+hzf3cF8KKj0FDaZQZjnatgqShCjDsuLjOhHgPws
+Cll3DT2eULFpS8cCggEBAOi3lwozMxNm98k8G9FD53oO1Tc3SwalX4llU93S9/Iv
+0TGwgz5aLocLYEaRIUA5HAAnVsgEdg5Z/G5cPX0fHt4zIzT89o4dvKLVJ1i2owEo
+A+m9rRVT0RfSvgVirDqAvEiRcU8gSrFNoTty16ibbUNrbCZP0vwr3tgKwuoOi6ka
+JeKg4JENihwE4i5NxHTiJkmnTqgNqECNOa/9F9eN9OTiSI8sJCYPwFDC9Z2C/1m+
+sV7eGLoyTxUhfJSIS4Ez7cLUL+gx9QHxk/er+EA11Lxl34V7NIarxROPXx4gQNyR
+26AgShDEVJOgETn7rlxIyl8BrqYs3Tz1qtbg24/MEXECggEALvDzDQ53g588sy2z
+th68c66NqB1cN1nTQhh2Pv9EZoO91PGAnu3gjGNAxvsE5RttFBlacJpJ4N8EsOEX
+hRwW3Iv3ZsdRUcNMQ+f5HHvNDW+kWD5DcnJIhj2GfTbUj4E3gLFBKBByi6OxIOYf
+FuDS5+4kdOdA+0EiDqmPCm7MrONB8LXis58Uk+C4l/dQwrwuTLwqoa0zoZW6NFwI
+2Cx2XiJFyl1UG94TmT2jALOcWMmdcWHp7g71ju7xeuDX8uHjAVe8rrWpNSOSGKuR
+redQMua+pMVHcyXNWdh5YwNhj7gpBWBNgsA5ki7Y34ZiaPe6sdoKjTc0EhPTV26x
+ITRnawKCAQEAhkK1Hytewkjbl/vXyKm2eFLqbAq4fEMYlToiYo3+gnlX5aUCcwyw
+XzUO/GdqmZTZNLV9ebQpMCp2Wm1ZBEDaHDPAkieHT4KUfBUhuCrWbCpb55SIsQ9E
+SMxK2OyOlMoE1pEioBXvQc+Zv6SICzxwkz78jDXtcZMV4+qBfWua45q0VMWgCc+X
+6IZgv3oaZ5y5njCqToZilatkpsYL5U5/BTYPypoa2OKpf26bKZ6UF4DO7/6OoY24
+1mYVa9QeS5JcD9U/NmTsB40KYw+SD+l+rJrRlD76lN+wuZIzvDu7cAWugflfAcFj
+UjA9AAjWFWPhbWKZEkI7+Zl6ULI1ugZv4QKCAQEAxWQ5RHQK+IE4TrK7oz4v0yuh
+HXG6sG1Jmkb5ebG4R7HSTurMGUhsSQm2QqtkVC/vXpQHEE0nTLZgNYgE6hPKHjAo
+txsl6k1XPRkD6JVDSRieKhIuqe4n8t4/4F4wz4FA5eVJqo/TT6qcSxCpZa8V/uei
+Y8o2S4e/q9L2o7YsJoTggOJ0ZOXgVJQY4ILL3tC0nBOJg2WceqzsArF4sRMfi98y
+sCgP0qycWUUSNC2s2JC4uhE7uWQfQ2oBpLjaD9fX+W3TQXH3c+clb6bsKpd51zvW
+jrSXbKs4relI3Aw8YLfx9l4/YquVbL2z5uDrolNLH9t0jy7f0t3hZfJzUuJ+ow==
+-----END RSA PRIVATE KEY-----

program/spider/cli/webserver/main.go

@@ -0,0 +1,59 @@
+// 文件上传服务器，json格式操作
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+)
+
+var (
+	path = flag.String("path", "./", "路径")
+	addr = flag.String("addr", ":20008", "地址")
+)
+
+func init() {
+	flag.Parse()
+}
+
+// uploadFileHandler
+func uploadFileHandler(w http.ResponseWriter, r *http.Request) {
+	log.Println("请求过来了")
+	// 设置CORS头部，允许所有源（*）进行所有方法的请求
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Content-Type", "application/json")
+
+	// 检查是否是POST请求
+	if r.Method != "POST" {
+		http.Error(w, "只接受POST请求", http.StatusBadRequest)
+		return
+	}
+	// 解析上传的文件
+	file, _, err := r.FormFile("file")
+	if err != nil {
+		http.Error(w, "解析文件失败", http.StatusBadRequest)
+		return
+	}
+	defer file.Close()
+	filename := r.FormValue("filename")
+	fo, err := os.Create(filepath.Join(*path, filename))
+	if err != nil {
+		http.Error(w, "创建文件"+filename+"失败", http.StatusBadRequest)
+		return
+	}
+	defer fo.Close()
+	io.Copy(fo, file)
+	// 返回成功响应
+	fmt.Fprint(w, fmt.Sprintf(`{"code":200,"msg":"ok","filename":"%s"}`, filename))
+}
+
+func main() {
+	http.HandleFunc("/upload", uploadFileHandler)
+	if err := http.ListenAndServeTLS(*addr, "./client.crt", "./client.key", nil); err != nil {
+		log.Fatal(err)
+	}
+}

program/spider/config.yaml

@@ -0,0 +1,27 @@
+# 配置文件
+# 是否启用无头浏览器，仅本地浏览器有效，在调试期间禁用，生产时启用
+headless : false
+# 是否渲染加载网页图像，不加在图像可以节省带宽，但是有些页面加载异常
+show_image : false
+
+# 调用浏览器，支持local本地浏览器，remote远程浏览器
+run_model : local
+# 本地浏览器，可以挂接跳板应用，这里写跳板地址，实现无缝IP切换
+# 127.0.0.1:30000
+proxy_address : ""
+# 远程浏览器， 这里填写远程浏览器资源（websocket URL）管理地址
+websocket_url : ""
+
+# 存储方式，可选项： console,mongodb,file,wechatrobot
+storage : "file"
+# mongodb 存储
+mongo_address : ""
+mongo_db : ""
+mongo_colesstion : ""
+# 文件存储
+storage_file : "./data.dat"
+
+# 文件下载目录
+download_path : "./"
+# 企业微信消息提醒
+wechat_robot_url : ""

program/spider/expandaction.go

@@ -0,0 +1,260 @@
+// 浏览器扩展能力
+package main
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/chromedp/cdproto/page"
+
+	"github.com/chromedp/cdproto/fetch"
+	"github.com/chromedp/chromedp"
+
+	"github.com/chromedp/cdproto/browser"
+)
+
+// Screenshot
+func (b *Browser) Screenshot(tabTitle, tabUrl string, timeout int64,
+	selectorType int, selector, save2file string) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+
+	var res []byte
+	var act chromedp.QueryAction
+	switch selectorType {
+	case selector_type_id:
+		act = chromedp.Screenshot(selector, &res, chromedp.ByID)
+	case selector_type_query:
+		act = chromedp.Screenshot(selector, &res, chromedp.ByQuery)
+	case selector_type_search:
+		act = chromedp.Screenshot(selector, &res, chromedp.BySearch)
+	case selector_type_jspath:
+		act = chromedp.Screenshot(selector, &res, chromedp.ByJSPath)
+	default:
+		act = chromedp.Screenshot(selector, &res, chromedp.ByQueryAll)
+	}
+
+	err = chromedp.Run(ctx, act)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(save2file, res, 0777)
+}
+
+// PrintToPDF
+func (b *Browser) PrintToPDF(tabTitle, tabUrl string, timeout int64, save2file string) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+
+	var res []byte
+	var act chromedp.QueryAction = chromedp.ActionFunc(func(ctx context.Context) error {
+		buf, _, err := page.PrintToPDF().
+			WithLandscape(false).
+			WithPaperWidth(16.3).
+			WithPaperHeight(11.69).
+			WithMarginTop(0.1).
+			WithMarginRight(0).
+			WithMarginBottom(0.1).
+			WithMarginLeft(0).
+			WithPrintBackground(true).
+			Do(ctx) // 通过cdp执行PrintToPDF
+		if err != nil {
+			return err
+		}
+		res = buf
+		return nil
+	})
+
+	err = chromedp.Run(ctx, act)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(save2file, res, 0777)
+}
+
+// GetBrowserTabs
+func (b *Browser) GetBrowserTabs(tabTitle, tabUrl string, timeout int64) ([]map[string]interface{}, error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return nil, err
+	}
+	ts, err := chromedp.Targets(ctx)
+	if err != nil {
+		return nil, err
+	}
+	ret := make([]map[string]interface{}, 0, 0)
+	for _, t := range ts {
+		ret = append(ret, map[string]interface{}{
+			"title": t.Title,
+			"url":   t.URL,
+		})
+	}
+	return ret, nil
+}
+
+// DownloadFile 只有在非headless模式下有效，与click方法其实是一致的
+func (b *Browser) DownloadFile(tabTitle, tabUrl string, timeout int64, selector string,
+	selectorType int, filename string, save2dir string) error {
+	defer Catch()
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	var act chromedp.QueryAction
+	switch selectorType {
+	case selector_type_id:
+		act = chromedp.Click(selector, chromedp.ByID)
+	case selector_type_query:
+		act = chromedp.Click(selector, chromedp.ByQuery)
+	case selector_type_search:
+		act = chromedp.Click(selector, chromedp.BySearch)
+	case selector_type_jspath:
+		act = chromedp.Click(selector, chromedp.ByJSPath)
+	default:
+		act = chromedp.Click(selector, chromedp.ByQueryAll)
+	}
+	done := make(chan bool, 1)
+	chromedp.ListenTarget(ctx, func(v interface{}) {
+		switch ev := v.(type) {
+		case *fetch.EventRequestPaused:
+			if ev.ResponseStatusCode == 0 {
+				go func() {
+					if err := chromedp.Run(ctx,
+						fetch.ContinueRequest(ev.RequestID).WithInterceptResponse(true),
+					); err != nil {
+						fmt.Println(err.Error())
+					}
+				}()
+			} else {
+				go func() {
+					fulfill := fetch.FulfillRequest(ev.RequestID, ev.ResponseStatusCode)
+					if ev.ResponseStatusCode == 200 {
+						headers := append(ev.ResponseHeaders, &fetch.HeaderEntry{
+							Name:  "Content-Disposition",
+							Value: fmt.Sprintf("attachment; filename=%s", filename),
+						})
+						fmt.Println("headers:")
+						for k, v := range headers {
+							fmt.Println(k, v.Name, v.Value)
+						}
+
+						fulfill = fulfill.WithResponseHeaders(headers)
+					}
+					if err := chromedp.Run(ctx, fulfill); err != nil {
+						fmt.Println(err.Error())
+					}
+				}()
+			}
+		case *browser.EventDownloadWillBegin:
+			//开始下载文件
+			fmt.Println("start download file:", ev.SuggestedFilename)
+		case *browser.EventDownloadProgress:
+			//下载进度
+			if ev.State == browser.DownloadProgressStateCompleted {
+				done <- true
+			}
+		}
+	})
+
+	err = chromedp.Run(ctx,
+		fetch.Enable().WithPatterns([]*fetch.RequestPattern{
+			{URLPattern: "*/*"},
+		}),
+		browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorAllowAndName).WithDownloadPath(save2dir).WithEventsEnabled(true),
+		act)
+	select {
+	case <-done:
+		return err
+	case <-time.After(60 * time.Second):
+		return err
+	}
+	return err
+}
+
+// GoHistoryBack
+func (b *Browser) GoHistoryBack(tabTitle, tabUrl string, timeout int64) error {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	var act chromedp.QueryAction = chromedp.NavigateBack()
+	return chromedp.Run(ctx,
+		act)
+}
+
+// SendImage2ChatBot
+func SendImage2ChatBot(uri, img, mentioned string) error {
+	rawStr := img[22:]
+	bs, err := base64.StdEncoding.DecodeString(rawStr)
+	if err != nil {
+		return err
+	}
+
+	h := md5.New()
+	h.Write(bs)
+	hash := h.Sum(nil)
+	hashStr := fmt.Sprintf("%x", hash)
+	postBody := fmt.Sprintf(`{
+    "msgtype": "image",
+    "image": {
+        "base64": "%s",
+        "md5": "%s",
+        "mentioned_list":["@%s"]
+    }
+}
+`, rawStr, strings.ToLower(hashStr), mentioned)
+
+	client := new(http.Client)
+	req, err := http.NewRequest("POST", uri,
+		strings.NewReader(postBody))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	bs, _ = ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	return nil
+}
+
+// SendText2ChatBot
+func SendText2ChatBot(uri, text, mentioned string) error {
+
+	postBody := fmt.Sprintf(`{
+    "msgtype": "text",
+    "text": {
+        "content": "%s",
+        "mentioned_list":["@%s"]
+    }
+}
+`, text, mentioned)
+
+	client := new(http.Client)
+	req, err := http.NewRequest("POST", uri,
+		strings.NewReader(postBody))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	_, _ = ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	return nil
+}

program/spider/log.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"encoding/csv"
+	"log"
+	"os"
+)
+
+type (
+	SpiderLog struct {
+		F       *os.File
+		LogFile *csv.Writer
+	}
+)
+
+var (
+	sl *SpiderLog
+)
+
+// NewSpiderLog
+func NewSpiderLog(sf string) (*SpiderLog, error) {
+	fo, err := os.Create(sf)
+	if err != nil {
+		return nil, err
+	}
+	return &SpiderLog{
+		fo,
+		csv.NewWriter(fo),
+	}, nil
+}
+
+// Close
+func (sl *SpiderLog) Close() {
+	sl.LogFile.Flush()
+	sl.F.Close()
+}
+
+// Log
+func (sl *SpiderLog) Log(arg ...string) {
+	err := sl.LogFile.Write(arg)
+	if err != nil {
+		log.Println(err)
+	}
+}

program/spider/main.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"flag"
+	"jycdp"
+	"log"
+
+	"os"
+)
+
+type (
+	Config struct {
+		RunModel               string `yaml:"run_model"`
+		WebsocketURL           string `yaml:"websocket_url"` //仅远程浏览器有用
+		ProxyAddr              string `yaml:"proxy_address"` //仅本地浏览器有用
+		HeadLess               bool   `yaml:"headless"`
+		ShowImage              bool   `yaml:"show_image"`
+		StorageType            string `yaml:"storage"`
+		StorageMongoURI        string `yaml:"mongo_address"`
+		StorageMongoDB         string `yaml:"mongo_db"`
+		StorageMongoCollection string `yaml:"mongo_colesstion"`
+		StorageFile            string `yaml:"storage_file"`
+		WeChatRobotURL         string `yaml:"wechat_robot_url"`
+		DownloadPath           string `yaml:"download_path"`
+	}
+)
+
+var (
+	configFile = flag.String("cf", "./config.yaml", "配置文件")
+	luaFile    = flag.String("sf", "./scripts/demo.lua", "lua文件")
+	dataFile   = flag.String("df", "./data.dat", "数据文件")
+	logFile    = flag.String("lf", "./log.dat", "日志文件")
+	storage    Storage
+	config     *Config = new(Config)
+)
+
+// init
+func init() {
+	flag.Parse()
+	jycdp.LoadConfig(*configFile, config)
+	var err error
+	switch config.StorageType {
+	case "console":
+		storage = NewConsoleStorage()
+	case "mongodb":
+		storage = NewMongoStorage(config.StorageMongoURI,
+			config.StorageMongoDB,
+			config.StorageMongoCollection)
+	case "wechatrobot":
+		storage = NewWeChatMsgStorage(config.WeChatRobotURL)
+	default:
+		var theDataFile = *dataFile
+		if theDataFile == "" {
+			theDataFile = config.StorageFile
+		}
+		fi, err := os.OpenFile(theDataFile, os.O_CREATE|os.O_APPEND|os.O_RDWR|os.O_SYNC|os.O_TRUNC, 0777)
+		if err != nil {
+			panic(err)
+		}
+		storage = NewFileStorage(fi)
+	}
+	if *logFile != "" {
+		sl, err = NewSpiderLog(*logFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+	//log.Println(config)
+}
+
+func main() {
+	bs, err := os.ReadFile(*luaFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	script := string(bs)
+
+	var vm *VM
+	if config.RunModel == "local" {
+		vm = NewLocalVM(config.HeadLess, config.ShowImage, config.ProxyAddr, config.DownloadPath, storage)
+	} else {
+		vm = NewRemoteVM(config.WebsocketURL, storage)
+	}
+	if err := vm.RunScript(script); err != nil {
+		log.Println(err.Error())
+	}
+	//最后执行
+	if sl != nil {
+		sl.Close()
+	}
+}

program/spider/scripts/demo.lua

@@ -0,0 +1,7 @@
+-- 无头浏览器采集教程
+-- 基础
+-- lua基础语法，不变
+
+local href="https://www.baidu.com"
+print(href)
+

program/spider/scripts/inc/fetch.lua

@@ -0,0 +1,69 @@
+-- fetch 远程请求模版
+local fetch_request_tpl = [[
+	var %s=null;
+	fetch("%s",{
+		method: '%s',
+		body: "%s",
+		headers: { %s},
+	})
+	.then(response => {
+		if (!response.ok) {
+			throw new Error('Network response was not ok');
+		}
+		if("%s"=="text"){
+			return response.text(); 
+		}else if("%s"=="json"){
+			return response.json(); 
+		}else{
+		return "我也不知道要返回什么格式";
+		}
+	})
+	.then(data=>{
+		%s = data
+	})
+	.catch(error => {
+		console.error('Error:', error);
+	});
+	""
+]]
+
+--fetch异步请求
+--path:用于选择tab页
+--result_type：结果返回类型 0文本 1列表 2字典
+--method:POST /GET 
+-- url: 请求URL
+--head:请求头
+--param:参数 POST请求下，有值
+--return_type:text/json
+--js_var:结果回写到js_var
+--retry_time:等待次数
+--sleep_timeout:等待间隔时间
+function fetch(path,result_type,method,url,head,param,return_type,js_var,retry_time,sleep_timeout)
+	local head_str = ""
+	for k,v in pairs(head) do
+		head_str=head_str..string.format('"%s":"%s",',k,v)
+	end
+	local fetch_run_js = string.format(fetch_request_tpl,js_var,url,method,param,head_str,return_type,return_type,js_var)
+	--print(fetch_run_js)
+	local ok,ret = browser_executejs(path,1000*10,0,fetch_run_js)
+	wait_jsvar_notnull(path,js_var,retry_time,sleep_timeout)
+	return browser_executejs(path,1000*10,result_type,js_var)
+end
+
+--守候等待js变量不为null
+function wait_jsvar_notnull(path,js_var,retry_time,sleep_timeout)
+	local flag=false
+	local i=1
+	local check_js_var_tpl = 'if(%s){"true"}else{"false"}'
+	local run_js=string.format(check_js_var_tpl,js_var)
+	repeat
+		local ok,ret = browser_executejs(path,1000*2,0,run_js)
+		if ok=="ok" and ret=="true" then
+			flag=true
+		else
+			print("waitting")
+			browser_sleep(sleep_timeout)
+		end
+		i=i+1
+	until flag or i>retry_time
+end

program/spider/scripts/inc/util.lua

@@ -0,0 +1,47 @@
+--公共文件
+--获取table长度
+function table_length(t)
+	if type(t)~="table" then
+		return 0
+	end
+	local len=0
+	for _,_ in pairs(t) do
+		len=len+1
+	end
+	return len
+end
+
+--保存网页为pdf文件
+function save_page_to_pdf(tab_title,tab_href,dir,title)
+	local filename = ""
+	if string.find(title,"2024年")~=nil then
+		filename = "2024_"..title.."__浏览器生成.pdf"
+	elseif string.find(title,"2023年")~=nil then
+		filename = "2023_"..title.."__浏览器生成.pdf"
+	else
+		filename = "2022_"..title.."__浏览器生成.pdf"
+	end
+	print("准备生成网页pdf",filename)
+	local ok2 = browser_print2pdf(tab_href,1000*60*20,dir..filename)
+	print("生成pdf",ok2)
+end
+
+--页面清理元素，只保留中心元素
+local remote_body_el_without_js=[[
+	var hold_el = document.querySelector("%s");
+	function removeAllChildren(parent) {
+	  var children = parent.childNodes;	
+	  for (var i = children.length - 1; i >= 0; i--) {
+		parent.removeChild(children[i]);
+	  }
+	}
+	removeAllChildren(document.body);
+	document.body.appendChild(hold_el);
+]]
+
+--页面清理元素，只保留中心元素
+function clear_page_el(tab_title,tab_href,timeout,css_selector)
+	local js = string.format(remote_body_el_without_js, css_selector)
+	local ok, ret = browser_executejs(tab_title,tab_href,timeout,0, js)
+	return ok,ret
+end

program/spider/scripts/学习教程/基础.lua

@@ -0,0 +1,7 @@
+-- 无头浏览器采集教程
+-- 基础
+-- lua基础语法，不变
+
+local href="https://www.baidu.com"
+print(href)
+

program/spider/scripts/学习教程/浏览器操作.lua

@@ -0,0 +1,11 @@
+-- 无头浏览器采集教程
+-- 基础
+-- 浏览器操作，打开百度
+
+local href="https://www.baidu.com"
+local timeout = 1000*10
+
+browser_navagite(timeout, href)
+browser_sleep(1000*5)
+
+print("结束")

program/spider/scripts/学习教程/浏览器操作2.lua

@@ -0,0 +1,14 @@
+-- 无头浏览器采集教程
+-- 基础
+-- 浏览器操作，打开百度，输入词，并搜索
+
+local href="https://www.baidu.com"
+local timeout = 1000*10
+
+browser_navagite(timeout, href)
+browser_sleep(1000*2)
+
+browser_keysend(timeout,2,"input#kw","lua开发".."\n")
+browser_sleep(1000*5)
+
+print("结束")

program/spider/scripts/学习教程/浏览器操作3.lua

@@ -0,0 +1,16 @@
+-- 无头浏览器采集教程
+-- 基础
+-- 浏览器操作，打开百度，输入词，并搜索 。另外一种写法
+
+local href="https://www.baidu.com"
+local timeout = 1000*10
+
+browser_navagite(timeout, href)
+browser_sleep(1000*2)
+
+browser_keysend(1000*5,2,"input#kw","lua开发")
+browser_sleep(1000*1)
+browser_click(1000*5,0,"su")
+browser_sleep(1000*10)
+
+print("结束")

program/spider/scripts/学习教程/浏览器操作4.lua

@@ -0,0 +1,18 @@
+-- 无头浏览器采集教程
+-- 基础
+-- 浏览器操作，打开百度，输入词，并搜索 。另外一种写法
+
+local href="https://www.baidu.com"
+local timeout = 1000*10
+
+browser_navagite(timeout, href)
+browser_sleep(1000*2)
+
+browser_keysend(1000*5,2,"input#kw","lua开发")
+browser_sleep(1000*1)
+browser_click(1000*5,0,"su")
+browser_sleep(1000*3)
+browser_reset()
+
+
+print("结束")

program/spider/scripts/学习教程/浏览器操作5.lua

@@ -0,0 +1,41 @@
+-- 无头浏览器采集教程
+-- 基础
+-- 浏览器操作，打开百度，输入词，并搜索 。另外一种写法
+
+local href="https://www.baidu.com"
+local timeout = 1000*10
+
+local info_list_js = [[
+	var ns=[];
+	document.querySelectorAll("div.c-container h3 a").forEach((v,i)=>{
+		var title = v.innerText;
+		var href = v.href;
+		ns.push({"title":title,"href":href});
+	});
+	ns
+]]
+
+
+browser_navagite(timeout, href)
+browser_sleep(1000*2)
+
+browser_keysend(1000*5,2,"input#kw","lua开发")
+browser_sleep(1000*1)
+browser_click(1000*5,0,"su")
+browser_sleep(1000*3)
+-- TODO 打开匹配链接
+
+local ok,list = browser_executejs(1000*5,1,info_list_js)
+
+print(ok,#list)
+if ok=="ok" then
+	for k,v  in pairs(list) do
+		print(v.title,v.href)
+		--TODO 需要补充，数据提取逻辑
+		
+	end
+else
+	print("执行js失败",info_list_js)
+end
+
+print("结束")

program/spider/scripts/学习教程/浏览器操作6.lua

@@ -0,0 +1,43 @@
+-- 无头浏览器采集教程
+-- 基础
+-- 浏览器操作，打开百度，输入词，并搜索 。另外一种写法
+
+local href="https://www.baidu.com"
+local timeout = 1000*120
+
+local info_list_js = [[
+	var ns=[];
+	document.querySelectorAll("div.c-container h3 a").forEach((v,i)=>{
+		var title = v.innerText;
+		var href = v.href;
+		ns.push({"title":title,"href":href,"index":i+1});
+	});
+	ns
+]]
+
+
+browser_navagite(timeout, href)
+browser_sleep(1000*2)
+
+browser_keysend(1000*5,2,"input#kw","lua开发")
+browser_sleep(1000*1)
+browser_click(1000*5,0,"su")
+browser_sleep(1000*5)
+-- TODO 打开匹配链接
+local ok,list = browser_executejs(1000*30,1,info_list_js)
+
+print(ok,#list)
+if ok=="ok" then
+	for k,v  in pairs(list) do
+		print(v.title,v.href)
+		--TODO 打开页面，等待3秒，并关闭
+		browser_navagite(true,1000*5, v.href)
+		browser_sleep(1000*3)
+		browser_closetabs(v.href,200)
+		browser_sleep(1000*1)
+	end
+else
+	print("执行js失败",info_list_js)
+end
+
+print("结束")

program/spider/scripts/模版/1层下载模版.lua

@@ -0,0 +1,41 @@
+-- 采集参数设置
+-- 一层级,信息
+
+-- 信息详情页地址
+local href = "https://czt.nmg.gov.cn/yjs/business/page/content?contentId=a422ec07340d08a5919b245f7e4d39f6"
+-- 部门名称，需要填写
+local department = "赤峰市政府2024年度预算"
+
+local timeout = 1000 * 60 * 120
+-- TODO 查找附件
+local find_info_attach_file_js = [[
+     var ns = [];
+       var extensions = ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'zip', 'rar','PDF','DOC','DOCX','XLS','XLSX','ZIP','RAR'];
+       var extensionRegex = new RegExp('\\.(' + extensions.join('|') + ')', 'i');
+       document.querySelectorAll("a").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        var href = v.href
+	if(extensionRegex.test(linkText.toLowerCase())||extensionRegex.test(href.toLowerCase())){
+        	ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+
+ 
+browser_navagite(timeout, href)
+browser_sleep(1000 * 2)
+local ok, list = browser_executejs(timeout, 1, find_info_attach_file_js)
+print('查找详情页附件链接', ok, list, #list)
+if ok == "ok" then
+    for _, v in pairs(list) do
+        -- 存储数据
+        print(v.title, v.href)
+        browser_save("", "", "", "", { ["department"] = department, ["info_title"] = v['title'], ["attach_href"] = v["href"] })
+    end
+else
+	print("抓取失败",ok)
+end
+print("任务完成")
+browser_sleep(1000 * 2)

program/spider/scripts/模版/2层JS翻页模版.lua

@@ -0,0 +1,121 @@
+-- 采集参数设置
+-- 二层级,单位信息列表/信息
+-- 不用拼接列表页地址，直接模拟点击列表页的下一页链接
+-- href_tpl 可以不用理会
+
+--https://www.mas.gov.cn/xxgk/opennessTarget/?branch_id=57a3df762c262ea9a00aae84&column_code=150101&topic_id=&type=bumen
+local href = "https://www.songjiang.gov.cn/Template/dynamic/currency/infoOpenMore.html?categorynum=004003002"
+-- 列表页CSS选择期，可以避免检测垃圾数据，不想定义，直接写a也行
+local info_list_css_selector = "div.ewb-infos td a"
+-- 三级页特征，用于找到tab页，准备在这个tab页执行js，关闭tab页
+local info_url_path_tpl = "contents"
+-- 下一页链接或者按钮的CSS选择期
+local next_page_css_selector = 'var a;document.querySelectorAll("ul.m-pagination-page li a").forEach((v,i)=>{if(v.innerText.indexOf("下一页")>-1){a=v;}});a'
+-- 列表页最大页数
+local max_pages = 7
+
+-- 用JS点击下一页，这个变量将弃用
+local href_tpl = "https://www.bjshy.gov.cn/web/zwgk/czsj/3fb44002-%d.html"
+
+local timeout = 1000 * 60 * 120
+-- TODO 
+-- 检查是否有下一页
+local is_has_next_page_js = [[
+	var ret="false";
+	//var obj = %s;
+	//if(obj){ret="true"};
+	ret="true";
+	ret
+]]
+--is_has_next_page_js = string.format(is_has_next_page_js, next_page_css_selector)
+-- 找信息
+local find_info_list_js = [[
+    var ns = [];
+    document.querySelectorAll("%s").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+      	var linkText = v.innerText;
+	var href = v.href;
+	 if((linkText.indexOf("预算")>-1 ||linkText.indexOf("预决算")>-1||linkText.indexOf("绩效")>-1 ||linkText.indexOf("目标")>-1 ) &&
+            (linkText.indexOf("2022年")>-1 || linkText.indexOf("2023年")>-1 || linkText.indexOf("2024年")>-1)) {
+			ns.push({"index":i,"title":linkText,"href":href})
+		}
+         
+	});
+    ns
+]]
+find_info_list_js = string.format(find_info_list_js, info_list_css_selector)
+--找附件
+local find_info_attach_file_js = [[
+        var ns = [];
+       var extensions = ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'zip', 'rar','PDF','DOC','DOCX','XLS','XLSX','ZIP','RAR'];
+       var extensionRegex = new RegExp('\\.(' + extensions.join('|') + ')', 'i');
+       document.querySelectorAll("a").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        var href = v.href
+	if(extensionRegex.test(linkText.toLowerCase())||extensionRegex.test(href.toLowerCase())){
+        	ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+
+--TODO 1. 打开主页
+browser_navagite(timeout, href)
+browser_sleep(1000 * 3)
+
+local has_next_page,current_page_has_data = true,false
+local page_no=1
+repeat
+	current_page_has_data=false
+    local ok, list = browser_executejs(1000*3, 1, find_info_list_js)
+    if ok == "ok" then
+        print("信息列表信息", list, #list)
+        for _, v in pairs(list) do
+            print(v.title, v.href, v.index)
+            --browser_click(1000*5, 3, string.format('document.querySelectorAll("%s")[', info_list_css_selector) .. v['index'] .. ']')
+            --browser_sleep(1000 * 1)
+			browser_navagite(true,1000*30, v.href)
+            --local path = browser_url_last_segs(2,v.href)
+            --print("path::",path)
+			path = v.href
+            local ok2, list2 = browser_executejs(path, 1000*3, 1, find_info_attach_file_js)
+			if ok2~="ok" then
+				path = browser_url_last_segs(2,v.href)
+				ok2, list2 = browser_executejs(path, 1000*3, 1, find_info_attach_file_js)
+			end
+            print('查找详情页附件链接', ok2, list2, #list2)
+            if ok2 == "ok" then
+                for _, v2 in pairs(list2) do
+                    -- 存储数据
+                    current_page_has_data=true
+                    print(v2.title, v2.href)
+                    browser_save("", "", "", "", { ["department"] = v['title'], ["info_title"] = v2['title'], ["attach_href"] = v2["href"] })
+                end
+            else
+				print("未下载",v.title,v.href)
+				browser_log("下载失败",v.title,v.href)
+			end
+            --browser_closetabs(path, 500)
+        end
+    else
+		print("执行列表页查询信息js失败",ok)
+	end
+    local ok3, ret = browser_executejs(1000, 0, is_has_next_page_js)
+    print(ok3,ret)
+	if ret=="true" then
+		has_next_page=true
+		browser_click(1000, 3, next_page_css_selector)
+		page_no=page_no+1
+	else
+		has_next_page=false
+	end
+	print("翻页：：",page_no,max_pages)
+	browser_sleep(1000 * 3)
+	--关闭其他页面，莫名其妙有tab关不掉
+	browser_closetabs_without("",href,1000*1)
+	
+until not has_next_page or  page_no > max_pages  --not current_page_has_data
+
+print("所有链接都爬完了")
+browser_sleep(1000 * 5)

program/spider/scripts/模版/2层JS翻页模版.lua_bak

@@ -0,0 +1,121 @@
+-- 采集参数设置
+-- 二层级,单位信息列表/信息
+-- 不用拼接列表页地址，直接模拟点击列表页的下一页链接
+-- href_tpl 可以不用理会
+
+--https://www.mas.gov.cn/xxgk/opennessTarget/?branch_id=57a3df762c262ea9a00aae84&column_code=150101&topic_id=&type=bumen
+local href = "https://www.putian.gov.cn/zwgk/zdlyxxgk/czzj/bmyjs/sjys/"
+-- 列表页CSS选择期，可以避免检测垃圾数据，不想定义，直接写a也行
+local info_list_css_selector = "a"
+-- 三级页特征，用于找到tab页，准备在这个tab页执行js，关闭tab页
+local info_url_path_tpl = "contents"
+-- 下一页链接或者按钮的CSS选择期
+local next_page_css_selector = 'document.querySelector("div.fy_tit_l a.next")'
+-- 列表页最大页数
+local max_pages = 10
+
+-- 用JS点击下一页，这个变量将弃用
+local href_tpl = "https://www.bjshy.gov.cn/web/zwgk/czsj/3fb44002-%d.html"
+
+local timeout = 1000 * 60 * 120
+-- TODO 
+-- 检查是否有下一页
+local is_has_next_page_js = [[
+	var ret="false";
+	//var obj = %s;
+	//if(obj){ret="true"};
+	ret="true";
+	ret
+]]
+--is_has_next_page_js = string.format(is_has_next_page_js, next_page_css_selector)
+-- 找信息
+local find_info_list_js = [[
+    var ns = [];
+    document.querySelectorAll("%s").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+      	var linkText = v.innerText;
+	var href = v.href;
+	if((linkText.indexOf("预算")>-1 ||linkText.indexOf("预决算")>-1) &&
+			(linkText.indexOf("2022年")>-1 || linkText.indexOf("2023年")>-1 || linkText.indexOf("2024年")>-1)) {
+			ns.push({"index":i,"title":linkText,"href":href})
+		}
+         
+	});
+    ns
+]]
+find_info_list_js = string.format(find_info_list_js, info_list_css_selector)
+--找附件
+local find_info_attach_file_js = [[
+        var ns = [];
+       var extensions = ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'zip', 'rar','PDF','DOC','DOCX','XLS','XLSX','ZIP','RAR'];
+       var extensionRegex = new RegExp('\\.(' + extensions.join('|') + ')', 'i');
+       document.querySelectorAll("a").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        var href = v.href
+	if(extensionRegex.test(linkText.toLowerCase())||extensionRegex.test(href.toLowerCase())){
+        	ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+
+--TODO 1. 打开主页
+browser_navagite(timeout, href)
+browser_sleep(1000 * 3)
+
+local has_next_page,current_page_has_data = true,false
+local page_no=1
+repeat
+	current_page_has_data=false
+    local ok, list = browser_executejs(1000*3, 1, find_info_list_js)
+    if ok == "ok" then
+        print("信息列表信息", list, #list)
+        for _, v in pairs(list) do
+            print(v.title, v.href, v.index)
+            --browser_click(1000*5, 3, string.format('document.querySelectorAll("%s")[', info_list_css_selector) .. v['index'] .. ']')
+            --browser_sleep(1000 * 1)
+			browser_navagite(true,1000*30, v.href)
+            --local path = browser_url_last_segs(2,v.href)
+            --print("path::",path)
+			path = v.href
+            local ok2, list2 = browser_executejs(path, 1000*3, 1, find_info_attach_file_js)
+			if ok2~="ok" then
+				path = browser_url_last_segs(2,v.href)
+				ok2, list2 = browser_executejs(path, 1000*3, 1, find_info_attach_file_js)
+			end
+            print('查找详情页附件链接', ok2, list2, #list2)
+            if ok2 == "ok" then
+                for _, v2 in pairs(list2) do
+                    -- 存储数据
+                    current_page_has_data=true
+                    print(v2.title, v2.href)
+                    browser_save("", "", "", "", { ["department"] = v['title'], ["info_title"] = v2['title'], ["attach_href"] = v2["href"] })
+                end
+            else
+				print("未下载",v.title,v.href)
+				browser_log("下载失败",v.title,v.href)
+			end
+            --browser_closetabs(path, 500)
+        end
+    else
+		print("执行列表页查询信息js失败",ok)
+	end
+    local ok3, ret = browser_executejs(1000, 0, is_has_next_page_js)
+    print(ok3,ret)
+	if ret=="true" then
+		has_next_page=true
+		browser_click(1000, 3, next_page_css_selector)
+		page_no=page_no+1
+	else
+		has_next_page=false
+	end
+	print("翻页：：",page_no,max_pages)
+	browser_sleep(1000 * 3)
+	--关闭其他页面，莫名其妙有tab关不掉
+	browser_closetabs_without("",href,1000*1)
+	
+until not has_next_page or  page_no > max_pages  --not current_page_has_data
+
+print("所有链接都爬完了")
+browser_sleep(1000 * 5)

program/spider/scripts/模版/2层URL翻页模版.lua

@@ -0,0 +1,105 @@
+-- 采集参数设置
+-- 二层级,单位信息列表/信息
+-- 拼接列表页地址，需要指定模版
+-- 列表中含有附件
+require "scripts/inc/util"
+-- 列表页模版
+local href = "https://czj.xinyang.gov.cn/c/113/1.html"
+local href_tpl = "https://czj.xinyang.gov.cn/c/113/%d.html"
+-- 列表页CSS选择期，可以避免检测垃圾数据，不想定义，直接写a也行
+local info_list_css_selector = "div.zfxxgk_zdgkc ul a"
+-- 列表页页数
+local first_page_no,last_page_no,page_step = 1,100,1
+
+local timeout = 1000 * 60 * 120
+-- TODO 
+-- 检查是否有下一页
+local is_has_next_page_js = [[
+	var ret="false";
+	//var obj = %s;
+	//if(obj){ret="true"};
+	ret="true";
+	ret
+]]
+is_has_next_page_js = string.format(is_has_next_page_js, next_page_css_selector)
+-- 找信息
+local find_info_list_js = [[
+    var ns = [];
+    document.querySelectorAll("%s").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+	 if((linkText.indexOf("预算")>-1 ||linkText.indexOf("预决算")>-1||linkText.indexOf("绩效")>-1 ||linkText.indexOf("目标")>-1 ) &&
+            (linkText.indexOf("2024年")>-1)) { //linkText.indexOf("2022年")>-1 || linkText.indexOf("2023年")>-1 || 
+            ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+find_info_list_js = string.format(find_info_list_js, info_list_css_selector)
+--找附件
+local find_info_attach_file_js = [[
+       var ns = [];
+       var extensions = ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'zip', 'rar','PDF','DOC','DOCX','XLS','XLSX','ZIP','RAR'];
+       var extensionRegex = new RegExp('\\.(' + extensions.join('|') + ')$', 'i');
+       document.querySelectorAll("a").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        var href = v.href
+	if(extensionRegex.test(linkText.toLowerCase())||extensionRegex.test(href.toLowerCase())){
+        	ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+
+--TODO 1. 打开主页
+local page_no=first_page_no
+local current_page_has_data=true
+repeat
+ 	if page_no>first_page_no then
+		href = string.format(href_tpl,page_no)
+	end
+	browser_navagite(1000*120, href)
+	browser_sleep(1000 * 1)
+		
+	current_page_has_data=false
+    local ok, list = browser_executejs(1000*30, 1, find_info_list_js)
+    if ok == "ok" then
+        print("信息列表信息", list, table_length(list))
+        for _, v in pairs(list) do
+            print(v.title, v.href, v.index)
+			browser_navagite(true,1000*30, v.href)
+            local path = v.href
+			local ok2, list2 = browser_executejs(v.href, 1000*30, 1, find_info_attach_file_js)
+			if ok2~="ok" then
+				path = browser_url_last_segs(2,v.href)
+				ok2, list2 = browser_executejs(path, 1000*3, 1, find_info_attach_file_js)
+			end
+            print('查找详情页附件链接', ok2, list2, table_length(list2))
+            if ok2 == "ok" then
+                for _, v2 in pairs(list2) do
+                    -- 存储数据
+                    current_page_has_data=true
+                    print(v2.title, v2.href)
+                    browser_save("", "", "", "", { ["department"] = v['title'], ["info_title"] = v2['title'], ["attach_href"] = v2["href"] ,["source"]=v.href})
+                end
+            end
+			
+			-- 找不到附件，保存主文档区域到文件
+			if table_length(list2)==0 then
+				print("找不到附件，准备生成网页pdf")
+				clear_page_el("",path,1000*2,"div.article_txt")
+				save_page_to_pdf("",path,"/Users/taozhang/Downloads/docs2/",v.title)
+				browser_save("", "", "", "", { ["department"] = v["title"], ["info_title"] =  v["title"], ["attach_href"] = "",["source"]=v.href })
+			end
+			
+            browser_closetabs(path, 500)
+        end
+    end
+    browser_reset()
+    page_no = page_no + page_step
+	print("翻页：：",page_no,last_page_no)
+until  page_no>last_page_no --not current_page_has_data or
+
+print("所有链接都爬完了")
+browser_sleep(1000 * 5)

program/spider/scripts/模版/2层URL翻页模版.lua_bak

@@ -0,0 +1,108 @@
+-- 采集参数设置
+-- 二层级,单位信息列表/信息
+-- 拼接列表页地址，需要指定模版
+-- 列表中含有附件
+
+-- 列表页地址
+local href = "https://www.nanning.gov.cn/zwgk/fdzdgknr/czxx/sbjyjs/sbjbmys/2023bmys/index.html"
+-- 列表页模版
+local href_tpl = "https://www.nanning.gov.cn/zwgk/fdzdgknr/czxx/sbjyjs/sbjbmys/2023bmys/index_%d.html"
+-- 列表页CSS选择期，可以避免检测垃圾数据，不想定义，直接写a也行
+local info_list_css_selector = "a"
+-- 三级页特征，用于找到tab页，准备在这个tab页执行js，关闭tab页
+local info_url_path_tpl = "t20240306"
+-- 列表页页数
+local first_page_no,last_page_no,page_step = 0,6,1
+
+
+local timeout = 1000 * 60 * 120
+-- TODO 
+-- 检查是否有下一页
+local is_has_next_page_js = [[
+	var ret="false";
+	//var obj = %s;
+	//if(obj){ret="true"};
+	ret="true";
+	ret
+]]
+is_has_next_page_js = string.format(is_has_next_page_js, next_page_css_selector)
+-- 找信息
+local find_info_list_js = [[
+    var ns = [];
+    document.querySelectorAll("%s").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        ns.push({"index":i,"title":linkText,"href":v.href})
+        if((linkText.indexOf("预算")>-1 ||linkText.indexOf("预决算")>-1) &&
+            (linkText.indexOf("2022年")>-1 || linkText.indexOf("2023年")>-1 || linkText.indexOf("2024年")>-1)) {
+            ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+find_info_list_js = string.format(find_info_list_js, info_list_css_selector)
+--找附件
+local find_info_attach_file_js = [[
+       var ns = [];
+       var extensions = ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'zip', 'rar','PDF','DOC','DOCX','XLS','XLSX','ZIP','RAR'];
+       var extensionRegex = new RegExp('\\.(' + extensions.join('|') + ')', 'i');
+       document.querySelectorAll("a").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        var href = v.href
+	if(extensionRegex.test(linkText.toLowerCase())||extensionRegex.test(href.toLowerCase())){
+        	ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+
+--TODO 1. 打开主页
+local page_no=first_page_no
+local current_page_has_data=true
+repeat
+	if page_no>first_page_no then
+		href = string.format(href_tpl,page_no)
+	end
+	browser_navagite(1000*120, href)
+	browser_sleep(1000 * 1)
+		
+	current_page_has_data=false
+    local ok, list = browser_executejs(1000*30, 1, find_info_list_js)
+    if ok == "ok" then
+        print("信息列表信息", list, #list)
+        for _, v in pairs(list) do
+            print(v.title, v.href, v.index)
+            --browser_click(1000*5, 3, string.format('document.querySelectorAll("%s")[', info_list_css_selector) .. v['index'] .. ']')
+            --browser_sleep(1000 * 1)
+			browser_navagite(true,1000*30, v.href)
+            --local path = string.match(v.href, info_url_path_tpl)
+            --print("path::",path)
+            local path = v.href
+			local ok2, list2 = browser_executejs(v.href, 1000*30, 1, find_info_attach_file_js)
+			if ok2~="ok" then
+				path = browser_url_last_segs(2,v.href)
+				ok2, list2 = browser_executejs(path, 1000*3, 1, find_info_attach_file_js)
+			end
+            print('查找详情页附件链接', ok2, list2, #list2)
+            if ok2 == "ok" then
+                for _, v2 in pairs(list2) do
+                    -- 存储数据
+                    current_page_has_data=true
+                    print(v2.title, v2.href)
+                    browser_save("", "", "", "", { ["department"] = v['title'], ["info_title"] = v2['title'], ["attach_href"] = v2["href"] })
+                end
+            end
+            browser_closetabs(path, 500)
+        end
+    end
+    -- browser_reset()
+    page_no = page_no + page_step
+	--关闭其他页面，莫名其妙有tab关不掉
+	browser_closetabs_without("",href,1000*1)
+	print("翻页：：",page_no,last_page_no)
+	browser_sleep(1000 * 2)
+until  page_no>last_page_no --not current_page_has_data or
+
+print("所有链接都爬完了")
+browser_sleep(1000 * 5)

program/spider/scripts/模版/3层带机构列表无需翻页模版.lua

@@ -0,0 +1,129 @@
+-- 采集参数设置，芜湖第一屏是机构清单
+-- 二层级,单位信息列表/信息
+-- 不用拼接列表页地址，直接模拟点击列表页的下一页链接
+-- href_tpl 可以不用理会
+local href = "https://www.jdz.gov.cn/zwzx/ztbd/yjsgk/2024/ys/"
+-- 机构选择
+local organ_list_css_selector = "div.ys-tab a"
+-- 列表页CSS选择期，可以避免检测垃圾数据，不想定义，直接写a也行
+local info_list_css_selector = "ul li a"
+
+local timeout = 1000 * 60 * 120
+
+-- JS函数集合
+local find_organ_list_js = [[
+	var ns = [];
+       document.querySelectorAll("%s").forEach((v,i)=>{
+		v.setAttribute("target","_blank");
+		ns.push({"index":i,"title":v.innerText,"href":v.href});
+		return
+	});
+    ns
+]]
+find_organ_list_js = string.format(find_organ_list_js, organ_list_css_selector)
+
+local find_info_list_js = [[
+    var ns = [];
+    document.querySelectorAll("%s").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        if((linkText.indexOf("预算")>-1 ||linkText.indexOf("预决算")>-1) &&
+            (linkText.indexOf("2022年")>-1 || linkText.indexOf("2023年")>-1 || linkText.indexOf("2024年")>-1)) {
+            ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+find_info_list_js = string.format(find_info_list_js, info_list_css_selector)
+
+
+local find_info_attach_file_js = [[
+       var ns = [];
+       var extensions = ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'zip', 'rar'];
+       var extensionRegex = new RegExp('\\.(' + extensions.join('|') + ')', 'i');
+       document.querySelectorAll("a").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        var href = v.href
+	if(extensionRegex.test(linkText.toLowerCase())||extensionRegex.test(href.toLowerCase())){
+        	ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+
+
+--获取table长度
+function getn(t)
+	if type(t)~="table" then
+		return 0
+	end
+	local len=0
+	for _,_ in pairs(t) do
+		len=len+1
+	end
+	return len
+end
+
+
+browser_navagite(timeout, href)
+browser_sleep(1000 * 2)
+
+
+
+local ok, list = browser_executejs(1000*30, 1, find_organ_list_js)
+local total,index = getn(list),1
+print('执行查找机构列表', ok, total)
+if ok == "ok" then
+    for _, v in pairs(list) do
+		-- 打开机构页面
+        print("机构信息",index,total,v.title, v.href)
+		index = index+1
+		-- 关闭其他页面，莫名其妙有tab关不掉
+		browser_closetabs_without("",href,1000*1)
+        --browser_click(1000*5, 3, string.format('document.querySelectorAll("%s")[', organ_list_css_selector) .. v['index'] .. ']')
+		browser_navagite(true,1000*30,v.href)
+		-- 点击一下更多
+		
+		
+		local path = browser_url_last_segs(2,v.href)
+		print("机构页path",path)
+		browser_click(path,1000*10,3,'document.querySelector("div.zfxxgk_zdgkc div.more a")')
+		browser_sleep(1000 * 1)
+		-- 找机构页面的信息列表
+		local ok2, list2 = browser_executejs(path,1000*30, 1, find_info_list_js)
+		print('执行查找信息列表', ok, list, #list)	
+		if ok2 == "ok" then
+			for _, v2 in pairs(list2) do
+				print(v2.title,v2.href)
+				--打开信息页
+				browser_click(path,1000*5, 3, string.format('document.querySelectorAll("%s")[', info_list_css_selector) .. v2['index'] .. ']')
+				browser_sleep(1000 * 1)
+				local path2 = browser_url_last_segs(1,v2.href)
+				print("详情页path",path2)
+				local ok3, list3 = browser_executejs(v2.href,1000*30, 1, find_info_attach_file_js)
+				print('执行查找附件列表', ok, list, #list)	
+				if ok3=="ok" then
+					for _, v3 in pairs(list3) do
+						print("找到附件",v.title,v2.title,v3.title,v3.href)
+						browser_save("", "", "", "", { ["department"] = v.title, ["info_title"] = v2.title, ["attach_href"] = v3.href })
+					end
+				else
+					print("错误",v2.title,v2.href)
+					browser_log("错误",v.title,v.href,v2.title,v2.href)
+				end
+				browser_closetabs(path2, 500)
+			end
+		else
+			print("错误",v.title,v.href)
+			browser_log("错误",v.title,v.href,"","")
+		end
+		browser_sleep(1000 * 2)
+		browser_closetabs(path, 500)
+    end
+else
+	print("抓取失败",ok)
+end
+
+print("任务完成")
+

program/spider/scripts/模版/3层带机构列表无需翻页模版.lua_bak

@@ -0,0 +1,128 @@
+-- 采集参数设置，芜湖第一屏是机构清单
+-- 二层级,单位信息列表/信息
+-- 不用拼接列表页地址，直接模拟点击列表页的下一页链接
+-- href_tpl 可以不用理会
+local href = "https://www.bjxch.gov.cn/zt/yjs/index.html"
+-- 机构选择
+local organ_list_css_selector = "div#Section1 div.box-danwei a"
+-- 列表页CSS选择期，可以避免检测垃圾数据，不想定义，直接写a也行
+local info_list_css_selector = ""
+-- 三级页特征，用于找到tab页，准备在这个tab页执行js，关闭tab页
+local info_url_path_tpl = "contents"
+-- 下一页链接或者按钮的CSS选择期
+local next_page_css_selector = 'document.querySelector("div#page_public_info a[aria-label=跳转至尾页]")'
+-- 列表页最大页数
+local max_pages = 1
+
+-- 用JS点击下一页，这个变量将弃用
+local href_tpl = "https://www.bjshy.gov.cn/web/zwgk/czsj/3fb44002-%d.html"
+local timeout = 1000 * 60 * 120
+
+-- JS函数集合
+local find_organ_list_js = [[
+	var ns = [];
+       document.querySelectorAll("%s").forEach((v,i)=>{
+		v.setAttribute("target","_blank");
+		ns.push({"index":i,"title":v.innerText,"href":v.href});
+		return
+		v.querySelectorAll("a").forEach((v1,i1)=>{
+		var linkText = v1.innerText;
+		var href = v1.href;
+		if (href==""){return;}
+		if( linkText.indexOf("财政预算")>-1 || linkText.indexOf("三公")>-1 || linkText.indexOf("专项资金")>-1 ){
+		ns.push({"index":i1,"title":linkText,"href":href});
+		}
+		});
+	});
+    ns
+]]
+find_organ_list_js = string.format(find_organ_list_js, organ_list_css_selector)
+
+local find_info_list_js = [[
+    var ns = [];
+    document.querySelectorAll("%s").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        //ns.push({"index":i,"title":linkText,"href":v.href})
+        if((linkText.indexOf("预算")>-1 ||linkText.indexOf("预决算")>-1) &&
+            (linkText.indexOf("2022年")>-1 || linkText.indexOf("2023年")>-1 || linkText.indexOf("2024年")>-1)) {
+            ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+find_info_list_js = string.format(find_info_list_js, info_list_css_selector)
+
+
+local find_info_attach_file_js = [[
+       var ns = [];
+       var extensions = ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'zip', 'rar'];
+       var extensionRegex = new RegExp('\\.(' + extensions.join('|') + ')', 'i');
+       document.querySelectorAll("a").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        var href = v.href
+	if(extensionRegex.test(linkText.toLowerCase())||extensionRegex.test(href.toLowerCase())){
+        	ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+
+browser_navagite(timeout, href)
+browser_sleep(1000 * 2)
+
+local ok, list = browser_executejs(1000*30, 1, find_organ_list_js)
+print('执行查找机构列表', ok, list, #list)
+if ok == "ok" then
+    for _, v in pairs(list) do
+		-- 打开机构页面
+        print("机构信息",v.title, v.href)
+		-- 关闭其他页面，莫名其妙有tab关不掉
+		browser_closetabs_without("",href,1000*1)
+        --browser_click(1000*5, 3, string.format('document.querySelectorAll("%s")[', organ_list_css_selector) .. v['index'] .. ']')
+		browser_navagite(true,1000*30,v.href)
+		-- 点击一下更多
+		
+		
+		local path = browser_url_last_segs(2,v.href)
+		print("机构页path",path)
+		browser_click(path,1000*10,3,'document.querySelector("div.zfxxgk_zdgkc div.more a")')
+		browser_sleep(1000 * 1)
+		-- 找机构页面的信息列表
+		local ok2, list2 = browser_executejs(path,1000*30, 1, find_info_list_js)
+		print('执行查找信息列表', ok, list, #list)	
+		if ok2 == "ok" then
+			for _, v2 in pairs(list2) do
+				print(v2.title,v2.href)
+				--打开信息页
+				browser_click(path,1000*5, 3, string.format('document.querySelectorAll("%s")[', info_list_css_selector) .. v2['index'] .. ']')
+				browser_sleep(1000 * 1)
+				local path2 = browser_url_last_segs(1,v2.href)
+				print("详情页path",path2)
+				local ok3, list3 = browser_executejs(v2.href,1000*30, 1, find_info_attach_file_js)
+				print('执行查找附件列表', ok, list, #list)	
+				if ok3=="ok" then
+					for _, v3 in pairs(list3) do
+						print("找到附件",v.title,v2.title,v3.title,v3.href)
+						browser_save("", "", "", "", { ["department"] = v.title, ["info_title"] = v2.title, ["attach_href"] = v3.href })
+					end
+				else
+					print("错误",v2.title,v2.href)
+					browser_log("错误",v.title,v.href,v2.title,v2.href)
+				end
+				browser_closetabs(path2, 500)
+			end
+		else
+			print("错误",v.title,v.href)
+			browser_log("错误",v.title,v.href,"","")
+		end
+		browser_sleep(1000 * 2)
+		browser_closetabs(path, 500)
+    end
+else
+	print("抓取失败",ok)
+end
+
+print("任务完成")
+

program/spider/scripts/模版/3级列表不翻页不定css.lua

@@ -0,0 +1,119 @@
+-- 采集参数设置
+-- 三层级,部门、单位列表/单位信息列表/信息
+-- 部门不翻页，信息列表也不翻页
+
+local href = "https://www.hlj.gov.cn/hlj/c108397/zfxxgk.shtml"
+-- 单位机构清单选择器
+local organ_list_css_selector = 'document.querySelectorAll("div#tableList table a")'
+-- 下一页链接或者按钮的CSS选择期
+local next_page_css_selector = 'document.querySelector("div#page_div a.nextbtn")'
+-- 列表页CSS选择期，可以避免检测垃圾数据，不想定义，直接写a也行
+local info_list_css_selector = "a"
+
+-- 检查是否有下一页
+local is_has_next_page_js = [[
+	var ret="false";
+	var obj = %s;
+	if(obj){ret="true"};
+	ret
+]]
+is_has_next_page_js = string.format(is_has_next_page_js, next_page_css_selector)
+
+-- 查找单位清单
+local find_organ_list_js = [[
+    var ns = [];
+    %s.forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        ns.push({"index":i,"href":v.href,"text":v.innerText});
+    });
+    ns
+]]
+find_organ_list_js = string.format(find_organ_list_js, organ_list_css_selector)
+-- 获取单位年度财政预算链接
+local find_organ_info_js = [[
+    var ns = [];
+    document.querySelectorAll("a").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        if( (linkText.indexOf("预算")>0 ||linkText.indexOf("预决算")>0) &&
+            (linkText.indexOf("2022年")>0 || linkText.indexOf("2023年")>0 || linkText.indexOf("2024年")>0)) {
+            ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+-- 获取信息中的附件
+local find_info_attach_file_js = [[
+       var ns = [];
+       var allow_exts= ["doc","docx","xls","xlsx","pdf","txt","jpg","png","ppt","pptx","zip","rar"]
+       var getFileExt=(text)=>{var tmp=text.split(".");return tmp[tmp.length-1];}
+       document.querySelectorAll("a").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        var ext = getFileExt(linkText);
+        if(ext.length==0 ||ext.length>4 ){
+            ext = getFileExt(v.href);
+        }
+        if(ext.length==0 ||ext.length>4 ){return;}
+        if(allow_exts.indexOf(ext)>=0){
+        ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+
+
+local timeout = 1000 * 60 * 120
+browser_sleep(1000 * 2)
+
+--TODO 1. 打开主页
+browser_navagite(timeout, href)
+browser_sleep(1000 * 2)
+--先点击部门预决算链接
+
+
+
+local has_next_page=false
+
+repeat
+	--TODO 2. 找到单位链接
+	print("找单位链接")
+	local ok, list = browser_executejs(timeout, 1, find_organ_list_js)
+	if ok == "ok" then
+	    print("返回拉取市政府下边子级单位清单", list, #list)
+	    for k, v in pairs(list) do
+	        print(v)
+	        --打开单位链接
+	        browser_click(6000, 3, organ_list_css_selector..'[' .. v['index'] .. ']')
+	        browser_sleep(1000 * 1)
+	        -- 这里不翻页翻页
+	        browser_click(1000*5, 3, string.format('document.querySelectorAll("a")[', info_list_css_selector) .. v['index'] .. ']')
+	        browser_sleep(1000 * 1)
+	        --local path = string.match(v.href, info_url_path_tpl)
+	        --print("path::",path)
+	        local ok2, list2 = browser_executejs(v.href, timeout, 1, find_info_attach_file_js)
+	        print('查找详情页附件链接', ok2, list2, #list2)
+	        if ok2 == "ok" then
+	            for _, v2 in pairs(list2) do
+	                -- 存储数据
+	                current_page_has_data=true
+	                print(v2.title, v2.href)
+	                browser_save("", "", "", "", { ["department"] = v['title'], ["info_title"] = v2['title'], ["attach_href"] = v2["href"] })
+	            end
+	        end
+	        browser_closetabs(v.href, 500)
+			--关闭列表页
+		    browser_closetabs('bmys', 1000)
+	    end
+	else
+	    print("--错误，第一页就打不开")
+	end
+	local ok, ret = browser_executejs(timeout, 0, is_has_next_page_js)
+	if ret == "true" then
+		has_next_page = true
+	else
+		has_next_page=false
+	end
+until not has_next_page
+print("找单位链接结束")
+browser_sleep(1000 * 60)

program/spider/scripts/模版/4层带机构带机构首页机构模版.lua

@@ -0,0 +1,161 @@
+-- 采集参数设置
+-- 三层级,部门、单位列表/单位信息列表/信息
+-- 部门不翻页，信息列表也不翻页
+
+local href = "https://www.pudong.gov.cn/zwgk/014005004/"
+-- 单位机构清单选择器
+local organ_list_css_selector = 'document.querySelectorAll("div#main td div.trout-region-info ul li a")'
+-- 下一页链接或者按钮的CSS选择期
+local next_page_css_selector = 'document.querySelector("div#page_div a.nextbtn")'
+-- 机构信息入口
+local organ_page_css_selector='document.querySelectorAll("div#main table a")'
+
+-- 列表页CSS选择期，可以避免检测垃圾数据，不想定义，直接写a也行
+local info_list_css_selector = "table a"
+
+-- 检查是否有下一页
+local is_has_next_page_js = [[
+	var ret="false";
+	//var obj = %s;
+	//if(obj){ret="true"};
+	ret = "true";
+	ret
+]]
+is_has_next_page_js = string.format(is_has_next_page_js, next_page_css_selector)
+
+-- 查找单位清单
+local find_organ_list_js = [[
+    var ns = [];
+    %s.forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        ns.push({"index":i,"href":v.href,"title":v.innerText});
+    });
+    ns
+]]
+find_organ_list_js = string.format(find_organ_list_js, organ_list_css_selector)
+-- 查找机构首页信息列表
+local find_orgin_main_page_list_js = [[
+   var ns = [];
+    %s.forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        ns.push({"index":i,"href":v.href,"title":v.innerText});
+    });
+    ns
+]]
+find_orgin_main_page_list_js = string.format(find_orgin_main_page_list_js, organ_page_css_selector)
+-- 获取单位年度财政预算链接
+local find_organ_info_js = [[
+    var ns = [];
+    document.querySelectorAll("a").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        if( (linkText.indexOf("预算")>0 ||linkText.indexOf("预决算")>0) &&
+            (linkText.indexOf("2022年")>0 || linkText.indexOf("2023年")>0 || linkText.indexOf("2024年")>0)) {
+            ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+-- 获取信息中的附件
+local find_info_attach_file_js = [[
+       var ns = [];
+       var allow_exts= ["doc","docx","xls","xlsx","pdf","txt","jpg","png","ppt","pptx","zip","rar"]
+       var getFileExt=(text)=>{var tmp=text.split(".");return tmp[tmp.length-1];}
+       document.querySelectorAll("a").forEach((v,i)=>{
+        v.setAttribute("target","_blank");
+        var linkText = v.innerText;
+        var ext = getFileExt(linkText);
+        if(ext.length==0 ||ext.length>4 ){
+            ext = getFileExt(v.href);
+        }
+        if(ext.length==0 ||ext.length>4 ){return;}
+        if(allow_exts.indexOf(ext)>=0){
+        ns.push({"index":i,"title":linkText,"href":v.href})
+        }
+    });
+    ns
+]]
+
+
+local timeout = 1000 * 60 * 120
+browser_sleep(1000 * 2)
+
+--TODO 1. 打开主页
+browser_navagite(timeout, href)
+browser_sleep(1000 * 2)
+--先点击部门预决算链接
+local has_next_page=false
+
+repeat
+	--TODO 2. 找到单位链接
+	print("找单位链接")
+	local ok, list = browser_executejs(1000*30, 1, find_organ_list_js)
+	if ok == "ok" then
+	    print("返回拉取市政府下边子级单位清单", list, #list)
+	    for k, v in pairs(list) do
+	        print("机构信息：",v.title,v.href)
+	        --打开单位首页
+	        --browser_click(6000, 3, organ_list_css_selector..'[' .. v['index'] .. ']')
+	        browser_navagite(true,1000*30, v.href)
+			browser_sleep(1000 * 3)
+			print(find_orgin_main_page_list_js)
+			local path = v.href
+			local ok2,list2 = browser_executejs(path, 1000*30, 1, find_orgin_main_page_list_js)
+			print("path",path,ok2,#list2)
+			if ok2~="ok" then
+				path = browser_url_last_segs(2,v.href)
+				print("path",path)
+				ok2,list2 = browser_executejs(path, 1000*30, 1, organ_page_selector_js)
+			end
+			print("path",path,ok2,#list2)
+			if ok2=="ok" then
+				for _,v2 in pairs(list2) do
+					print("机构首页信息类型列表::",v2.title,v2.href)
+					browser_navagite(true,1000*30,v2.href)
+					browser_sleep(1000 * 1)
+					
+					--TODO 进入部门信息列表页
+					local ok3, list3 = browser_executejs(v2.href, timeout, 1, find_organ_info_js)
+					print('查找详情页附件链接', ok3, list3, #list3)
+					if ok3=="ok" then
+						for _,v3 in pairs(list3) do
+							browser_navagite(true,1000*30,v3.href)
+							browser_sleep(1000 * 1)
+						
+							local ok4, list4 = browser_executejs(v3.href, timeout, 1, find_info_attach_file_js)
+							print('查找详情页附件链接', ok4, list4, #list4)
+							if ok4 == "ok" then
+								for _, v4 in pairs(list4) do
+									-- 存储数据
+									current_page_has_data=true
+									print(v4.title, v4.href)
+									browser_save("", "", "", "", { ["department"] = v['title'], ["info_title"] = v4['title'], ["attach_href"] = v4["href"] })
+								end
+							end
+							browser_closetabs(v3.href, 500)
+							
+						end
+					end
+					
+					
+					browser_closetabs(v2.href, 500)
+				end
+			else
+				print("查找机构首页列表失败",ok2)
+			end
+			browser_closetabs(path, 500)
+		end
+	else
+	    print("--错误，第一页就打不开")
+	end
+	local ok, ret = browser_executejs(timeout, 0, is_has_next_page_js)
+	if ret == "true" then
+		has_next_page = true
+	else
+		has_next_page=false
+	end
+	-- 关闭其他页面，莫名其妙有tab关不掉
+	browser_closetabs_without("",href,1000*1)
+until not has_next_page
+print("找单位链接结束")
+browser_sleep(1000 * 60)

program/spider/storage.go

@@ -0,0 +1,140 @@
+/*
+存储;
+暂时提供3种存储方式：
+
+1）控制台
+2） MongoDb
+3) 文件
+*/
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type (
+	//Storage
+	Storage interface {
+		//保存数据
+		Save(spiderCode, siteName, siteChannelName, siteChannelUrl string, data map[string]interface{}) error
+	}
+
+	//MongoStorage
+	MongoStorage struct {
+		c *mongo.Collection
+	}
+	//调试模式下的数据存储
+	ConsoleStorage struct {
+	}
+	//FileStorage
+	FileStorage struct {
+		f io.WriteCloser
+	}
+	//企业微信机器人消息
+	WeChatMsgStorage struct {
+		RobotURL string
+	}
+)
+
+// NewMongoStorage
+func NewMongoStorage(mongoURI, db, collection string) *MongoStorage {
+	client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoURI))
+	if err != nil {
+		panic(err)
+	}
+	return &MongoStorage{
+		c: client.Database(db).Collection(collection),
+	}
+
+}
+
+// NewDebugerStorage
+func NewConsoleStorage() *ConsoleStorage {
+	return &ConsoleStorage{}
+}
+
+// NewFileStorage
+func NewFileStorage(f io.WriteCloser) *FileStorage {
+	return &FileStorage{f: f}
+}
+
+// NewWeChatMsgStorage
+func NewWeChatMsgStorage(robotUrl string) *WeChatMsgStorage {
+	return &WeChatMsgStorage{robotUrl}
+}
+
+// Save
+func (ms *MongoStorage) Save(spiderCode, siteName, siteChannelName, siteChannelUrl string, data map[string]interface{}) error {
+	data["spider_code"] = spiderCode
+	data["site_name"] = siteName
+	data["site_channel_name"] = siteChannelName
+	data["site_channel_url"] = siteChannelUrl
+	_, err := ms.c.InsertOne(context.TODO(), data)
+	return err
+}
+
+// Save
+func (ds *ConsoleStorage) Save(spiderCode, siteName, siteChannelName, siteChannelUrl string, data map[string]interface{}) error {
+	data["spider_code"] = spiderCode
+	data["site_name"] = siteName
+	data["site_channel_name"] = siteChannelName
+	data["site_channel_url"] = siteChannelUrl
+	fmt.Println(" data=")
+	for k, v := range data {
+		fmt.Printf("\t%s = %v \n", k, v)
+	}
+	return nil
+}
+
+// Save
+func (fs *FileStorage) Save(spiderCode, siteName, siteChannelName, siteChannelUrl string, data map[string]interface{}) error {
+	data["spider_code"] = spiderCode
+	data["site_name"] = siteName
+	data["site_channel_name"] = siteChannelName
+	data["site_channel_url"] = siteChannelUrl
+	buf := new(bytes.Buffer)
+	json.NewEncoder(buf).Encode(data)
+	fs.f.Write([]byte(","))
+	fs.f.Write(buf.Bytes())
+	fs.f.Write([]byte("\n"))
+	return nil
+}
+
+// Save
+func (ws *WeChatMsgStorage) Save(spiderCode, siteName, siteChannelName, siteChannelUrl string, data map[string]interface{}) error {
+	msg := fmt.Sprintf("%v", data)
+	body := struct {
+		MsgType string `json:"msgtype"`
+		Text    struct {
+			Content string `json:"content"`
+		} `json:"text"`
+	}{
+		MsgType: "text",
+		Text: struct {
+			Content string `json:"content"`
+		}{msg},
+	}
+	bs, err := json.Marshal(body)
+	if err != nil {
+		return err
+	}
+	client := new(http.Client)
+	req, err := http.NewRequest("POST", ws.RobotURL, bytes.NewReader(bs))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	return resp.Body.Close()
+}

program/spider/util.go

@@ -0,0 +1,141 @@
+/**
+ * 工具包封装
+ */
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"strings"
+	"time"
+
+	"github.com/yuin/gopher-lua"
+)
+
+// Catch 异常崩溃拦截
+func Catch() {
+	if err := recover(); err != nil {
+		log.Println(err)
+	}
+}
+
+// MapToTable converts a Go map to a lua table
+func MapToTable(m map[string]interface{}) *lua.LTable {
+	// Main table pointer
+	resultTable := &lua.LTable{}
+
+	// Loop map
+	for key, element := range m {
+
+		switch element.(type) {
+		case float64:
+			resultTable.RawSetString(key, lua.LNumber(element.(float64)))
+		case int64:
+			resultTable.RawSetString(key, lua.LNumber(element.(int64)))
+		case string:
+			resultTable.RawSetString(key, lua.LString(element.(string)))
+		case bool:
+			resultTable.RawSetString(key, lua.LBool(element.(bool)))
+		case []byte:
+			resultTable.RawSetString(key, lua.LString(string(element.([]byte))))
+		case map[string]interface{}:
+
+			// Get table from map
+			tble := MapToTable(element.(map[string]interface{}))
+
+			resultTable.RawSetString(key, tble)
+
+		case time.Time:
+			resultTable.RawSetString(key, lua.LNumber(element.(time.Time).Unix()))
+
+		case []map[string]interface{}:
+
+			// Create slice table
+			sliceTable := &lua.LTable{}
+
+			// Loop element
+			for _, s := range element.([]map[string]interface{}) {
+
+				// Get table from map
+				tble := MapToTable(s)
+
+				sliceTable.Append(tble)
+			}
+
+			// Set slice table
+			resultTable.RawSetString(key, sliceTable)
+
+		case []interface{}:
+
+			// Create slice table
+			sliceTable := &lua.LTable{}
+
+			// Loop interface slice
+			for _, s := range element.([]interface{}) {
+
+				// Switch interface type
+				switch s.(type) {
+				case map[string]interface{}:
+
+					// Convert map to table
+					t := MapToTable(s.(map[string]interface{}))
+
+					// Append result
+					sliceTable.Append(t)
+
+				case float64:
+
+					// Append result as number
+					sliceTable.Append(lua.LNumber(s.(float64)))
+
+				case string:
+
+					// Append result as string
+					sliceTable.Append(lua.LString(s.(string)))
+
+				case bool:
+
+					// Append result as bool
+					sliceTable.Append(lua.LBool(s.(bool)))
+				}
+			}
+
+			// Append to main table
+			resultTable.RawSetString(key, sliceTable)
+		}
+	}
+
+	return resultTable
+}
+
+// TabletoMap converts a lua table to go map
+func TableToMap(t *lua.LTable) map[string]interface{} {
+	ret := make(map[string]interface{})
+	t.ForEach(func(k, v lua.LValue) {
+		key := fmt.Sprint(k)
+		if val, ok := v.(*lua.LTable); ok {
+			ret[key] = TableToMap(val)
+		} else {
+			if val, ok := v.(lua.LString); ok {
+				ret[key] = string(val)
+			} else if val, ok := v.(lua.LNumber); ok {
+				ret[key] = int64(val)
+			} else if val, ok := v.(lua.LBool); ok {
+				ret[key] = bool(val)
+			} else {
+				ret[key] = fmt.Sprint(v)
+			}
+		}
+	})
+	return ret
+}
+
+// urlLastSegs
+func urlLastSegs(href string, segs int) string {
+	tmp := strings.Split(href, "/")
+	if len(tmp) < segs {
+		return href
+	}
+	return strings.Join(tmp[len(tmp)-segs-1:], "/")
+}

program/spider/vm.go

@@ -0,0 +1,227 @@
+/**
+ * 虚拟机
+ */
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/chromedp/chromedp"
+	"github.com/yuin/gopher-lua"
+	"github.com/yuin/gopher-lua/parse"
+)
+
+const (
+	run_on_device_remote = iota
+	run_on_device_local
+)
+
+type (
+	//虚拟机
+	VM struct {
+		WsAddr       string
+		Headless     bool
+		ShowImage    bool
+		ProxyAddr    string
+		RunMode      int
+		DownloadPath string
+		B            *Browser
+		S            Storage
+	}
+	//浏览器，（不用之前封装的，这个更轻量）
+	Browser struct {
+		Ctx      context.Context
+		CancelFn context.CancelFunc
+	}
+)
+
+// NewVM
+func NewRemoteVM(wsAddr string, s Storage) *VM {
+	return &VM{WsAddr: wsAddr, RunMode: run_on_device_remote, S: s}
+}
+
+// NewVM
+func NewLocalVM(headless bool, showImage bool, proxyAddr, downloadPath string, s Storage) *VM {
+	return &VM{Headless: headless, ProxyAddr: proxyAddr, ShowImage: showImage,
+		RunMode:      run_on_device_local,
+		DownloadPath: downloadPath,
+		S:            s}
+}
+
+// Quit
+func (b *Browser) Quit() {
+	if b != nil && b.CancelFn != nil {
+		b.CancelFn()
+		b.Ctx = nil
+		b.CancelFn = nil
+	}
+}
+
+// createRemoteBrowser 创建远程浏览器
+func createRemoteBrowser(wsAddr string) *Browser {
+	allocCtx, cancelFn := chromedp.NewRemoteAllocator(context.TODO(),
+		wsAddr)
+	incCtx, _ := chromedp.NewContext(allocCtx)
+	return &Browser{
+		incCtx, cancelFn,
+	}
+}
+
+// createLocalBrowser 创建本地浏览器
+func createLocalBrowser(headless,
+	showImage bool,
+	proxyAddr, downloadPath string) *Browser {
+	baseCtx, _ := chromedp.NewContext(context.Background())
+	chromeOptions := []chromedp.ExecAllocatorOption{
+		chromedp.NoFirstRun,
+		chromedp.NoDefaultBrowserCheck,
+		chromedp.DisableGPU,
+		chromedp.NoSandbox,
+		chromedp.WindowSize(1920, 1080),
+		chromedp.Flag("enable-automation", false), // 防止监测webdriver
+		chromedp.Flag("disable-blink-features", "AutomationControlled"), //禁用 blink 特征 作者：知识货栈 https://www.bilibili.com/read/cv24371371/ 出处：bilibili
+		chromedp.Flag("lang", "zh-CN"),
+		chromedp.Flag("mixed-forms-disable-autofill", false), //从https转http不再检查
+		chromedp.Flag("ignore-certificate-errors", true),     //忽略错误
+		chromedp.Flag("ignore-urlfetcher-cert-requests", true),
+		chromedp.Flag("enable-automation", false),                       // 防止监测webdriver
+		chromedp.Flag("disable-blink-features", "AutomationControlled"), //禁用 blink 特征
+		chromedp.Flag("force-dev-mode-highlighting", true),
+		chromedp.Flag("disable-extensions", false), //是否禁用扩展
+		chromedp.Flag("headless", headless),
+		chromedp.Flag("user-agent", "Chrome 9 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/93.0.4577.58 Safari/537.36 Edg/93.0.961.33"),
+		chromedp.Flag("disable-keep-alive", true),
+		chromedp.Flag("disable-dev-shm-usage", false),
+		chromedp.Flag("default-browser-check", false),
+		chromedp.Flag("disable-web-security", true), //禁用网络安全标志
+		chromedp.Flag("mute-audio", false),
+		chromedp.Flag("https-upgrades", "disabled"),
+		chromedp.Flag("accept-language", `zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-TW;q=0.6`),
+		//chromedp.Flag("blink-settings", "imagesEnabled=true"),
+		//chromedp.Flag("incognito", true),     //隐私模式
+		chromedp.Flag("disable-cache", true), //不用缓存
+	}
+	if proxyAddr != "" {
+		chromeOptions = append(chromeOptions,
+			chromedp.ProxyServer(fmt.Sprintf("socks5://%s", proxyAddr)))
+	}
+	if downloadPath != "" {
+		chromeOptions = append(chromeOptions,
+			chromedp.Flag("download-path", downloadPath))
+	}
+
+	if showImage {
+		chromeOptions = append(chromeOptions,
+			chromedp.Flag("blink-settings", "imagesEnabled=true"),
+		)
+	} else {
+		chromeOptions = append(chromeOptions,
+			chromedp.Flag("blink-settings", "imagesEnabled=false"),
+		)
+	}
+	allocCtx, _ := chromedp.NewExecAllocator(baseCtx, chromeOptions...)
+	// 创建一个浏览器实例
+	incCtx, incCancelFn := chromedp.NewContext(allocCtx,
+		chromedp.WithLogf(log.Printf))
+	return &Browser{
+		incCtx, incCancelFn,
+	}
+}
+
+// 重置浏览器
+func (vm *VM) ResetBrowser() {
+	if vm.B != nil && vm.B.CancelFn != nil {
+		vm.B.CancelFn()
+		vm.B.Ctx = nil
+		vm.B.CancelFn = nil
+	}
+	var b *Browser
+	if vm.RunMode == run_on_device_local {
+		b = createLocalBrowser(vm.Headless, vm.ShowImage, vm.ProxyAddr, vm.DownloadPath)
+	} else {
+		b = createRemoteBrowser(vm.WsAddr)
+	}
+	if vm.B == nil {
+		vm.B = b
+	} else {
+		vm.B.Ctx, vm.B.CancelFn = b.Ctx, b.CancelFn
+	}
+}
+
+// BindLuaState 绑定虚拟机函数
+func (vm *VM) BindLuaState(state *lua.LState) {
+	state.SetGlobal("browser_reset", state.NewFunction(func(l *lua.LState) int {
+		vm.ResetBrowser()
+		return 0
+	}))
+	state.SetGlobal("browser_save", state.NewFunction(func(l *lua.LState) int {
+		spiderCode := l.ToString(-5)
+		siteName := l.ToString(-4)
+		siteChannelName := l.ToString(-3)
+		siteChannelUrl := l.ToString(-2)
+		table := l.ToTable(-1)
+		data := TableToMap(table)
+		vm.S.Save(spiderCode, siteName, siteChannelName, siteChannelUrl, data)
+		return 0
+	}))
+	state.SetGlobal("browser_url_last_segs", state.NewFunction(func(l *lua.LState) int {
+		segs := l.ToInt(-2)
+		href := l.ToString(-1)
+		if segs == 0 {
+			segs = 2
+		}
+		s := urlLastSegs(href, segs)
+		l.Push(lua.LString(s))
+		return 1
+	}))
+	//最多传10个string参数，不支持其他类型
+	state.SetGlobal("browser_log", state.NewFunction(func(l *lua.LState) int {
+		params := []string{}
+		for i := -10; i < 0; i++ {
+			p := l.ToString(i)
+			if p != "" {
+				params = append(params, p)
+			}
+		}
+		if sl != nil {
+			sl.Log(params...)
+		}
+		return 0
+	}))
+
+}
+
+// runScript 执行lua代码
+func (vm *VM) RunScript(script string) error {
+	defer Catch()
+
+	var state *lua.LState = lua.NewState()
+	defer state.Close()
+	//方法绑定
+	vm.ResetBrowser() //先创建浏览器对象
+	vm.BindLuaState(state)
+	vm.B.BindLuaState(state)
+	defer func() {
+		if vm.B != nil {
+			vm.B.Quit()
+		}
+	}()
+
+	reader := strings.NewReader(script)
+	chunk, err := parse.Parse(reader, "code")
+	if err != nil {
+		return err
+	}
+	proto, err := lua.Compile(chunk, script)
+	if err != nil {
+		return err
+	}
+	lfunc := state.NewFunctionFromProto(proto)
+	state.Push(lfunc)
+	state.Call(0, 0)
+
+	return nil
+}

program/tun/README.md

@@ -0,0 +1 @@
+socks5 代理跳板

program/tun/config.yaml

@@ -0,0 +1,10 @@
+# 管理地址
+monitor_address : ":20080"
+public_ip_address : "127.0.0.1"
+# ip参数设置
+connect_timeout : 90
+# 跳板端口范围配置（多个）
+portrange :
+  - 30000
+  - 30002
+

program/tun/main.go

@@ -0,0 +1,238 @@
+package main
+
+import (
+	"container/list"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"jycdp"
+	"log"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	tun_status_normal = iota
+	tun_status_busy
+	tun_status_idle
+)
+
+type (
+	//Springboard 跳板
+	Springboard struct {
+		TcpServeAddr string `yaml:"address"`
+		Cache        *list.List
+		Proxy        *IpItem //代理地址
+	}
+	//Config
+	Config struct {
+		MonitorAddress         string `yaml:"monitor_address"`
+		ConnectTimeout         int64  `yaml:"connect_timeout"`
+		SpringboardesPortRange [2]int `yaml:"portrange"`
+		PublicIpAddr           string `yaml:"public_ip_address"`
+	}
+	//流量监听 ，只监听请求端
+	Monitor struct {
+		s *Springboard
+	}
+	//
+	MyMux struct{}
+)
+
+// Write 实现写接口
+func (m *Monitor) Write(b []byte) (int, error) {
+	if strings.Contains(string(b), "pleasechangemyip.com") {
+		m.s.ChangeIp()
+	}
+	return len(b), nil
+}
+
+var (
+	configFile         = flag.String("c", "./config.yaml", "配置文件")
+	config     *Config = new(Config)
+	status     []int
+	tunLock    = new(sync.RWMutex)
+)
+
+// forward 流量转发
+func (s *Springboard) forward(src, dest io.ReadWriteCloser) {
+	defer func() {
+		if src != nil {
+			_ = src.Close()
+		}
+		if dest != nil {
+			_ = dest.Close()
+		}
+	}()
+	if src == nil || dest == nil {
+		return
+	}
+	_, _ = io.Copy(io.MultiWriter(&Monitor{s}, src), dest)
+}
+
+// processClient 处理客户端请求
+func (s *Springboard) ProcessClient(client net.Conn) {
+	defer func() {
+		if err := recover(); err != nil {
+			fmt.Printf("{err %v}", err)
+		}
+	}()
+	var targetConn net.Conn
+	var err error
+	for {
+		//默认复用上一次对可用IP
+		targetConn, err = net.DialTimeout("tcp",
+			s.Proxy.Raw,
+			time.Duration(config.ConnectTimeout)*time.Second)
+		if err == nil {
+			break
+		}
+		//TODO 移除有问题的链接
+		//roulette.ClearBadProxy(s.Proxy.Index)
+		//跳板主动切换IP
+		s.Proxy, _ = roulette.Get()
+	}
+	s.Cache.PushBack([2]net.Conn{client, targetConn})
+	go s.forward(client, targetConn)
+	go s.forward(targetConn, client)
+}
+
+// StartServe 开启TCP服务
+func (s *Springboard) StartServe() {
+	s.Proxy, _ = roulette.Get()
+	log.Println(s.TcpServeAddr, s.Proxy.Raw)
+	conn, err := net.Listen("tcp", s.TcpServeAddr)
+	if err != nil {
+		log.Println(err.Error())
+		return
+	}
+	log.Println("springboard listen:", s.TcpServeAddr)
+	for {
+		client, err := conn.Accept()
+		if err != nil {
+			log.Println("Listen failed: %v\n", err)
+			continue
+		}
+		go s.ProcessClient(client)
+	}
+}
+
+// ChangeIp 切换IP
+func (s *Springboard) ChangeIp() {
+	s.Proxy, _ = roulette.Get()
+	fmt.Print("C")
+	//fmt.Printf("(%s)", s.Proxy.Raw)
+	for e := s.Cache.Front(); e != nil; e = e.Next() {
+		connes, _ := e.Value.([2](net.Conn))
+		if connes[0].Close() != nil {
+			_ = connes[0].Close()
+		}
+		if connes[1].Close() != nil {
+			_ = connes[0].Close()
+		}
+	}
+}
+
+// Apply 批量申请代理端口资源，用于启动docker，传入proxy_server
+func Apply(length int) (ret []string, err error) {
+	tunLock.Lock()
+	defer tunLock.Unlock()
+	ret = make([]string, length, length)
+	pos, start := 0, config.SpringboardesPortRange[0]
+	tmp := make([]int, length, length)
+	for i, v := range status {
+		if v == tun_status_idle {
+			//status[i] = tun_status_busy
+			tmp[pos] = i
+			pos += 1
+		}
+		if pos == length {
+			break
+		}
+	}
+	if pos < length-1 {
+		return nil, errors.New("没有足够的tun资源")
+	}
+	//改状态
+	for i, v := range tmp {
+		status[v] = tun_status_busy
+		ret[i] = fmt.Sprintf("%s:%d", config.PublicIpAddr, start+v)
+	}
+	return
+}
+
+// Repay 归还释放tun资源
+func Repay(address []string) {
+	ports := make([]int, len(address), len(address))
+	for i, v := range address {
+		ports[i], _ = strconv.Atoi(strings.Split(v, ":")[1])
+	}
+	tunLock.Lock()
+	defer tunLock.Unlock()
+	start := config.SpringboardesPortRange[0]
+	for _, v := range ports {
+		status[v-start] = tun_status_idle
+	}
+}
+
+// monitor
+func monitor() {
+	//申请tun端口
+	http.HandleFunc("/apply", func(w http.ResponseWriter, r *http.Request) {
+		lenStr := r.FormValue("len")
+		if lenStr == "" {
+			lenStr = "1"
+		}
+		length, _ := strconv.Atoi(lenStr)
+		ret, err := Apply(length)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			fmt.Fprint(w, "没有足够的tun资源，请减少申请数量")
+		} else {
+			json.NewEncoder(w).Encode(ret)
+		}
+	})
+	http.HandleFunc("/repay", func(w http.ResponseWriter, r *http.Request) {
+		data := struct {
+			Data []string `json:"address"`
+		}{}
+		json.NewDecoder(r.Body).Decode(&data)
+		Repay(data.Data)
+		fmt.Fprint(w, "ok")
+	})
+	http.ListenAndServe(config.MonitorAddress, nil)
+}
+
+// init 初始化
+func init() {
+	flag.Parse()
+	jycdp.LoadConfig(*configFile, config)
+	length := config.SpringboardesPortRange[1] - config.SpringboardesPortRange[0] + 1
+	status = make([]int, length, length)
+	//全部设置为闲
+	for i, _ := range status {
+		status[i] = tun_status_idle
+	}
+	reloadProxies()
+	go watch()
+}
+
+// main
+func main() {
+	//
+	for i := config.SpringboardesPortRange[0]; i <= config.SpringboardesPortRange[1]; i++ {
+		s := &Springboard{
+			TcpServeAddr: fmt.Sprintf(":%d", i),
+			Cache:        list.New(),
+		}
+		go s.StartServe()
+	}
+	monitor()
+	//
+}

program/tun/proxyservice.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"time"
+)
+
+const (
+	watchInterval = 10
+	base64Table   = "ABNOPqrceQRSTklmUDEFGXYZabnopfghHVWdijstuvwCIJKLMxyz0123456789+/"
+)
+
+var (
+	coder = base64.NewEncoding(base64Table)
+)
+
+// reloadProxies 重新加载代理池
+func reloadProxies() {
+	fmt.Print("请求远程proxy service")
+	resp, err := http.Get("http://proxy.spdata.jianyu360.com/proxy/getallip")
+	if err != nil {
+		fmt.Print(err.Error())
+		return
+	}
+	bs, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Print(err.Error())
+		return
+	}
+	resp.Body.Close()
+	data := make([]*IpItem, 0, 0)
+	newPool := make([]*IpItem, 0, 0)
+	err = json.Unmarshal(bs, &data)
+	if err != nil {
+		fmt.Print(err.Error())
+		return
+	}
+	index := 0
+	//5分钟以内过期的都不要
+	lifeTimieout := time.Now().Unix() + 5*60
+	for _, v := range data {
+		fmt.Print(">")
+		if v.LifeTime <= lifeTimieout {
+			fmt.Print("O")
+			continue
+		}
+		rawIp, err := coder.DecodeString(v.Ip)
+		if err != nil {
+			continue
+		}
+		ipStr := string(rawIp)
+		v.Ip = ipStr
+		for _, p := range v.Ports {
+			addr := ipStr + ":" + p
+			if checkDial(addr) {
+				fmt.Print("+")
+				v.Raw = addr
+				break
+			} else {
+				fmt.Print("?")
+			}
+		}
+		if v.Raw != "" {
+			v.Index = index
+			index += 1
+			newPool = append(newPool, v)
+		}
+	}
+	if len(newPool) > 0 {
+		roulette.Init(newPool)
+		fmt.Printf("--{proxy size %d}--", roulette.Length)
+	} else {
+		fmt.Print("远程服务没有返回有效proxy")
+	}
+}
+
+// checkDial 检查是否能tcp连接通
+func checkDial(addr string) bool {
+	conn, err := net.DialTimeout("tcp", addr, 1*time.Second)
+	if err == nil {
+		conn.Close()
+	}
+	return err == nil
+}
+
+func watch() {
+	for {
+		time.Sleep(time.Duration(int64(watchInterval)) * time.Minute)
+		reloadProxies()
+	}
+}

program/tun/roulette.go

@@ -0,0 +1,74 @@
+// Proxy代理池时间轮盘切换
+package main
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type (
+	//IpItem
+	IpItem struct {
+		Ip       string   `json:"ip"`
+		LifeTime int64    `json:"lifetime"`
+		Ports    []string `json:"ports"`
+		IpType   int      `json:"iptype"` //proxy 服务器类型
+		Raw      string   //ip:port
+		Index    int
+	}
+	//
+	Roulette struct {
+		Data   []*IpItem
+		Length int32 //数据长度
+		Cursor int32
+		lock   *sync.RWMutex
+	}
+)
+
+var (
+	roulette = NewRoulette()
+)
+
+// NewRoulette
+func NewRoulette() *Roulette {
+	r := Roulette{lock: new(sync.RWMutex)}
+	return &r
+}
+
+// Init
+func (r *Roulette) Init(data []*IpItem) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+	r.Data = data
+	r.Length = int32(len(data))
+	atomic.StoreInt32(&r.Cursor, 0)
+}
+
+func (r *Roulette) findOne() *IpItem {
+	var pos int = 0
+	if atomic.CompareAndSwapInt32(&r.Cursor, r.Length-1, 0) {
+		pos = 0
+	} else {
+		pos = int(atomic.AddInt32(&r.Cursor, 1))
+	}
+	return r.Data[pos]
+}
+
+// 从时间轮中获取一个对象,不要近2分钟要过期的代理
+func (r *Roulette) Get() (*IpItem, error) {
+	for j := 0; j < 10000000; j++ {
+		for i := 0; i < int(r.Length); i++ {
+			if ip := r.findOne(); ip.LifeTime >= time.Now().Unix()+2*60 {
+				return ip, nil
+			} else {
+				fmt.Print("O")
+			}
+		}
+		fmt.Print("W")
+		time.Sleep(20 * time.Second)
+	}
+	return nil, errors.New("找不到合适的代理")
+}