xiugai

address/action.go

@@ -0,0 +1,452 @@
+/**
+ * 浏览器行为封装
+ */
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/chromedp/cdproto/network"
+
+	"github.com/chromedp/cdproto/browser"
+	"github.com/chromedp/cdproto/page"
+	"github.com/chromedp/chromedp"
+	"github.com/yuin/gopher-lua"
+)
+
+const (
+	selector_type_id        = 0
+	selector_type_query     = 1
+	selector_type_search    = 2
+	selector_type_jspath    = 3
+	selector_type_query_all = 4
+
+	execute_return_type_string = 0
+	execute_return_type_list   = 1
+	execute_return_type_table  = 2
+)
+
+// findTab 根据标题、url找tab
+func (b *Browser) findTabContext(tabTitle, tabUrl string, timeoutInt64 int64) (ctx context.Context, err error) {
+	if timeoutInt64 == 0 {
+		timeoutInt64 = 5000
+	}
+	timeout := time.Duration(timeoutInt64) * time.Millisecond
+	if tabTitle == "" && tabUrl == "" {
+		ctx, _ = context.WithTimeout(b.Ctx, timeout)
+		return ctx, nil
+	} else {
+		ts, err := chromedp.Targets(b.Ctx)
+		if err != nil {
+			return nil, err
+		}
+		for _, t := range ts {
+			if (tabTitle != "" && strings.Contains(t.Title, tabTitle)) || (tabUrl != "" && strings.Contains(t.URL, tabUrl)) {
+				// log.Printf("find tab param<title,url>: %s %s found %s %s", tabTitle, tabUrl,
+				// 	t.Title, t.URL)
+				newCtx, _ := chromedp.NewContext(b.Ctx, chromedp.WithTargetID(t.TargetID))
+				ctx, _ = context.WithTimeout(newCtx, timeout)
+				return ctx, nil
+			}
+		}
+	}
+	return nil, errors.New("can't find tab")
+}
+
+// CloseTabs关闭页面
+func (b *Browser) CloseTabs(tabTitle, tabUrl string, timeoutInt64 int64) (err error) {
+	if timeoutInt64 == 0 {
+		timeoutInt64 = 5
+	}
+	timeout := time.Duration(timeoutInt64) * time.Millisecond
+
+	ts, err := chromedp.Targets(b.Ctx)
+	if err != nil {
+		return err
+	}
+	for _, t := range ts {
+		if (tabTitle != "" && strings.Contains(t.Title, tabTitle)) || (tabUrl != "" && strings.Contains(t.URL, tabUrl)) {
+			newCtx, _ := chromedp.NewContext(b.Ctx, chromedp.WithTargetID(t.TargetID))
+			ctx, _ := context.WithTimeout(newCtx, timeout)
+			chromedp.Run(
+				ctx,
+				page.Close(),
+			)
+		}
+	}
+	return nil
+}
+
+// Navigate 导航到指定网址
+func (b *Browser) Navigate(tabTitle string, tabUrl string, isNewTab bool, targetUrl string, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	//新标签页
+	if isNewTab {
+		ctx, _ = chromedp.NewContext(ctx)
+	}
+	//
+	return chromedp.Run(ctx,
+		chromedp.Navigate(targetUrl))
+}
+
+// Navigate 导航到指定网址,并保存请求资源，如图片等
+func (b *Browser) NavigateAndSaveRes(tabTitle string, tabUrl string, timeout int64, isNewTab bool, targetUrl string, saveFileTypeList, save2dir string) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	//新标签页
+	if isNewTab {
+		ctx, _ = chromedp.NewContext(ctx)
+	}
+	//
+	saveFileType := strings.Split(saveFileTypeList, " ")
+	isNeedRes := func(fileType string) bool {
+		for _, v := range saveFileType {
+			if strings.Contains(fileType, v) {
+				return true
+			}
+		}
+		return false
+	}
+	fnURL2FileName := func(requestURL string) string {
+		u, err := url.Parse(requestURL)
+		if err != nil {
+			return ""
+		}
+		_, filename := filepath.Split(u.Path)
+		return filename
+	}
+	var cache = map[network.RequestID]string{}
+	chromedp.ListenTarget(ctx, func(v interface{}) {
+		switch ev := v.(type) {
+		case *network.EventRequestWillBeSent: //准备下载
+			cache[ev.RequestID] = ev.Request.URL
+		case *network.EventResponseReceived: //检查回应头的contenttype
+			contentType, _ := ev.Response.Headers["Content-Type"].(string)
+			fmt.Println(contentType)
+			if !isNeedRes(contentType) {
+				delete(cache, ev.RequestID)
+			}
+		case *network.EventLoadingFinished: //下载完成
+			if uri, ok := cache[ev.RequestID]; ok {
+				filename := fnURL2FileName(uri)
+				fmt.Println("save2file", filename)
+				if filename != "" {
+					filePath := filepath.Join(save2dir, filename)
+					var buf []byte
+					if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
+						var err error
+						buf, err = network.GetResponseBody(ev.RequestID).Do(ctx)
+						return err
+					})); err == nil {
+						os.WriteFile(filePath, buf, 0777)
+					} else {
+						fmt.Println(err.Error())
+					}
+				}
+			}
+		}
+	})
+
+	//
+	err = chromedp.Run(ctx,
+		chromedp.Navigate(targetUrl))
+	//下载存储
+	return err
+}
+
+// ExecuteJS 执行脚本
+func (b *Browser) ExecuteJS(tabTitle, tabUrl, script string, ret interface{}, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	return chromedp.Run(ctx,
+		chromedp.Evaluate(script, ret))
+}
+
+// Click 点击
+func (b *Browser) Click(tabTitle, tabUrl, selector string, selectorType int, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	var act chromedp.QueryAction
+	switch selectorType {
+	case selector_type_id:
+		act = chromedp.Click(selector, chromedp.ByID)
+	case selector_type_query:
+		act = chromedp.Click(selector, chromedp.ByQuery)
+	case selector_type_search:
+		act = chromedp.Click(selector, chromedp.BySearch)
+	case selector_type_jspath:
+		act = chromedp.Click(selector, chromedp.ByJSPath)
+	default:
+		act = chromedp.Click(selector, chromedp.ByQueryAll)
+	}
+
+	err = chromedp.Run(ctx,
+		act)
+	return err
+
+}
+
+// KeySend 键盘输入
+func (b *Browser) KeySend(tabTitle, tabUrl, selector, sendStr string, selectorType int, timeout int64) (err error) {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	var act chromedp.QueryAction
+	switch selectorType {
+	case selector_type_id:
+		act = chromedp.SendKeys(selector, sendStr, chromedp.ByID)
+	case selector_type_query:
+		act = chromedp.SendKeys(selector, sendStr, chromedp.ByQuery)
+	case selector_type_search:
+		act = chromedp.SendKeys(selector, sendStr, chromedp.BySearch)
+	case selector_type_jspath:
+		act = chromedp.SendKeys(selector, sendStr, chromedp.ByJSPath)
+	default:
+		act = chromedp.SendKeys(selector, sendStr, chromedp.ByQueryAll)
+	}
+	return chromedp.Run(ctx,
+		act)
+}
+
+// WaitVisible 等待元素可见
+func (b *Browser) WaitVisible(tabTitle, tabUrl, selector string, selectorType int, timeout int64) error {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	var act chromedp.QueryAction
+	switch selectorType {
+	case selector_type_id:
+		act = chromedp.WaitVisible(selector, chromedp.ByID)
+	case selector_type_query:
+		act = chromedp.WaitVisible(selector, chromedp.ByQuery)
+	case selector_type_search:
+		act = chromedp.WaitVisible(selector, chromedp.BySearch)
+	case selector_type_jspath:
+		act = chromedp.WaitVisible(selector, chromedp.ByJSPath)
+	default:
+		act = chromedp.WaitVisible(selector, chromedp.ByQueryAll)
+	}
+	return chromedp.Run(ctx,
+		act)
+}
+
+// 重置浏览器
+func (b *Browser) Reset() {
+
+}
+
+// DownloadFile 只有在非headless模式下有效，与click方法其实是一致的
+func (b *Browser) DownloadFile(tabTitle, tabUrl string, timeout int64, selector string, selectorType int, save2dir string) error {
+	ctx, err := b.findTabContext(tabTitle, tabUrl, timeout)
+	if err != nil {
+		return err
+	}
+	var act chromedp.QueryAction
+	switch selectorType {
+	case selector_type_id:
+		act = chromedp.Click(selector, chromedp.ByID)
+	case selector_type_query:
+		act = chromedp.Click(selector, chromedp.ByQuery)
+	case selector_type_search:
+		act = chromedp.Click(selector, chromedp.BySearch)
+	case selector_type_jspath:
+		act = chromedp.Click(selector, chromedp.ByJSPath)
+	default:
+		act = chromedp.Click(selector, chromedp.ByQueryAll)
+	}
+
+	return chromedp.Run(ctx,
+		browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorAllowAndName).WithDownloadPath(save2dir).WithEventsEnabled(true),
+		act)
+}
+
+// BindLuaState
+func (b *Browser) BindLuaState(state *lua.LState) {
+	//执行暂停
+	state.SetGlobal("browser_sleep", state.NewFunction(func(l *lua.LState) int {
+		fmt.Println("---browser_sleep---")
+		timeout := l.ToInt64(-1)
+		if timeout == 0 {
+			timeout = 5
+		}
+		time.Sleep(time.Duration(timeout) * time.Millisecond)
+		return 0
+	}))
+	//关闭tabl页
+	state.SetGlobal("browser_closetabs", state.NewFunction(func(l *lua.LState) int {
+		fmt.Println("---browser_closetabs---")
+		timeout := l.ToInt64(-3)
+		tabTitle := l.ToString(-2)
+		tabUrl := l.ToString(-1)
+		if timeout == 0 {
+			timeout = 5
+		}
+		b.CloseTabs(tabTitle, tabUrl, timeout)
+		return 0
+	}))
+	//注册打开地址
+	state.SetGlobal("browser_navagite", state.NewFunction(func(l *lua.LState) int {
+		fmt.Println("---browser_navagite---")
+		tabTitle := l.ToString(-5)  //指定标签页title
+		tabUrl := l.ToString(-4)    //指定标签页url
+		isNewTab := l.ToBool(-3)    //是否打开新的标签页
+		timeout := l.ToInt64(-2)    //网页打开的超时时间
+		targetUrl := l.ToString(-1) //打开网页的链接
+		if err := b.Navigate(tabTitle, tabUrl, isNewTab, targetUrl, timeout); err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+	//执行浏览器端js
+	state.SetGlobal("browser_executejs", state.NewFunction(func(l *lua.LState) int {
+		fmt.Println("---browser_executejs---")
+		tabTitle := l.ToString(-5)
+		tabUrl := l.ToString(-4)
+		timeout := l.ToInt64(-3)
+		returnType := l.ToInt(-2) //返回数据类型
+		script := l.ToString(-1)  //执行的js
+
+		switch returnType {
+		case execute_return_type_string: //返回string
+			var ret string
+			if err := b.ExecuteJS(tabTitle, tabUrl, script, &ret, timeout); err == nil {
+				l.Push(lua.LString("ok"))
+				l.Push(lua.LString(ret))
+			} else {
+				l.Push(lua.LString("err"))
+				l.Push(lua.LString(err.Error()))
+			}
+		case execute_return_type_list: //返回list
+			var ret = make([]interface{}, 0, 0)
+			var tmp = make(map[string]interface{})
+			if err := b.ExecuteJS(tabTitle, tabUrl, script, &ret, timeout); err == nil {
+				for i, v := range ret {
+					tmp[strconv.Itoa(i)] = v
+				}
+				l.Push(lua.LString("ok"))
+				l.Push(MapToTable(tmp))
+			} else {
+				l.Push(lua.LString("err"))
+				l.Push(lua.LString(err.Error()))
+			}
+		case execute_return_type_table: //返回table
+			var ret = make(map[string]interface{})
+			if err := b.ExecuteJS(tabTitle, tabUrl, script, &ret, timeout); err == nil {
+				l.Push(lua.LString("ok"))
+				l.Push(MapToTable(ret))
+			} else {
+				l.Push(lua.LString("err"))
+				l.Push(lua.LString(err.Error()))
+			}
+		}
+		return 2
+	}))
+	//按键
+	state.SetGlobal("browser_keysend", state.NewFunction(func(l *lua.LState) int {
+		fmt.Println("---browser_keysend---")
+		tabTitle := l.ToString(-6)
+		tabUrl := l.ToString(-5)
+		timeout := l.ToInt64(-4)
+		words := l.ToString(-3)
+		selectorType := l.ToInt(-2)
+		selector := l.ToString(-1)
+		fmt.Println(selector, words, selectorType, timeout)
+		err := b.KeySend(tabTitle, tabUrl, selector, words, selectorType, timeout)
+		if err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+	//点击
+	state.SetGlobal("browser_click", state.NewFunction(func(l *lua.LState) int {
+		fmt.Println("---browser_click---")
+		tabTitle := l.ToString(-5)
+		tabUrl := l.ToString(-4)
+		timeout := l.ToInt64(-3)
+		selectorType := l.ToInt(-2)
+		selector := l.ToString(-1)
+
+		err := b.Click(tabTitle, tabUrl, selector, selectorType, timeout)
+		if err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+
+	state.SetGlobal("browser_waitvisible", state.NewFunction(func(l *lua.LState) int {
+		fmt.Println("---browser_waitvisible---")
+		tabTitle := l.ToString(-5)
+		tabUrl := l.ToString(-4)
+		timeout := l.ToInt64(-3)
+		selectorType := l.ToInt(-2) //选择器类型
+		selector := l.ToString(-1)  //选择器
+
+		err := b.WaitVisible(tabTitle, tabUrl, selector, selectorType, timeout)
+		if err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+
+	//点击
+	state.SetGlobal("browser_downloadfile", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-6)
+		tabUrl := l.ToString(-5)
+		timeout := l.ToInt64(-4)
+		selectorType := l.ToInt(-3)
+		selector := l.ToString(-2)
+		save2dir := l.ToString(-1)
+
+		err := b.DownloadFile(tabTitle, tabUrl, timeout, selector, selectorType, save2dir)
+		if err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+
+	//注册打开地址
+	state.SetGlobal("browser_navagite_download_res", state.NewFunction(func(l *lua.LState) int {
+		tabTitle := l.ToString(-7)
+		tabUrl := l.ToString(-6)
+		timeout := l.ToInt64(-5)
+		isNewTab := l.ToBool(-4)
+		targetUrl := l.ToString(-3)
+		saveFileTypeList := l.ToString(-2)
+		savedir := l.ToString(-1)
+		if err := b.NavigateAndSaveRes(tabTitle, tabUrl, timeout, isNewTab, targetUrl, saveFileTypeList, savedir); err != nil {
+			l.Push(lua.LString(err.Error()))
+		} else {
+			l.Push(lua.LString("ok"))
+		}
+		return 1
+	}))
+}

address/go.mod

@@ -1,3 +1,30 @@
 module address
 
 go 1.21.0
+
+require (
+	github.com/chromedp/cdproto v0.0.0-20240512230644-b3296df1660c
+	github.com/chromedp/chromedp v0.9.5
+	github.com/yuin/gopher-lua v1.1.1
+	go.mongodb.org/mongo-driver v1.15.0
+)
+
+require (
+	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/gobwas/httphead v0.1.0 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.3.2 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+)

address/go.sum

@@ -0,0 +1,78 @@
+github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
+github.com/chromedp/cdproto v0.0.0-20240512230644-b3296df1660c h1:IrHOOrmmJtVS1Z7tW+z71ZHTe6nYUqARg19Od8ECsJg=
+github.com/chromedp/cdproto v0.0.0-20240512230644-b3296df1660c/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
+github.com/chromedp/chromedp v0.9.5 h1:viASzruPJOiThk7c5bueOUY91jGLJVximoEMGoH93rg=
+github.com/chromedp/chromedp v0.9.5/go.mod h1:D4I2qONslauw/C7INoCir1BJkSwBYMyZgx8X276z3+Y=
+github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
+github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
+github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
+github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
+github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.3.2 h1:zlnbNHxumkRvfPWgfXu8RBwyNR1x8wh9cf5PTOCqs9Q=
+github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
+github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
+github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
+github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc=
+go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

address/main.go

@@ -6,81 +6,78 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"strconv"
+	"time"
 )
 
-// 定义百度API响应的结构体
-type BaiduGeocodingResponse struct {
-	Status  int    `json:"status"`
-	Message string `json:"message"`
-	Result  Result `json:"result"`
+type NominationResponse struct {
+	Latitude  string `json:"lat"`
+	Longitude string `json:"lon"`
 }
 
-type Result struct {
-	Location Location `json:"location"`
-	Address  string   `json:"formatted_address"`
-}
-
-type Location struct {
-	Lng float64 `json:"lng"`
-	Lat float64 `json:"lat"`
-}
+func main() {
+	// 创建一个自定义的 Transport 实例
+	transport := &http.Transport{
+		Proxy: func(req *http.Request) (*url.URL, error) {
+			// return url.Parse("http://127.0.0.1:1081")
+			return url.Parse("socks5://127.0.0.1:7890")
+		},
+	}
 
-// 调用百度API获取经纬度的函数
-func getLatLngFromBaiduAPI(address string, apiKey string) (*Location, error) {
-	// 百度API的基础URL
-	baseURL := "http://api.map.baidu.com/geocoding/v3/"
+	// 创建一个自定义的 Client 实例
+	client := &http.Client{
+		Transport: transport,       // 设置代理
+		Timeout:   time.Second * 3, // 设置超时
+	}
+	// 待转换的地址
+	address := "北京市海淀区清华大学"
 
-	// 构造请求参数
+	// 构造 Nominatim API 的 URL
 	params := url.Values{}
-	params.Set("address", address)
-	params.Set("output", "json")
-	params.Set("ak", apiKey) // 使用你的API密钥
-
-	// 拼接完整的URL
-	urlStr := baseURL + "?" + params.Encode()
-
-	// 发起HTTP GET请求
-	resp, err := http.Get(urlStr)
+	params.Set("q", address)
+	params.Set("format", "json")
+	params.Set("limit", "1")
+	params.Set("addressdetails", "1")
+	params.Set("namedetails", "1")
+
+	urlStr := "https://nominatim.openstreetmap.org/search?" + params.Encode()
+	// 发送 GET 请求
+	resp, err := client.Get(urlStr)
 	if err != nil {
-		return nil, err
+		// 处理错误
+		fmt.Println("发生了错误:err", err)
+		return
 	}
 	defer resp.Body.Close()
 
-	// 读取响应体
+	// 读取响应内容
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return nil, err
-	}
-
-	// 解析JSON响应
-	var response BaiduGeocodingResponse
-	if err := json.Unmarshal(body, &response); err != nil {
-		return nil, err
-	}
-
-	// 检查状态码
-	if response.Status != 0 {
-		return nil, fmt.Errorf("geocoding failed: %s", response.Message)
+		fmt.Println("读取响应失败:", err)
+		return
 	}
 
-	// 返回经纬度
-	return &response.Result.Location, nil
-}
-
-func main() {
-	// 替换为你的API密钥
-	apiKey := "iRhAwfK725GcVy7WJmLv6Twtw85HYkl1"
-
-	// 要查询的地址
-	address := "北京市朝阳区三里屯"
-
-	// 调用函数获取经纬度
-	location, err := getLatLngFromBaiduAPI(address, apiKey)
+	// 解析 JSON 格式的响应数据
+	var results []NominationResponse
+	err = json.Unmarshal(body, &results)
 	if err != nil {
-		fmt.Printf("Error getting latitude and longitude: %v\n", err)
+		fmt.Println("解析响应数据失败:", err)
 		return
 	}
 
-	// 打印结果
-	fmt.Printf("Latitude: %f, Longitude: %f\n", location.Lat, location.Lng)
+	// 如果有结果
+	if len(results) > 0 {
+		result := results[0]
+		// 打印转换后的经纬度
+		fmt.Println("经度:", result.Longitude)
+		fmt.Println("纬度:", result.Latitude)
+		lo, _ := strconv.ParseFloat(result.Longitude, 64)
+		la, _ := strconv.ParseFloat(result.Latitude, 64)
+
+		// 格式化为保留6位小数的字符串
+		strs := fmt.Sprintf("%.6f-%.6f", lo, la)
+		fmt.Println(strs)
+	} else {
+		fmt.Println("没有找到匹配的地址信息")
+	}
 }

address/storage.go

@@ -0,0 +1,98 @@
+/*
+存储;
+暂时提供3种存储方式：
+
+1）控制台
+2） MongoDb
+3) 文件
+*/
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type (
+	//Storage
+	Storage interface {
+		//保存数据
+		Save(spiderCode, siteName, siteChannelName, siteChannelUrl string, data map[string]interface{}) error
+	}
+
+	//MongoStorage
+	MongoStorage struct {
+		c *mongo.Collection
+	}
+	//调试模式下的数据存储
+	ConsoleStorage struct {
+	}
+	//FileStorage
+	FileStorage struct {
+		f io.WriteCloser
+	}
+)
+
+// NewMongoStorage
+func NewMongoStorage(mongoURI, db, collection string) *MongoStorage {
+	client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoURI))
+	if err != nil {
+		panic(err)
+	}
+	return &MongoStorage{
+		c: client.Database(db).Collection(collection),
+	}
+
+}
+
+// NewDebugerStorage
+func NewConsoleStorage() *ConsoleStorage {
+	return &ConsoleStorage{}
+}
+
+// NewFileStorage
+func NewFileStorage(f io.WriteCloser) *FileStorage {
+	return &FileStorage{f: f}
+}
+
+// Save
+func (ms *MongoStorage) Save(spiderCode, siteName, siteChannelName, siteChannelUrl string, data map[string]interface{}) error {
+	data["spider_code"] = spiderCode
+	data["site_name"] = siteName
+	data["site_channel_name"] = siteChannelName
+	data["site_channel_url"] = siteChannelUrl
+	_, err := ms.c.InsertOne(context.TODO(), data)
+	return err
+}
+
+// Save
+func (ds *ConsoleStorage) Save(spiderCode, siteName, siteChannelName, siteChannelUrl string, data map[string]interface{}) error {
+	data["spider_code"] = spiderCode
+	data["site_name"] = siteName
+	data["site_channel_name"] = siteChannelName
+	data["site_channel_url"] = siteChannelUrl
+	fmt.Println(" data=")
+	for k, v := range data {
+		fmt.Printf("\t%s = %v \n", k, v)
+	}
+	return nil
+}
+
+// Save
+func (fs *FileStorage) Save(spiderCode, siteName, siteChannelName, siteChannelUrl string, data map[string]interface{}) error {
+	data["spider_code"] = spiderCode
+	data["site_name"] = siteName
+	data["site_channel_name"] = siteChannelName
+	data["site_channel_url"] = siteChannelUrl
+	buf := new(bytes.Buffer)
+	json.NewEncoder(buf).Encode(data)
+	fs.f.Write([]byte(","))
+	fs.f.Write(buf.Bytes())
+	return nil
+}

address/test.lua

@@ -0,0 +1,28 @@
+function getLngLat(c)
+	local href = "http://www.daquan.la/jingwei/"
+	browser_navagite(true,1000*60*120, href)
+	browser_keysend(href,1000*30,0,"address",c.."\n")
+	--todo 执行等待
+	local ready=false
+	repeat
+		local ok,lng = browser_executejs(href,1000,0,'document.querySelector("#lng").value')
+		if ok=="ok" and lng~="等待解析" then
+			ready=true
+		else
+			browser_sleep(200)
+		end
+	until ready
+	--获取数据
+	local ok,lnglat = browser_executejs(href,1000*30,2,[[
+		var lnglat={};
+		lnglat["lng"]=document.querySelector("#lng").value;
+		lnglat["lat"]=document.querySelector("#lat").value;
+		lnglat
+	]])
+	if ok=="ok" and lnglat~=nil then
+		browser_save("", "", "", "", { ["company"] = c, ["经度"] = lnglat.lng, ["维度"] = lnglat.lat})
+	end
+	browser_closetabs_without("about:blank",1000)
+	return lnglat.lng, lnglat.lat
+end
+getLngLat("郑州大学")

address/util.go

@@ -0,0 +1,131 @@
+/**
+ * 工具包封装
+ */
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/yuin/gopher-lua"
+)
+
+// Catch 异常崩溃拦截
+func Catch() {
+	if err := recover(); err != nil {
+		log.Println(err)
+	}
+}
+
+// MapToTable converts a Go map to a lua table
+func MapToTable(m map[string]interface{}) *lua.LTable {
+	// Main table pointer
+	resultTable := &lua.LTable{}
+
+	// Loop map
+	for key, element := range m {
+
+		switch element.(type) {
+		case float64:
+			resultTable.RawSetString(key, lua.LNumber(element.(float64)))
+		case int64:
+			resultTable.RawSetString(key, lua.LNumber(element.(int64)))
+		case string:
+			resultTable.RawSetString(key, lua.LString(element.(string)))
+		case bool:
+			resultTable.RawSetString(key, lua.LBool(element.(bool)))
+		case []byte:
+			resultTable.RawSetString(key, lua.LString(string(element.([]byte))))
+		case map[string]interface{}:
+
+			// Get table from map
+			tble := MapToTable(element.(map[string]interface{}))
+
+			resultTable.RawSetString(key, tble)
+
+		case time.Time:
+			resultTable.RawSetString(key, lua.LNumber(element.(time.Time).Unix()))
+
+		case []map[string]interface{}:
+
+			// Create slice table
+			sliceTable := &lua.LTable{}
+
+			// Loop element
+			for _, s := range element.([]map[string]interface{}) {
+
+				// Get table from map
+				tble := MapToTable(s)
+
+				sliceTable.Append(tble)
+			}
+
+			// Set slice table
+			resultTable.RawSetString(key, sliceTable)
+
+		case []interface{}:
+
+			// Create slice table
+			sliceTable := &lua.LTable{}
+
+			// Loop interface slice
+			for _, s := range element.([]interface{}) {
+
+				// Switch interface type
+				switch s.(type) {
+				case map[string]interface{}:
+
+					// Convert map to table
+					t := MapToTable(s.(map[string]interface{}))
+
+					// Append result
+					sliceTable.Append(t)
+
+				case float64:
+
+					// Append result as number
+					sliceTable.Append(lua.LNumber(s.(float64)))
+
+				case string:
+
+					// Append result as string
+					sliceTable.Append(lua.LString(s.(string)))
+
+				case bool:
+
+					// Append result as bool
+					sliceTable.Append(lua.LBool(s.(bool)))
+				}
+			}
+
+			// Append to main table
+			resultTable.RawSetString(key, sliceTable)
+		}
+	}
+
+	return resultTable
+}
+
+// TabletoMap converts a lua table to go map
+func TableToMap(t *lua.LTable) map[string]interface{} {
+	ret := make(map[string]interface{})
+	t.ForEach(func(k, v lua.LValue) {
+		key := fmt.Sprint(k)
+		if val, ok := v.(*lua.LTable); ok {
+			ret[key] = TableToMap(val)
+		} else {
+			if val, ok := v.(lua.LString); ok {
+				ret[key] = string(val)
+			} else if val, ok := v.(lua.LNumber); ok {
+				ret[key] = int64(val)
+			} else if val, ok := v.(lua.LBool); ok {
+				ret[key] = bool(val)
+			} else {
+				ret[key] = fmt.Sprint(v)
+			}
+		}
+	})
+	return ret
+}

address/util/baidu_sem.lua

@@ -0,0 +1,41 @@
+--[[
+ 	百度刷排名脚本
+]]--
+local timeout = 360
+local keywords={"剑鱼标讯","北京剑鱼","剑鱼招标"}
+local waitting_click_links_js = [[
+var ns=[];
+document.querySelectorAll("div.result.c-container").forEach((v,i)=>{  
+	if (v.innerText.indexOf("广告")==-1 && Math.random() >0.5 ){  
+	ns.push({"index":i,"title":v.querySelector("h3").innerText}); 
+   }    
+});
+ns
+]]
+
+for k,v in pairs(keywords) do
+	-- 1. 先暂停一下，刷新一下IP
+	browser_sleep(1)
+	browser_navagite(timeout,"http://pleasechangemyip.com")
+	-- 2. 打开百度
+	browser_navagite(timeout,"https://www.baidu.com")
+	-- 检查是否弹出百度身份验证
+	local ok,browser_title = browser_executejs(timeout,0,"document.title")
+	if ok=="ok" and string.find(browser_title, "安全", 1)==nil then
+		-- 3. 输入关键词
+		browser_keysend(timeout,v.."\n","#kw",0)
+		browser_sleep(5)
+		-- 4. 点击对我们有利的链接
+		local ok,list = browser_executejs(timeout,1,waitting_click_links_js)
+		if ok=="ok" and list~=nil then
+			for k2,v2 in pairs(list) do
+				local selector='document.querySelectorAll("div.result.c-container")['..v2['index']..'].querySelector("a")'
+				browser_click(timeout,3,selector)
+			end
+		end
+		browser_sleep(5)
+	end
+	--browser_closetabs()
+	-- 5.重置浏览器，避免打开多TAB，即便主动关闭TAB，也会引起浏览器问题
+	browser_reset()
+end

address/util/bing_sem.lua

@@ -0,0 +1,61 @@
+local timeout=60
+local words = {"剑鱼标讯","剑鱼标讯官网","剑鱼标讯平台","剑鱼招标","剑鱼招标网","北京剑鱼","剑鱼标讯入口"}
+local href="https://cn.bing.com"
+
+local filter_ad_js = [[
+var ns=[];
+//ol#b_results li.b_ad ul li h2 广告的selector
+document.querySelectorAll("ol#b_results li.b_ad ul li h2").forEach((v,i)=>{  
+	//if(Math.random()>0.1){
+		v.setAttribute("taget","_blank");
+		ns.push(i)  
+	//}
+});ns
+]]
+for k,v in pairs(words) do
+	-- 1. 切换IP
+	browser_navagite(timeout,"http://pleasechangemyip.com")
+	browser_sleep(1)
+	-- 1.1 随机循环几次，造成1个IP访问多次的假象
+	local amount,index = math.random(1,3),1
+	repeat
+		local click_ad_flag = false
+		print(v.." 第"..index.."/"..amount.." 轮操作")
+		-- 2. 打开bing
+		local ok = browser_navagite(timeout,href)
+		if ok=="ok" then
+			browser_sleep(1)
+			-- 3. 输入关键词
+			browser_keysend(timeout,v.."\n","#sb_form_q",0)
+			browser_sleep(2)
+			browser_waitvisible(30,"#b_tween_searchResults",0)
+			-- 4. 找到链接，点击
+			local ad_amount,click_count = 0,0
+			local ok,list = browser_executejs(timeout,1,filter_ad_js)
+			--print(ok)
+			--print(list)
+			if ok == "ok" and list~=nil then
+				for _,v2 in pairs(list) do
+					local selector='document.querySelectorAll("ol#b_results li.b_ad ul li h2 a")['..v2..']'
+					--print(selector)
+					ad_amount=amount+1
+					--一定概率点击广告
+					if math.random(1,10)>3 then
+						click_ad_flag = true
+						browser_click(timeout,3,selector)
+						click_count=click_count+1
+					end
+				end
+			end
+			print("\t 本页面有广告"..ad_amount.."个 点击"..click_count.."次")
+			if click_ad_flag then
+				browser_sleep(20)
+			end
+			browser_reset()
+		else
+			print(ok)
+			browser_navagite(timeout,"http://pleasechangemyip.com")
+		end
+		index=index+1
+	until(index>=amount)
+end

address/util/hb.lua

@@ -0,0 +1,26 @@
+local href = "http://www.hebzfcgwssc.com/Mall/HeBei/gyssearch.aspx"
+local listjs = [[var ns=[];document.querySelectorAll("div.gystable table.gys-border-top tbody tr td.gys-img-tt").forEach((v,i)=>{ns.push({"index":i,"href":'http://www.hebzfcgwssc.com/Mall/HeBei/gysmain.aspx?sid='+v.getAttribute("data")})});ns]]
+--local listjs = [[var ns=[];document.querySelectorAll("div.gystable table.gys-border-top tbody tr").forEach((v,i)=>{ns.push({"index":i,"href":'http://www.hebzfcgwssc.com/Mall/HeBei/gysmain.aspx?sid='+v.getAttribute("data")})});ns]]
+
+local coverclickjs = [[$(".gys-border-top").off('click').on("click", ".gys-img-tt", function() {window.open("gysmain.aspx?sid=" + $(this).parent().attr("data"))});]]
+
+local timeout = 1000*60*60
+--打开列表页
+browser_navagite(timeout,href)--timeout:页面打开时间
+--等待元素加载完毕
+browser_waitvisible("body > div.container.mt20 > div > div.gystable")
+--执行js获取可点击td
+local ok,list = browser_executejs(2000,1,listjs)
+if ok=="ok" then
+    for k,item in pairs(list) do
+        --修改网站自带的点击事件（注入js）
+        browser_executejs(2,coverclickjs)
+        browser_sleep(1000)
+        local selector = 'document.querySelectorAll("table.gys-border-top tr td.gys-img-tt")['..item["index"]..']'
+        print("111111",selector)
+        --browser_navagite(true,timeout,item['href'])
+        browser_click(10000,3,selector)--点击打开页面5秒
+        browser_waitvisible("/gysmain.aspx",timeout,1,"div.input-area-gys")--无效等待加载 点击后页面地址栏不存在/gysmain.aspx
+        browser_closetabs(timeout,"","/gysmain.aspx")
+    end
+end

address/util/hb1.lua

@@ -0,0 +1,59 @@
+-- 河北省政府采购网电子商城
+--
+
+--供应商页面
+local channel_list = {
+	'http://www.hebzfcgwssc.com/Mall/HeBei/gyssearch.aspx?page=1&gysname='
+}
+local item_selector_js = [[var ns=[];document.querySelectorAll("div.gystable table.gys-border-top tbody tr").forEach((v,i)=>{ns.push({"index":i,"href":'http://www.hebzfcgwssc.com/Mall/HeBei/gysmain.aspx?sid='+v.getAttribute("data")})});ns]]
+local has_next_page_js=[[var hasNextPage="true";if(document.querySelector("input.next.disable")){hasNextPage="false"};hasNextPage]]
+
+--三级页输入商品
+local input_item_js = [[var input = document.getElementsByClassName("input-floor-gys")[0];input.value = "Hello World";]]
+
+local timeout = 1000*60*60*8 --任务执行最大时间8小时
+--获取详情页数据
+function get_detail_text()
+	-- 等待详情页加载完成
+	print('#################')
+	--browser_waitvisible("/gysmain.aspx",timeout,1,"div.input-area-gys")
+	--browser_closetabs(1000,"","/gysmain.aspx")
+	--browser_executejs("/gysmain.aspx",timeout,1,input_item_js)
+	--browser_click("sid",timeout,1,"button.shop")
+	--browser_sleep(5000)
+
+	print("--准备获取代理商信息")
+end
+
+--主体架构 遍历所有栏目
+for k,v in pairs(channel_list) do
+	local pageno=1
+	browser_sleep(100)
+	browser_navagite(timeout,v)
+	print('打开供应商页')
+	-- 数据获取
+	local ok,list = browser_executejs(timeout,1,item_selector_js)
+	if ok=="ok" then
+		for _,item in pairs(list) do
+			print('打开第'..item["index"]..'个')
+			browser_navagite(true,timeout,item['href'])
+			--browser_sleep(2000)
+			print('444#################')
+			get_detail_text()
+			--关闭详情页
+			browser_closetabs(timeout,"","/gysmain.aspx")
+			browser_sleep(10000)
+			--关闭详情页
+			--browser_closetabs(timeout,"","sid")
+		end
+	else
+		print(list)
+	end
+	pageno = pageno+1
+	browser_sleep(10000)
+	browser_reset()
+end
+
+
+-- 切换IP方法
+-- browser_navagite(120,"http://pleasechangemyip.com")

address/util/hcy.lua

@@ -0,0 +1,101 @@
+-- 北京华采云数据采集 
+-- 界面点击操作，需要延时等待，至少200毫秒
+local channel_list = {"http://mkt-bjzc.zhongcy.com/mall-view/product/search?businessType=1&cid=1000990&sourceFlag=0001",
+"http://mkt-bjzc.zhongcy.com/mall-view/product/search?businessType=1&cid=1000988&sourceFlag=0001",
+"http://mkt-bjzc.zhongcy.com/mall-view/product/search?businessType=1&cid=1001069&sourceFlag=0001",
+"http://mkt-bjzc.zhongcy.com/mall-view/product/search?businessType=1&cid=1000989&sourceFlag=0001"}
+
+local item_selector_js = [[var ns=[];document.querySelectorAll("div.product-list-order div.product-list div.product-pic a").forEach((v,i)=>{ns.push({"index":i,"href":v.getAttribute("href")})});ns]]
+local content_selector_js = [[var fn=function(s){let obj = document.querySelector(s);if(obj){return obj.innerText}else{return ""}}; ret={"name":fn("div#productView h2#itemName"),"price":fn("div#productView div#sellPriceView"),"company":fn("div#gc-goods-supplier"),"tel":fn("div[type=page] div.layui-layer-content"),"brand":fn("div#productView :nth-child(4) div"),"producer":fn("div#productView :nth-child(5) div")};ret]]
+local has_next_page_js=[[var hasNextPage="true";if(document.querySelector("div#pageList a.layui-laypage-next.layui-disabled")){hasNextPage="false"};hasNextPage]]
+local has_agent_next_page_js=[[var hasNextPage="true";if(document.querySelector("div#gc-detail-supplier-paging a.layui-laypage-next.layui-disabled")){hasNextPage="false";};hasNextPage]]
+local agent_list_js=[[var ns=[];document.querySelectorAll("div#gc-tab-agent table tbody").forEach((v,i)=>{ns.push(i)});ns]]
+local agent_content_js=[[var ret="";var obj=document.querySelector("div[type=page] div.layui-layer-content");if(obj){ret=obj.innerText};ret]]
+local timeout = 1000*60*60*8 --任务执行最大时间8小时
+
+
+--获取详情页数据
+function get_detail_content()
+    -- 等待详情页加载完成
+    browser_waitvisible("product/detail",timeout,1,"div#gc-goods-supplier i")
+    -- 1.弹出公司联系方式
+    browser_click("product/detail",timeout,1,"div#gc-goods-supplier i")
+    browser_sleep(100)
+    local ok,data = browser_executejs("product/detail",timeout,2,content_selector_js)
+    --关闭飘窗
+    browser_click("product/detail",timeout,1,"div[type=page] a.layui-layer-close")
+    browser_sleep(500)
+    print("--准备获取代理商信息")
+    if ok=="ok" then
+        -- 2.获取所有的商品供应商联系方式
+        browser_click("product/detail",timeout,1,"div#item-tab-card ul :nth-child(2)")
+        browser_sleep(200)
+        -- 这里也需要翻页
+        local has_agent_next_page="true"
+        local agent_tels={}
+        repeat
+            local ok2,list2 = browser_executejs("product/detail",timeout,1,agent_list_js)
+            if ok2=="ok" then
+	            for _,agent_no in pairs(list2) do
+	                -- 点击电话小图标
+	                local agent_tel_selector = "div#gc-tab-agent table :nth-child("..(agent_no+2)..") a.customer-tel1 i"
+	                browser_click("product/detail",timeout,1,agent_tel_selector)
+	                browser_sleep(200)
+	                local _,agent_tel = browser_executejs("product/detail",timeout,0,agent_content_js)
+	                browser_click("product/detail",timeout,1,"div[type=page] a.layui-layer-close")
+	                print(agent_tel)
+	                browser_sleep(200)
+	                table.insert(agent_tels,agent_tel)
+	            end
+				print("--代理商翻页")
+	             _,has_agent_next_page=browser_executejs("product/detail",timeout,0,has_agent_next_page_js)
+	            if has_agent_next_page=="true" then
+	                browser_click("product/detail",timeout,1,"div#gc-detail-supplier-paging a.layui-laypage-next")
+	                browser_sleep(200)
+	            end
+           	else
+				has_agent_next_page="false"
+			end
+        until has_agent_next_page=="false"
+        data["agent_tels"]=agent_tels
+        browser_save("spider001","北京京华云采","","",data)
+    end
+end
+
+--主体架构 遍历所有栏目
+for k,v in pairs(channel_list) do
+	local has_next_page = "true"
+	local pageno=1
+	repeat --翻页到没有下一页为止
+		browser_sleep(100)
+		if pageno == 1 then
+			browser_navagite(timeout,v)
+		else 
+			--翻页
+			browser_click(timeout,1,"div#pageList a.layui-laypage-next")
+			browser_sleep(500)
+			browser_waitvisible(timeout,1,"div#pageList")
+		end
+		local _,has_next_page = browser_executejs(timeout,0,has_next_page_js)
+		-- 数据获取
+		local ok,list = browser_executejs(timeout,1,item_selector_js)
+		if ok=="ok" then
+			for _,item in pairs(list) do 
+				local selector = 'document.querySelectorAll("div.product-list-order div.product-list div.product-pic a")['..item["index"]..']'
+				browser_click(timeout,3,selector)
+				browser_sleep(500)
+				get_detail_content()
+				--关闭详情页
+				browser_closetabs(timeout,"","product/detail")
+			end
+		else
+			print(list)
+		end
+		pageno = pageno+1
+		browser_sleep(1000)
+	until has_next_page=="false"
+	browser_reset()
+end
+
+-- 切换IP方法
+-- browser_navagite(120,"http://pleasechangemyip.com")

address/util/test.lua

@@ -0,0 +1,47 @@
+local href = "https://www.sdnpc.com/col/col31/index.html"
+local listjs = [[let ns=[];document.querySelectorAll("div.xxgg_cont #newslist_78_78_78 a").forEach((v,i)=>{ns.push({"index":i,"href":v.getAttribute("href")})});ns]]
+local detailjs=[[var ret="";var obj=document.querySelector("body > div.hdWzy.wzyContent");if(obj){ret=obj.innerText};ret]]
+--browser_navagite(timeout,"http://pleasechangemyip.com")
+--browser_sleep(500)
+local timeout = 1000*60
+
+-- 获取详情页信息
+function get_content()
+    -- 等待详情页加载完成
+    browser_waitvisible("/art",500,0,"body > div.hdWzy.wzyContent")--等待含"/art"路径的tab页元素加载完毕，运行到此已经打开一个列表页tab和一个详情页tab
+    -- 获取源码
+    local ok,data = browser_executejs("/art",2000,0,detailjs)
+    if ok=="ok" then
+        print("=========================================================================================================")
+    end
+end
+
+-- 列表页
+
+--打开列表页
+browser_navagite(timeout,href)--timeout:页面打开时间
+--等待元素加载完毕
+browser_waitvisible("#newslist_78_78_78")
+--执行js获取详情页a链接
+local ok,list = browser_executejs(1,listjs)
+--遍历所有a链接
+if ok=="ok" then
+    for k,v in pairs(list) do
+        --组装链接
+        --local detailHref = "https://www.sdnpc.com/"..v["href"]
+        --print("download ",detailHref)
+        --点击打开详情页
+        local selector = 'document.querySelectorAll("div.xxgg_cont #newslist_78_78_78 a")['..v["index"]..']'
+        print("00000000000000000000000000")
+        browser_click(5000,3,selector)--点击打开页面5秒
+        browser_sleep(1000)
+        --获取详情页信息
+        get_content()
+        --关闭详情页
+        print("1111111111111111111111111111111111")
+        browser_closetabs(1000,"","/art")
+        browser_sleep(500)
+        print("222222222222222222")
+    end
+end
+print(ok,list)

address/util/vps.lua

@@ -0,0 +1,149 @@
+--剑鱼VPS监控
+
+local login_url = "https://www.360vps.cn/54ak/user.asp"
+local auth_param={"jianyubiaoxun","Jy#20240215vps"}
+
+local dm_login_url = "http://dm.spdata.jianyu360.com/"
+local dm_auth_param={"admin","Topnet1q2w3e"}
+
+local timeout = 1000*60*10
+-- 登录
+function do_vps_login()
+	browser_navagite(timeout,login_url)
+	browser_keysend(timeout,1,"div.login input[name=username]",auth_param[1])
+	browser_keysend(timeout,1,"div.login input[name=password]",auth_param[2])
+	browser_click(timeout,1,"input#ld_btn")
+	browser_sleep(1000*2)
+end
+
+-- 查找无效vps
+function find_fail_vps()
+	browser_navagite(timeout,dm_login_url)
+	browser_keysend(timeout,1,"input[name=s_name]",dm_auth_param[1])
+	browser_keysend(timeout,1,"input[name=s_pwd]",dm_auth_param[2])
+	browser_click(timeout,1,"button[type=submit]")
+	browser_sleep(1000*5)
+	browser_navagite(timeout,"http://dm.spdata.jianyu360.com/front/client/proxy")
+	
+	local ok,ret = browser_executejs(timeout,0,[[
+		var ns = "";
+		var now = Date.now() / 1000 / 60;
+		document.querySelectorAll("table tr")
+			.forEach((v, i) => {
+				if (i > 0) {
+					let status = v.querySelector("td:nth-child(6)")
+						.innerText;
+					let lastAccess = v.querySelector("td:nth-child(10)")
+						.innerText;
+					let id = v.querySelector("td:nth-child(2)")
+						.innerText;
+					
+					let diffTime = now - (new Date(lastAccess).getTime()/1000/60);
+					if (status=="无效" && diffTime>10){
+						ns=ns+id+";";
+					}
+				}
+			});
+		ns
+	]])
+	if ok=="ok" then
+		return ret
+	else
+		return ""
+	end
+end
+
+-- 重启VPS
+function reset_vps(v)
+	browser_navagite(true,timeout,v["url"])
+	local ok,body_innertext = browser_executejs("/vpsadm/",timeout,0,[[var ret=document.body.innerText;ret]])
+	if ok=="ok" then
+		-- 这里重启调用有问题，会弹出一个警告窗口，脚本关不掉
+		--print(body_innertext)
+		--browser_sleep(1000*60)
+		browser_click("/vpsadm/",1000*5,1,"button#proceed-button")
+		browser_sleep(1000*2)
+		browser_click("/vpsadm/",1000*5,1,"button#proceed-button")
+		browser_sleep(1000*3)	
+		browser_click("panel/index",1000*5,1,"button#btnReboot")
+		--print("点击重启按钮")
+		browser_sleep(1000*3)
+		browser_closetabs("panel/index",1000*5)
+	else
+		print("can't load body innertext!",ok,body_innertext)
+	end
+end
+
+--每一轮次的任务
+function do_check_job()
+	local fail_vps_ret = find_fail_vps()
+	print(fail_vps_ret)
+	browser_save("","","","",{title="超过10分钟未恢复的VPS，稍后会自动重启，请关注",msg=fail_vps_ret})
+	do_vps_login()
+	browser_sleep(1000*2)
+	browser_click(timeout,1,"a.user-product-icon")
+	browser_sleep(1000*5)
+	local pageno=1
+	repeat
+		if pageno>1 then
+			browser_click(timeout,3,[[var ret=null;document.querySelectorAll("a").forEach((v,i)=>{ if(v.innerText=="下一页"){ret=v;}});ret]])
+			browser_sleep(1000*5)
+		end
+		--执行JS找到无效VPS
+		local ok,list = browser_executejs(timeout,1,[[
+			var ns = [];
+			var fail_vps = "]]..fail_vps_ret..[[".split(";");
+			var vps = [];
+			fail_vps.forEach((v, i) => {
+				var tmp = v.split("_");
+				vps.push({
+					"name": tmp[0],
+					"port": tmp[1]
+				});
+			});
+			document.querySelectorAll("table.ny-table tr")
+				.forEach((v, i) => {
+					let nameTd = v.querySelector("td:nth-child(2)");
+					let ipTd = v.querySelector("td:nth-child(4)");
+					if (nameTd && ipTd) {
+						let name = nameTd.innerText;
+						let ip = ipTd.innerText;
+	                    let ipHtml = ipTd.innerHTML;
+	                    let id = /id=(\d+)/.exec(ipHtml)[1]
+						let index = i;
+						vps.forEach((v, i) => {
+							if (name.includes(v.name) && ip.includes(v.port)) {
+								ns.push({"index":index,"url":"https://www.360vps.cn/user/vpsadm2.asp?id="+id+"&go=a"});
+							}
+						});
+			
+					}
+				});
+			ns
+		]])
+		
+		if ok=="ok" then
+			for k,v in pairs(list) do
+				reset_vps(v)
+				browser_sleep(1000*2)
+			end
+		end
+		
+		pageno = pageno + 1
+		local ok,has_next_page = browser_executejs(timeout,0,[[var ret="false";document.querySelectorAll("a").forEach((v,i)=>{ if(v.innerText=="下一页"){ret="true"}});ret]])
+		--print(ok,has_next_page)
+	until ok~="ok" or has_next_page~="true"
+	browser_sleep(1000*5)
+	browser_reset()
+end
+
+--每10分钟检查一次
+repeat
+	local now = os.date("%Y-%m-%d %H:%M:%S",os.time())
+	print(now,"开始执行任务")
+	do_check_job()
+	now = os.date("%Y-%m-%d %H:%M:%S",os.time())
+	print(now,"任务执行完成")
+	print("===========")
+	browser_sleep(1000*60*10)
+until 1==1

address/util/zj_czys.lua

@@ -0,0 +1,20 @@
+-- 浙江省财政预算
+
+local js={
+	["attach_selector_js"]=[[var ns=[];document.querySelectorAll("a").forEach((v,i)=>{ var href=v.getAttribute("href"); if(href && href.endsWith(".pdf")){ ns.push({"href":href,"index":i});}  } );ns]]
+}
+
+
+local timeout = 1000*60*120
+browser_sleep(1000)
+browser_navagite(timeout,"https://www.zj.gov.cn/art/2022/1/30/art_1552361_59664101.html")
+local ok,list = browser_executejs(timeout,1,js["attach_selector_js"])
+if ok=="ok" then
+    for k,v in pairs(list) do 
+    	print("download ",v["href"])
+		browser_navagite_download_res("https://www.zj.gov.cn/"..v["href"],"text pdf xls xlsx html htm","./")
+		browser_sleep(1000)
+	end
+end
+
+browser_sleep(10000)

address/vm.go

@@ -0,0 +1,198 @@
+/**
+ * 虚拟机
+ */
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/chromedp/chromedp"
+	"github.com/yuin/gopher-lua"
+	"github.com/yuin/gopher-lua/parse"
+)
+
+const (
+	run_on_device_remote = iota
+	run_on_device_local
+)
+
+type (
+	//虚拟机
+	VM struct {
+		WsAddr    string
+		Headless  bool
+		ShowImage bool
+		ProxyAddr string
+		RunMode   int
+		B         *Browser
+		S         Storage
+	}
+	//浏览器，（不用之前封装的，这个更轻量）
+	Browser struct {
+		Ctx      context.Context
+		CancelFn context.CancelFunc
+	}
+)
+
+// NewVM
+func NewRemoteVM(wsAddr string, s Storage) *VM {
+	return &VM{WsAddr: wsAddr, RunMode: run_on_device_remote, S: s}
+}
+
+// NewVM
+func NewLocalVM(headless bool, showImage bool, proxyAddr string, s Storage) *VM {
+	return &VM{Headless: headless, ProxyAddr: proxyAddr, ShowImage: showImage,
+		RunMode: run_on_device_local,
+		S:       s}
+}
+
+// Quit
+func (b *Browser) Quit() {
+	if b != nil && b.CancelFn != nil {
+		b.CancelFn()
+		b.Ctx = nil
+		b.CancelFn = nil
+	}
+}
+
+// createRemoteBrowser 创建远程浏览器
+func createRemoteBrowser(wsAddr string) *Browser {
+	allocCtx, cancelFn := chromedp.NewRemoteAllocator(context.TODO(), wsAddr)
+	incCtx, _ := chromedp.NewContext(allocCtx)
+	return &Browser{
+		incCtx, cancelFn,
+	}
+}
+
+// createLocalBrowser 创建本地浏览器
+func createLocalBrowser(headless, showImage bool, proxyAddr string) *Browser {
+	baseCtx, _ := chromedp.NewContext(context.Background())
+
+	chromeOptions := append(chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.NoDefaultBrowserCheck,                                  //不检查默认浏览器
+		chromedp.Flag("enable-automation", false),                       // 防止监测webdriver
+		chromedp.Flag("disable-blink-features", "AutomationControlled"), //禁用 blink 特征
+		chromedp.Flag("force-dev-mode-highlighting", true),
+		chromedp.Flag("disable-extensions", false), //是否禁用扩展
+		chromedp.Flag("headless", headless),
+		chromedp.Flag("user-agent", "Chrome 9 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/93.0.4577.58 Safari/537.36 Edg/93.0.961.33"),
+		chromedp.Flag("disable-keep-alive", true),
+		chromedp.Flag("disable-gpu", true),
+		chromedp.Flag("no-sandbox", true),
+		chromedp.Flag("disable-dev-shm-usage", false),
+		chromedp.Flag("default-browser-check", false),
+		chromedp.Flag("ignore-certificate-errors", true), //忽略错误
+		chromedp.Flag("disable-web-security", true),      //禁用网络安全标志
+		chromedp.Flag("mute-audio", false),
+		chromedp.Flag("accept-language", `zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-TW;q=0.6`),
+		chromedp.Flag("ignore-certificate-errors", false),
+		//chromedp.Flag("blink-settings", "imagesEnabled=true"),
+		//chromedp.Flag("incognito", true),     //隐私模式
+		chromedp.Flag("disable-cache", true), //不用缓存
+	)
+	if proxyAddr != "" {
+		chromeOptions = append(chromeOptions,
+			chromedp.ProxyServer(fmt.Sprintf("socks5://%s", proxyAddr)))
+	}
+
+	if showImage {
+		chromeOptions = append(chromeOptions,
+			chromedp.Flag("blink-settings", "imagesEnabled=true"),
+		)
+	} else {
+		chromeOptions = append(chromeOptions,
+			chromedp.Flag("blink-settings", "imagesEnabled=false"),
+		)
+	}
+	allocCtx, _ := chromedp.NewExecAllocator(baseCtx, chromeOptions...)
+	// 创建一个浏览器实例
+	incCtx, incCancelFn := chromedp.NewContext(allocCtx,
+		chromedp.WithLogf(log.Printf))
+	return &Browser{
+		incCtx, incCancelFn,
+	}
+}
+
+// 重置浏览器
+func (vm *VM) ResetBrowser() {
+	if vm.B != nil && vm.B.CancelFn != nil {
+		vm.B.CancelFn()
+		vm.B.Ctx = nil
+		vm.B.CancelFn = nil
+	}
+	var b *Browser
+	if vm.RunMode == run_on_device_local {
+		b = createLocalBrowser(vm.Headless, vm.ShowImage, vm.ProxyAddr)
+	} else {
+		b = createRemoteBrowser(vm.WsAddr)
+	}
+	if vm.B == nil {
+		vm.B = b
+	} else {
+		vm.B.Ctx, vm.B.CancelFn = b.Ctx, b.CancelFn
+	}
+}
+
+// BindLuaState 绑定虚拟机函数
+func (vm *VM) BindLuaState(state *lua.LState) {
+	state.SetGlobal("browser_reset", state.NewFunction(func(l *lua.LState) int {
+		vm.ResetBrowser()
+		return 0
+	}))
+	state.SetGlobal("browser_save", state.NewFunction(func(l *lua.LState) int {
+		spiderCode := l.ToString(-5)
+		siteName := l.ToString(-4)
+		siteChannelName := l.ToString(-3)
+		siteChannelUrl := l.ToString(-2)
+		table := l.ToTable(-1)
+		data := TableToMap(table)
+		vm.S.Save(spiderCode, siteName, siteChannelName, siteChannelUrl, data)
+		return 0
+	}))
+}
+
+// runScript 执行lua代码
+func (vm *VM) RunScript(script string) error {
+	defer Catch()
+
+	var state *lua.LState = lua.NewState()
+	defer state.Close()
+	//方法绑定
+	vm.ResetBrowser() //先创建浏览器对象
+	vm.BindLuaState(state)
+	vm.B.BindLuaState(state)
+	defer func() {
+		if vm.B != nil {
+			vm.B.Quit()
+		}
+	}()
+
+	reader := strings.NewReader(script)
+	chunk, err := parse.Parse(reader, "code")
+	if err != nil {
+		return err
+	}
+	proto, err := lua.Compile(chunk, script)
+	if err != nil {
+		return err
+	}
+	lfunc := state.NewFunctionFromProto(proto)
+	state.Push(lfunc)
+	state.Call(0, 0)
+
+	// state.CallByParam(lua.P{
+	// 	Fn:      state.GetGlobal("getLngLat"),
+	// 	NRet:    1,
+	// 	Protect: true,
+	// }, lua.LString("郑州大学"))
+	// // ret := state.Get(1)
+	// // // 如果是2个返回值， NRet改为2
+	// // ret2 := state.Get(2)
+
+	// // fmt.Println(ret, ret2)
+
+	return nil
+}

addresssss/main.go

@@ -0,0 +1,45 @@
+package main
+
+//-----linux
+/*import (
+	"usermanager"
+	"utils"
+
+	"github.com/fvbock/endless"
+	log "github.com/sirupsen/logrus"
+)
+
+func Server() {
+	endless.ListenAndServe(utils.SysConfig["webport"].(string), usermanager.Middleware(usermanager.Route))
+}*/
+
+//-----windows
+import (
+	"config"
+	"log"
+	"usermanager"
+	"utils"
+
+	"github.com/fvbock/endless"
+)
+
+var (
+	Route = mux.NewRouter()
+)
+
+func init() {
+	Route.HandleFunc("/list", list).Methods("GET", "POST")
+}
+
+func Server() {
+	endless.ListenAndServe("9999", Route)
+}
+
+func main() {
+	log.Println("Start server...")
+	Server()
+}
+
+func list() {
+
+}

biddingToLabel/go.mod

@@ -2,19 +2,33 @@ module biddingToLabel
 
 go 1.21.0
 
-require app.yhyue.com/moapp/jybase v0.0.0-20240424025716-c77615e9004e
+require (
+	app.yhyue.com/moapp/jybase v0.0.0-20240424025716-c77615e9004e
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40
+)
 
 require (
+	app.yhyue.com/moapp/esv1 v0.0.0-20220414031211-3da4123e648d // indirect
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/olivere/elastic v6.2.37+incompatible // indirect
+	github.com/olivere/elastic/v7 v7.0.32 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.0.2 // indirect
-	github.com/xdg-go/stringprep v1.0.2 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.mongodb.org/mongo-driver v1.9.1 // indirect
-	golang.org/x/crypto v0.0.0-20210920023735-84f357641f63 // indirect
+	go.mongodb.org/mongo-driver v1.10.1 // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/net v0.0.0-20220531201128-c960675eff93 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 )

biddingToLabel/go.sum

@@ -1,17 +1,24 @@
+app.yhyue.com/moapp/esv1 v0.0.0-20220414031211-3da4123e648d h1:WPsYuuptAd3UEgN+jPzpnsDe/OvcshDUUtOTZPYGSJ8=
 app.yhyue.com/moapp/esv1 v0.0.0-20220414031211-3da4123e648d/go.mod h1:91/lSD/hS+ckMVP3WdidRzDhC60lLMdyce9QHy0cSMA=
 app.yhyue.com/moapp/jybase v0.0.0-20240424025716-c77615e9004e h1:1LaaIJVFQrwtjMKagxxq1JHZ93I48xzmJk6DI5PiCL8=
 app.yhyue.com/moapp/jybase v0.0.0-20240424025716-c77615e9004e/go.mod h1:XHNATN6tsJKHdCB0DbUtFdPPHXexTUFyB3RlO+lUUoM=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/RoaringBitmap/roaring v1.5.0/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/aws/aws-sdk-go v1.35.20/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -33,6 +40,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
 github.com/donnie4w/go-logger v0.0.0-20170827050443-4740c51383f4/go.mod h1:L7S4x0R7vv3xoOhGuyAJyCO2MYzWOpccM4Isn8jIUgY=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -52,6 +61,8 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
@@ -62,6 +73,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
 github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -87,6 +100,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -102,6 +116,7 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr
 github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -122,18 +137,25 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic v6.2.37+incompatible h1:UfSGJem5czY+x/LqxgeCBgjDn6St+z8OnsCuxwD3L0U=
 github.com/olivere/elastic v6.2.37+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
+github.com/olivere/elastic/v7 v7.0.22 h1:esBA6JJwvYgfms0EVlH7Z+9J4oQ/WUADF2y/nCNDw7s=
 github.com/olivere/elastic/v7 v7.0.22/go.mod h1:VDexNy9NjmtAkrjNoI7tImv7FR4tf5zUA3ickqu5Pc8=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -181,8 +203,12 @@ github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
 github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
@@ -194,7 +220,12 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3
 go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
 go.mongodb.org/mongo-driver v1.9.1 h1:m078y9v7sBItkt1aaoe2YlvWEXcD263e1a4E1fBrJ1c=
 go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@@ -204,6 +235,7 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i
 go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -211,6 +243,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210920023735-84f357641f63 h1:kETrAMYZq6WVGPa8IIixL0CaEcIUNi+1WX7grUoi3y8=
 golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -235,8 +269,13 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220531201128-c960675eff93 h1:MYimHLfoXEpOhqd/zgoA/uoXzHB86AEky4LAx5ij9xA=
 golang.org/x/net v0.0.0-20220531201128-c960675eff93/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -270,9 +309,11 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -281,6 +322,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -314,6 +356,7 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
@@ -338,6 +381,9 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -355,4 +401,6 @@ gorm.io/driver/mysql v1.0.5/go.mod h1:N1OIhHAIhx5SunkMGqWbGFVeh4yTNWKmMo1GOAsohL
 gorm.io/gorm v1.21.3/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40 h1:xTeRmpFgwOdu+NbWg/YntX3MnQpttm7jj33C1+JdBTk=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

biddingToLabel/main.go

@@ -5,25 +5,62 @@ import (
 	"sync"
 
 	"log"
+	"time"
 
 	"app.yhyue.com/moapp/jybase/common"
 	"app.yhyue.com/moapp/jybase/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
 )
 
 var (
-	Mgo  *mongodb.MongodbSim
-	Mgos *mongodb.MongodbSim
+	Mgo          *mongodb.MongodbSim
+	Mgos         *mongodb.MongodbSim
+	Es           *elastic.Elastic
+	updateEsPool = make(chan []map[string]interface{}, 5000)
+	updateEsSp   = make(chan bool, 5) //保存协程
 )
 
 func init() {
 	Mgo = mongodb.NewMgoWithUser("172.17.189.140:27080,172.17.189.141:27081", "bidding", "root", "top@123", 50)
 	Mgos = mongodb.NewMgo("172.17.4.85:27080", "xzh", 50)
+	// Es = elastic.NewEs("07", "http://192.168.3.241:9205", 20, "jybid", "Top2023_JEB01i@31")
+	Es = &elastic.Elastic{
+		S_esurl: "http://127.0.0.1:9801",
+		// S_esurl:  "http://172.17.4.184:19908",
+		I_size:   20,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	Es.InitElasticSize()
+	// go updateEsMethod()
 }
 
 func main() {
+	count := Es.Count("bidding", `{
+  "query": {
+    "bool": {
+      "must": [
+        {
+          "term": {
+            "_id": "<nil>"
+          }
+        },
+        {
+          "term": {
+            "autoid": 299834181
+          }
+        }
+      ]
+    }
+  }
+}`)
+	log.Println("count", count)
+	return
 	sess := Mgo.GetMgoConn()
 	defer Mgo.DestoryMongoConn(sess)
-	q := map[string]interface{}{}
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{"$gte": mongodb.StringTOBsonId("5fedf5800000000000000000")},
+	}
 	it := sess.DB("qfw").C("bidding").Find(&q).Sort("_id").Iter()
 	pool := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
@@ -40,12 +77,39 @@ func main() {
 				wg.Done()
 			}()
 			// data := map[string]interface{}{}
+			biddingID := mongodb.BsonIdToSId(tmp["_id"])
 			buyerclass := common.ObjToString(tmp["buyerclass"])
+			tag_topinformationMap := map[string]string{}
 			tag_subinformationMap := map[string]string{}
 			buyerclassMap := map[string]string{}
+			businesslabelMap := map[string]string{}
 			property_formMap := map[string]string{}
 			if buyerclass != "" {
 				buyerclassMap[buyerclass] = "1"
+				buyer_type := getStr(buyerclass)
+				businesslabelMap[buyer_type] = "1"
+				updateEsPool <- []map[string]interface{}{
+					map[string]interface{}{"_id": biddingID},
+					map[string]interface{}{"buyer_type": buyer_type},
+				}
+			} else {
+				businesslabelMap["其它"] = "1"
+				updateEsPool <- []map[string]interface{}{
+					map[string]interface{}{"_id": biddingID},
+					map[string]interface{}{"buyer_type": "其它"},
+				}
+			}
+			if tmp["tag_topinformation"] != nil {
+				tag_topinformation := common.ObjArrToStringArr(tmp["tag_topinformation"].([]interface{}))
+				for _, v := range tag_topinformation {
+					tag_topinformationMap[v] = "1"
+				}
+			}
+			if tmp["tag_topinformation_ai"] != nil {
+				tag_topinformation_ai := common.ObjArrToStringArr(tmp["tag_topinformation_ai"].([]interface{}))
+				for _, v := range tag_topinformation_ai {
+					tag_topinformationMap[v] = "1"
+				}
 			}
 			if tmp["tag_subinformation"] != nil {
 				tag_subinformation := common.ObjArrToStringArr(tmp["tag_subinformation"].([]interface{}))
@@ -68,13 +132,13 @@ func main() {
 			buyer := common.ObjToString(tmp["buyer"])
 			agency := common.ObjToString(tmp["buyer"])
 			s_winner := common.ObjToString(tmp["s_winner"])
-			buyerclassMap1, tag_subinformationMap1, property_formMap1 := buyerclassMap, tag_subinformationMap, property_formMap
-			tag_subinformationMap2, property_formMap2 := tag_subinformationMap, property_formMap
-			getname(buyer, buyerclassMap1, tag_subinformationMap1, property_formMap1)
-			getname1(agency, tag_subinformationMap2, property_formMap2)
+			buyerclassMap1, tag_subinformationMap1, tag_topinformationMap1, property_formMap1 := buyerclassMap, tag_subinformationMap, tag_topinformationMap, property_formMap
+			tag_subinformationMap2, tag_topinformationMap2, property_formMap2 := tag_subinformationMap, tag_topinformationMap, property_formMap
+			getname(buyer, buyerclassMap1, businesslabelMap, tag_subinformationMap1, tag_topinformationMap1, property_formMap1)
+			getname1(agency, tag_subinformationMap2, tag_topinformationMap2, property_formMap2)
 			for _, v := range strings.Split(s_winner, ",") {
-				tag_subinformationMap3, property_formMap3 := tag_subinformationMap, property_formMap
-				getname1(v, tag_subinformationMap3, property_formMap3)
+				tag_subinformationMap3, tag_topinformationMap3, property_formMap3 := tag_subinformationMap, tag_topinformationMap, property_formMap
+				getname1(v, tag_subinformationMap3, tag_topinformationMap3, property_formMap3)
 			}
 		}(tmp)
 		tmp = make(map[string]interface{})
@@ -83,7 +147,7 @@ func main() {
 	log.Println("is over ~ ", total)
 }
 
-func getname(name string, buyerclassMap, tag_subinformationMap, property_formMap map[string]string) {
+func getname(name string, buyerclassMap, businesslabelMap, tag_subinformationMap, tag_topinformationMap, property_formMap map[string]string) {
 	if name != "" {
 		data := map[string]interface{}{}
 		bd, ok := Mgos.FindOne("label", map[string]interface{}{"name": name})
@@ -99,6 +163,17 @@ func getname(name string, buyerclassMap, tag_subinformationMap, property_formMap
 					}
 				}
 			}
+			if (*bd)["buyer_type"] != nil {
+				if business_label1, oks := (*bd)["buyer_type"].(map[string]string); oks {
+					for k, _ := range business_label1 {
+						businesslabelMap[k] = "1"
+					}
+				} else if business_label1, oks := (*bd)["buyer_type"].(map[string]interface{}); oks {
+					for k, _ := range business_label1 {
+						businesslabelMap[k] = "1"
+					}
+				}
+			}
 			if (*bd)["tag_subinformation"] != nil {
 				if tag_subinformation1, oks := (*bd)["tag_subinformation"].(map[string]string); oks {
 					for k, _ := range tag_subinformation1 {
@@ -110,6 +185,17 @@ func getname(name string, buyerclassMap, tag_subinformationMap, property_formMap
 					}
 				}
 			}
+			if (*bd)["tag_topinformation"] != nil {
+				if tag_topinformation1, oks := (*bd)["tag_topinformation"].(map[string]string); oks {
+					for k, _ := range tag_topinformation1 {
+						tag_topinformationMap[k] = "1"
+					}
+				} else if tag_topinformation1, oks := (*bd)["tag_topinformation"].(map[string]interface{}); oks {
+					for k, _ := range tag_topinformation1 {
+						tag_topinformationMap[k] = "1"
+					}
+				}
+			}
 			if (*bd)["property_form"] != nil {
 				if property_form1, oks := (*bd)["property_form"].(map[string]string); oks {
 					for k, _ := range property_form1 {
@@ -124,9 +210,15 @@ func getname(name string, buyerclassMap, tag_subinformationMap, property_formMap
 			if len(buyerclassMap) > 0 {
 				data["buyerclass"] = buyerclassMap
 			}
+			if len(businesslabelMap) > 0 {
+				data["buyer_type"] = businesslabelMap
+			}
 			if len(tag_subinformationMap) > 0 {
 				data["tag_subinformation"] = tag_subinformationMap
 			}
+			if len(tag_topinformationMap) > 0 {
+				data["tag_topinformation"] = tag_topinformationMap
+			}
 			if len(property_formMap) > 0 {
 				data["property_form"] = property_formMap
 			}
@@ -138,9 +230,15 @@ func getname(name string, buyerclassMap, tag_subinformationMap, property_formMap
 			if len(buyerclassMap) > 0 {
 				data["buyerclass"] = buyerclassMap
 			}
+			if len(businesslabelMap) > 0 {
+				data["buyer_type"] = businesslabelMap
+			}
 			if len(tag_subinformationMap) > 0 {
 				data["tag_subinformation"] = tag_subinformationMap
 			}
+			if len(tag_topinformationMap) > 0 {
+				data["tag_topinformation"] = tag_topinformationMap
+			}
 			if len(property_formMap) > 0 {
 				data["property_form"] = property_formMap
 			}
@@ -151,7 +249,7 @@ func getname(name string, buyerclassMap, tag_subinformationMap, property_formMap
 	}
 }
 
-func getname1(name string, tag_subinformationMap, property_formMap map[string]string) {
+func getname1(name string, tag_subinformationMap, tag_topinformationMap, property_formMap map[string]string) {
 	if name != "" {
 		data := map[string]interface{}{}
 		bd, ok := Mgos.FindOne("label", map[string]interface{}{"name": name})
@@ -167,6 +265,17 @@ func getname1(name string, tag_subinformationMap, property_formMap map[string]st
 					}
 				}
 			}
+			if (*bd)["tag_topinformation"] != nil {
+				if tag_topinformation1, oks := (*bd)["tag_topinformation"].(map[string]string); oks {
+					for k, _ := range tag_topinformation1 {
+						tag_topinformationMap[k] = "1"
+					}
+				} else if tag_topinformation1, oks := (*bd)["tag_topinformation"].(map[string]interface{}); oks {
+					for k, _ := range tag_topinformation1 {
+						tag_topinformationMap[k] = "1"
+					}
+				}
+			}
 			if (*bd)["property_form"] != nil {
 				if property_form1, oks := (*bd)["property_form"].(map[string]string); oks {
 					for k, _ := range property_form1 {
@@ -181,6 +290,9 @@ func getname1(name string, tag_subinformationMap, property_formMap map[string]st
 			if len(tag_subinformationMap) > 0 {
 				data["tag_subinformation"] = tag_subinformationMap
 			}
+			if len(tag_topinformationMap) > 0 {
+				data["tag_topinformation"] = tag_topinformationMap
+			}
 			if len(property_formMap) > 0 {
 				data["property_form"] = property_formMap
 			}
@@ -192,6 +304,9 @@ func getname1(name string, tag_subinformationMap, property_formMap map[string]st
 			if len(tag_subinformationMap) > 0 {
 				data["tag_subinformation"] = tag_subinformationMap
 			}
+			if len(tag_topinformationMap) > 0 {
+				data["tag_topinformation"] = tag_topinformationMap
+			}
 			if len(property_formMap) > 0 {
 				data["property_form"] = property_formMap
 			}
@@ -201,3 +316,63 @@ func getname1(name string, tag_subinformationMap, property_formMap map[string]st
 		}
 	}
 }
+
+func getStr(b string) string {
+	if b == "" {
+		return "其它"
+	}
+	a1 := "(交通|运输物流|工信|农业|住建|城管|市政|出版广电|检察院|科技|民政|生态环境|市场监管|水利|应急管理|自然资源|财政|档案|党委办|组织|发改|宣传|政府办|政务中心|人大|政协|法院|公安|国资委|海关|机关事务|纪委|军队|人社|商务|审计税务|司法|体育|统计|统战|文旅|民宗|银保监|证监|气象|社会团体|公共资源交易)"
+	a2 := "(卫健委|医疗)"
+	a3 := "(教育|学校)"
+	a4 := "(人行|金融业)"
+	a5 := "(信息技术|电信行业|农林牧渔|建筑业|传媒|制造业|住宿餐饮|采矿业|能源化工|批发零售)"
+	if strings.Contains(a1, b) {
+		return "政府机构"
+	} else if strings.Contains(a2, b) {
+		return "医疗单位"
+	} else if strings.Contains(a3, b) {
+		return "教育单位"
+	} else if strings.Contains(a4, b) {
+		return "金融企业"
+	} else if strings.Contains(a5, b) {
+		return "商业公司"
+	} else {
+		return "其它"
+	}
+	return "其它"
+}
+
+func updateEsMethod() {
+	arru := make([][]map[string]interface{}, 200)
+	indexu := 0
+	for {
+		select {
+		case v := <-updateEsPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk("bidding", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk("bidding", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		}
+	}
+}

config.json

@@ -1,32 +1,32 @@
 {
   "mongo" : {
     "bid_mgo": {
-      "addr": "127.0.0.1:12005",
+      "addr": "192.168.3.206:27080",
       "dbname" : "qfw",
-      "username": "zhengkun",
-      "password": "zk@123123"
+      "username": "",
+      "password": ""
     },
     "qy_mgo": {
-      "addr": "127.0.0.1:12005",
+      "addr": "192.168.3.206:27080",
       "dbname" : "mixdata",
-      "username": "zhengkun",
-      "password": "zk@123123"
+      "username": "",
+      "password": ""
     },
     "spi_mgo": {
-      "addr": "127.0.0.1:12004",
+      "addr": "192.168.3.206:27080",
       "dbname" : "mixdata",
       "username": "",
       "password": ""
     },
     "ext_mgo": {
-      "addr": "127.0.0.1:12001",
+      "addr": "192.168.3.206:27080",
       "dbname" : "qfw",
       "username": "",
       "password": ""
     }
   },
   "mysql": {
-    "addr": "127.0.0.1:15001",
+    "addr": "192.168.3.149:4000",
     "dbname" : "global_common_data",
     "username": "zhengkun",
     "password": "Zk#20220824"
@@ -38,8 +38,8 @@
     "password": "pwdTopJy123"
   },
   "es": {
-    "addr": "http://127.0.0.1:13003",
-    "username": "es_all",
-    "password": "TopJkO2E_d1x"
+    "addr": "http://192.168.3.149:9200",
+    "username": "",
+    "password": ""
   }
 }

ent_legal/legal_add.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"data_ent_wuye/ent_contact"
 	ul "data_ent_wuye/ent_util"
-	"fmt"
 	"strconv"
 	"strings"
 	"sync"
+
 	"time"
 	"unicode/utf8"
 
@@ -28,12 +28,18 @@ var BidFields = map[string]interface{}{
 func LegalAdd() {
 	t := time.Now()
 	startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -2).Unix()
+	// startTime := int64(1717948800)
 	endTime := startTime + 86400
+	// endTime := int64(1718035200)
 	startId := strconv.FormatInt(startTime, 16) + "0000000000000000"
 	endId := strconv.FormatInt(endTime, 16) + "0000000000000000"
 	RunLegalInfo(startId, endId)
 }
 
+func shuaAdd() {
+
+}
+
 // 正常标讯增量信息····
 func RunLegalInfo(gtid string, lteid string) {
 	log.Println("增量~~~")
@@ -46,7 +52,7 @@ func RunLegalInfo(gtid string, lteid string) {
 		},
 	}
 	log.Println("查询语句 ~ ", q)
-	it := sess.DB(ul.SourceMgo.DbName).C("").Find(&q).Sort("_id").Select(BidFields).Iter()
+	it := sess.DB(ul.SourceMgo.DbName).C("bidding").Find(&q).Sort("_id").Select(BidFields).Iter()
 	pool := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
 	total := 0
@@ -129,8 +135,7 @@ func FindLegal(tmp map[string]interface{}) {
 }
 
 func FindLegalInfo(name, peason, phone string) {
-	query := fmt.Sprintf(`SELECT id FROM information.ent_info  WHERE company_name = '%s'`, name)
-	rows, err := ul.ClickHouseConn.Query(context.Background(), query)
+	rows, err := ul.ClickHouseConn.Query(context.Background(), `SELECT id FROM information.ent_info  WHERE company_name = ?`, name)
 	if err != nil {
 		log.Println(err)
 	}
@@ -147,7 +152,7 @@ func FindLegalInfo(name, peason, phone string) {
 	if count > 0 {
 		UpdateContact(id, name, peason, phone)
 	} else {
-		CreateLegalInfo("save", name, peason, phone)
+		CreateLegalInfo(name, peason, phone)
 	}
 }
 
@@ -156,27 +161,90 @@ func UpdateContact(id, name, peason, phone string) {
 	//更新标签
 }
 
-func CreateLegalInfo(dataType, name, peason, phone string) {
-	if dataType == "save" {
-		base_info := map[string]interface{}{}
-		//法人信息所需要的大部分企业信息
-		qyxy_info := GetQyxyInfo(name)
-		base_info["company_name"] = name
-		area := qu.ObjToString(qyxy_info["company_area"])
-		city := qu.ObjToString(qyxy_info["company_city"])
-		district := qu.ObjToString(qyxy_info["company_district"])
-		base_info["area_code"], base_info["city_code"], base_info["district_code"] = ul.CalculateRegionCode(area, city, district)
-		id := uuid.New().String()
-		id = strings.ReplaceAll(id, "-", "")
-		base_info["id"] = id
-		base_info["createtime"] = time.Now().Unix()
-		/*
-			···
-			···
-		*/
-		ent_contact.InjectContactAddTask(id, name, peason, phone)
+func CreateLegalInfo(company_name, peason, phone string) {
+	tmp := GetQyxyInfo(company_name)
+	use_flag := qu.IntAll(tmp["use_flag"])
+	is_history := qu.IntAll(tmp["is_history"])
+	company_type := qu.ObjToString(tmp["company_type"])
+	if use_flag > 5 || is_history != 0 || company_name == "" || company_type == "个体工商户" {
+		return
+	}
+	id := uuid.New().String()
+	id = strings.ReplaceAll(id, "-", "")
+	area := qu.ObjToString(tmp["company_area"])
+	city := qu.ObjToString(tmp["company_city"])
+	district := qu.ObjToString(tmp["company_district"])
+	area_code, city_code, district_code := ul.CalculateRegionCode(area, city, district)
+	company_id := qu.ObjToString(tmp["_id"])
+	company_address := qu.ObjToString(tmp["company_address"])
+	company_label, company_label_str := getCompanyLabel(company_name)
+	company_code := qu.ObjToString(tmp["company_code"])
+	credit_no := qu.ObjToString(tmp["credit_no"])
+	org_code := qu.ObjToString(tmp["org_code"])
+	tax_code := qu.ObjToString(tmp["tax_code"])
+	establish_date := qu.Int64All(tmp["establish_date"])
+	legal_person := qu.ObjToString(tmp["legal_person"])
+	legal_person_caption := qu.ObjToString(tmp["legal_person_caption"])
+	company_status := qu.ObjToString(tmp["company_status"])
+	authority := qu.ObjToString(tmp["authority"])
+	issue_date := qu.Int64All(tmp["issue_date"])
+	operation_startdate := qu.ObjToString(tmp["operation_startdate"])
+	operation_enddate := qu.ObjToString(tmp["operation_enddate"])
+	capital := qu.ObjToString(tmp["capital"])
+	business_scope := qu.ObjToString(tmp["business_scope"])
+	comeintime := qu.Int64All(tmp["comeintime"])
+	updatetime := qu.Int64All(tmp["updatetime"])
+	legal_person_type := qu.IntAll(tmp["legal_person_type"])
+	real_capital := qu.ObjToString(tmp["real_capital"])
+	en_name := qu.ObjToString(tmp["en_name"])
+	list_code := qu.ObjToString(tmp["list_code"])
+	employee_no := qu.IntAll(tmp["employee_no"])
+	website := qu.ObjToString(tmp["website"])
+	company_phone := qu.ObjToString(tmp["company_phone"])
+	company_email := qu.ObjToString(tmp["company_email"])
+	query := `INSERT INTO information.ent_info (id, company_id, company_name, company_label, company_code, credit_no, org_code, tax_code, establish_date, legal_person, legal_person_caption, company_status, company_type, authority, issue_date, operation_startdate, operation_enddate, capital, company_address, business_scope, comeintime, updatetime, legal_person_type, real_capital, en_name, area_code, city_code, district_code, list_code, employee_no, website, company_phone, company_email) VALUES(?, ?, ?, bitmapBuild(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+	err := ul.ClickHouseConn.Exec(context.Background(), query, id, company_id, company_name, company_label, company_code, credit_no, org_code, tax_code, establish_date, legal_person, legal_person_caption, company_status, company_type, authority, issue_date, operation_startdate, operation_enddate, capital, company_address, business_scope, comeintime, updatetime, legal_person_type, real_capital, en_name, area_code, city_code, district_code, list_code, employee_no, website, company_phone, company_email)
+	if err != nil {
+		log.Println(err, "clickhouse存入失败", company_name)
 	} else {
-		//告知通讯录...
+		data := map[string]interface{}{
+			"_id":                  id,
+			"company_name":         company_name,
+			"name":                 company_name,
+			"company_id":           company_id,
+			"company_address":      company_address,
+			"area_code":            area_code,
+			"city_code":            city_code,
+			"district_code":        district_code,
+			"company_label":        company_label_str,
+			"company_code":         company_code,
+			"credit_no":            credit_no,
+			"org_code":             org_code,
+			"tax_code":             tax_code,
+			"establish_date":       establish_date,
+			"legal_person":         legal_person,
+			"legal_person_caption": legal_person_caption,
+			"company_status":       company_status,
+			"company_type":         company_type,
+			"authority":            authority,
+			"issue_date":           issue_date,
+			"operation_startdate":  operation_startdate,
+			"operation_enddate":    operation_enddate,
+			"capital":              capital,
+			"business_scope":       business_scope,
+			"comeintime":           comeintime,
+			"updatetime":           updatetime,
+			"legal_person_type":    legal_person_type,
+			"real_capital":         real_capital,
+			"en_name":              en_name,
+			"list_code":            list_code,
+			"employee_no":          employee_no,
+			"website":              website,
+			"company_phone":        company_phone,
+			"company_email":        company_email,
+		}
+		Ch <- data
+		ent_contact.InjectContactAddTask(id, company_name, peason, phone)
 	}
 }
 

ent_legal/legal_full.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"data_ent_wuye/ent_contact"
 	"data_ent_wuye/ent_util"
+	"database/sql"
+	"fmt"
 	"log"
 	"regexp"
 	"strconv"
@@ -25,13 +27,14 @@ var (
 
 func LegalFull() {
 	//从现有主题库...直接获取
-	// getZhuTi()
+	// InjectTidbFull()
+
+	//马克信息···
+	// getMK()
+	//特企
+	// getTQ()
 	//凭安全量企业···查到数据告知通讯录
 	getPingAn()
-	//特企
-	getTQ()
-	//马克信息···
-	getMK()
 }
 
 func getZhuTi() {
@@ -49,7 +52,7 @@ L:
 			if len(*dataArr) == 0 {
 				break
 			}
-			pool := make(chan bool, 10)
+			pool := make(chan bool, 20)
 			wg := &sync.WaitGroup{}
 			for _, v := range *dataArr {
 				pool <- true
@@ -69,11 +72,15 @@ L:
 					area_code := qu.ObjToString(v["area_code"])
 					city_code := qu.ObjToString(v["city_code"])
 					district_code := qu.ObjToString(v["district_code"])
-					company_label, company_label_str := []uint64{uint64(0)}, ""
+					company_label := []uint64{uint64(0)}
+					// company_label_str := ""
 					qyxy_info := GetQyxyInfo(company_name)
+					log.Println("111", company_name)
 					if qyxy_info != nil {
-						company_label, company_label_str = getCompanyLabel(company_name)
+						// company_label, company_label_str = getCompanyLabel(company_name)
+						company_label, _ = getCompanyLabel(company_name)
 					}
+					log.Println("222", company_name)
 					company_code := qu.ObjToString(qyxy_info["company_code"])
 					credit_no := qu.ObjToString(qyxy_info["credit_no"])
 					org_code := qu.ObjToString(qyxy_info["org_code"])
@@ -109,47 +116,48 @@ L:
 					website := qu.ObjToString(qyxy_info["website"])
 					company_phone := qu.ObjToString(qyxy_info["company_phone"])
 					company_email := qu.ObjToString(qyxy_info["company_email"])
-					query := `INSERT INTO information.ent_info (id, company_id, company_name, company_label, company_code, credit_no, org_code, tax_code, establish_date, legal_person, legal_person_caption, company_status, company_type, authority, issue_date, operation_startdate, operation_enddate, capital, company_address, business_scope, comeintime, updatetime, legal_person_type, real_capital, en_name, area_code, city_code, district_code, list_code, employee_no, website, company_phone, company_email) VALUES(?, ?, ?, bitmapBuild(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+					query := `INSERT INTO information.ent_info_copy (id, company_id, company_name, company_label, company_code, credit_no, org_code, tax_code, establish_date, legal_person, legal_person_caption, company_status, company_type, authority, issue_date, operation_startdate, operation_enddate, capital, company_address, business_scope, comeintime, updatetime, legal_person_type, real_capital, en_name, area_code, city_code, district_code, list_code, employee_no, website, company_phone, company_email) VALUES(?, ?, ?, bitmapBuild(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
 					err := ent_util.ClickHouseConn.Exec(context.Background(), query, id, company_id, company_name, company_label, company_code, credit_no, org_code, tax_code, establish_dates, legal_person, legal_person_caption, company_status, company_type, authority, issue_dates, operation_startdate, operation_enddate, capital, company_address, business_scope, comeintime, updatetime, legal_person_type, real_capital, en_name, area_code, city_code, district_code, list_code, employee_no, website, company_phone, company_email)
+					log.Println("333", company_name)
 					if err != nil {
 						log.Println(err, "clickhouse存入失败", company_name)
 					} else {
-						data := map[string]interface{}{
-							"_id":                  id,
-							"company_name":         company_name,
-							"company_id":           company_id,
-							"company_address":      company_address,
-							"area_code":            area_code,
-							"city_code":            city_code,
-							"district_code":        district_code,
-							"company_label":        company_label_str,
-							"company_code":         company_code,
-							"credit_no":            credit_no,
-							"org_code":             org_code,
-							"tax_code":             tax_code,
-							"establish_date":       establish_dates,
-							"legal_person":         legal_person,
-							"legal_person_caption": legal_person_caption,
-							"company_status":       company_status,
-							"company_type":         company_type,
-							"authority":            authority,
-							"issue_date":           issue_dates,
-							"operation_startdate":  operation_startdate,
-							"operation_enddate":    operation_enddate,
-							"capital":              capital,
-							"business_scope":       business_scope,
-							"comeintime":           comeintime,
-							"updatetime":           updatetime,
-							"legal_person_type":    legal_person_type,
-							"real_capital":         real_capital,
-							"en_name":              en_name,
-							"list_code":            list_code,
-							"employee_no":          employee_no,
-							"website":              website,
-							"company_phone":        company_phone,
-							"company_email":        company_email,
-						}
-						Ch <- data
+						// data := map[string]interface{}{
+						// 	"_id":                  id,
+						// 	"company_name":         company_name,
+						// 	"company_id":           company_id,
+						// 	"company_address":      company_address,
+						// 	"area_code":            area_code,
+						// 	"city_code":            city_code,
+						// 	"district_code":        district_code,
+						// 	"company_label":        company_label_str,
+						// 	"company_code":         company_code,
+						// 	"credit_no":            credit_no,
+						// 	"org_code":             org_code,
+						// 	"tax_code":             tax_code,
+						// 	"establish_date":       establish_dates,
+						// 	"legal_person":         legal_person,
+						// 	"legal_person_caption": legal_person_caption,
+						// 	"company_status":       company_status,
+						// 	"company_type":         company_type,
+						// 	"authority":            authority,
+						// 	"issue_date":           issue_dates,
+						// 	"operation_startdate":  operation_startdate,
+						// 	"operation_enddate":    operation_enddate,
+						// 	"capital":              capital,
+						// 	"business_scope":       business_scope,
+						// 	"comeintime":           comeintime,
+						// 	"updatetime":           updatetime,
+						// 	"legal_person_type":    legal_person_type,
+						// 	"real_capital":         real_capital,
+						// 	"en_name":              en_name,
+						// 	"list_code":            list_code,
+						// 	"employee_no":          employee_no,
+						// 	"website":              website,
+						// 	"company_phone":        company_phone,
+						// 	"company_email":        company_email,
+						// }
+						// Ch <- data
 					}
 				}(v)
 			}
@@ -166,21 +174,144 @@ L:
 	log.Println("主体库全量任务结束")
 }
 
+func InjectTidbFull() {
+	log.Println("主体库全量任务开始")
+	pool := make(chan bool, 10) //控制线程数
+	wg := &sync.WaitGroup{}
+	finalId := 0
+	lastInfo := ent_util.MysqlGlobalTool.SelectBySql(fmt.Sprintf(`SELECT id FROM %s where createtime < "2024-04-19 00:00:00" ORDER BY id DESC LIMIT 1`, "dws_f_ent_baseinfo"))
+	if len(*lastInfo) > 0 {
+		finalId = qu.IntAll((*lastInfo)[0]["id"])
+	}
+	log.Println("last id", finalId)
+	lastid, count := 0, 0
+	for {
+		log.Println("重新查询，lastid---", lastid)
+		q := fmt.Sprintf(`SELECT id,name_id,name,company_id,address,area_code,city_code,district_code FROM %s WHERE id > %d and createtime < "2024-04-19 00:00:00" ORDER BY id ASC limit 10000`, "dws_f_ent_baseinfo", lastid)
+		var stmtOut *sql.Stmt
+		var tx *sql.Tx
+		var err error
+		if tx == nil {
+			stmtOut, err = ent_util.MysqlGlobalTool.DB.Prepare(q)
+		} else {
+			stmtOut, err = tx.Prepare(q)
+		}
+		rows, err := stmtOut.Query()
+		if err != nil {
+			log.Println("err : ", err)
+		}
+		columns, err := rows.Columns()
+		if lastid >= finalId {
+			log.Println("is over ...", count)
+			break
+		}
+		for rows.Next() {
+			scanArgs := make([]interface{}, len(columns))
+			values := make([]interface{}, len(columns))
+			ret := make(map[string]interface{})
+			for k := range values {
+				scanArgs[k] = &values[k]
+			}
+			err = rows.Scan(scanArgs...)
+			if err != nil {
+				log.Println("---", err)
+				break
+			}
+			for i, col := range values {
+				if v, ok := col.([]uint8); ok {
+					ret[columns[i]] = string(v)
+				} else {
+					ret[columns[i]] = col
+				}
+			}
+			lastid = qu.IntAll(ret["id"])
+			count++
+			if count%5000 == 0 {
+				log.Println("cur index", count, lastid)
+			}
+			pool <- true
+			wg.Add(1)
+			go func(v map[string]interface{}) {
+				defer func() {
+					<-pool
+					wg.Done()
+				}()
+				id := qu.ObjToString(v["name_id"])
+				company_name := qu.ObjToString(v["name"])
+				if company_name == "" {
+					return
+				}
+				company_id := qu.ObjToString(v["company_id"])
+				company_address := qu.ObjToString(v["address"])
+				area_code := qu.ObjToString(v["area_code"])
+				city_code := qu.ObjToString(v["city_code"])
+				district_code := qu.ObjToString(v["district_code"])
+				company_label := []uint64{uint64(0)}
+				// company_label_str := ""
+				qyxy_info := GetQyxyInfo(company_name)
+				if qyxy_info != nil {
+					// company_label, company_label_str = getCompanyLabel(company_name)
+					company_label, _ = getCompanyLabel(company_name)
+				}
+				company_code := qu.ObjToString(qyxy_info["company_code"])
+				credit_no := qu.ObjToString(qyxy_info["credit_no"])
+				org_code := qu.ObjToString(qyxy_info["org_code"])
+				tax_code := qu.ObjToString(qyxy_info["tax_code"])
+				establish_date := qu.ObjToString(qyxy_info["establish_date"])
+				establish_dates := int64(0)
+				if establish_date != "" {
+					establish_date1, _ := time.ParseInLocation(date.Date_Short_Layout, establish_date, time.Local)
+					establish_dates = establish_date1.Unix()
+				}
+				issue_date := qu.ObjToString(qyxy_info["issue_date"])
+				issue_dates := int64(0)
+				if issue_date != "" {
+					issue_date1, _ := time.ParseInLocation(date.Date_Short_Layout, issue_date, time.Local)
+					issue_dates = issue_date1.Unix()
+				}
+				legal_person := qu.ObjToString(qyxy_info["legal_person"])
+				legal_person_caption := qu.ObjToString(qyxy_info["legal_person_caption"])
+				company_status := qu.ObjToString(qyxy_info["company_status"])
+				company_type := qu.ObjToString(qyxy_info["company_type"])
+				authority := qu.ObjToString(qyxy_info["authority"])
+				operation_startdate := qu.ObjToString(qyxy_info["operation_startdate"])
+				operation_enddate := qu.ObjToString(qyxy_info["operation_enddate"])
+				capital := qu.ObjToString(qyxy_info["capital"])
+				business_scope := qu.ObjToString(qyxy_info["business_scope"])
+				comeintime := qu.Int64All(qyxy_info["comeintime"])
+				updatetime := qu.Int64All(qyxy_info["updatetime"])
+				legal_person_type := qu.IntAll(qyxy_info["legal_person_type"])
+				real_capital := qu.ObjToString(qyxy_info["real_capital"])
+				en_name := qu.ObjToString(qyxy_info["en_name"])
+				list_code := qu.ObjToString(qyxy_info["list_code"])
+				employee_no := qu.IntAll(qyxy_info["employee_no"])
+				website := qu.ObjToString(qyxy_info["website"])
+				company_phone := qu.ObjToString(qyxy_info["company_phone"])
+				company_email := qu.ObjToString(qyxy_info["company_email"])
+				query := `INSERT INTO information.ent_info_copy (id, company_id, company_name, company_label, company_code, credit_no, org_code, tax_code, establish_date, legal_person, legal_person_caption, company_status, company_type, authority, issue_date, operation_startdate, operation_enddate, capital, company_address, business_scope, comeintime, updatetime, legal_person_type, real_capital, en_name, area_code, city_code, district_code, list_code, employee_no, website, company_phone, company_email) VALUES(?, ?, ?, bitmapBuild(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+				err := ent_util.ClickHouseConn.Exec(context.Background(), query, id, company_id, company_name, company_label, company_code, credit_no, org_code, tax_code, establish_dates, legal_person, legal_person_caption, company_status, company_type, authority, issue_dates, operation_startdate, operation_enddate, capital, company_address, business_scope, comeintime, updatetime, legal_person_type, real_capital, en_name, area_code, city_code, district_code, list_code, employee_no, website, company_phone, company_email)
+				if err != nil {
+					log.Println(err, "clickhouse存入失败", company_name)
+				}
+			}(ret)
+			ret = make(map[string]interface{})
+		}
+		_ = rows.Close()
+		stmtOut.Close()
+		wg.Wait()
+	}
+	log.Println("is over ...")
+}
+
 func getPingAn() {
 	sess := ent_util.QyxyMgo.GetMgoConn()
 	defer ent_util.QyxyMgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{}
-	it := sess.DB("mixdata").C("qyxy_std").Find(&q).Select(map[string]interface{}{
-		"_id": 1, "use_flag": 1, "is_history": 1, "company_type": 1, "company_name": 1, "company_area": 1,
-		"company_city": 1, "company_district": 1, "company_address": 1, "company_code": 1, "credit_no": 1,
-		"org_code": 1, "tax_code": 1, "establish_date": 1, "legal_person": 1, "legal_person_caption": 1, "company_status": 1,
-		"authority": 1, "issue_date": 1, "operation_startdate": 1, "operation_enddate": 1, "capital": 1, "business_scope": 1,
-		"comeintime": 1, "updatetime": 1, "legal_person_type": 1, "real_capital": 1, "en_name": 1, "list_code": 1,
-		"employee_no": 1, "website": 1, "company_phone": 1, "company_email": 1,
-	}).Iter()
-	pool := make(chan bool, 10)
+	it := sess.DB("mixdata").C("qyxy_std").Find(&q).Sort("-_id").Iter()
+	pool := make(chan bool, 20)
 	wg := &sync.WaitGroup{}
 	total := 0
+	ctx := context.Background()
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
 		if total%1000 == 0 {
 			log.Println("cur index ", total)
@@ -200,7 +331,7 @@ func getPingAn() {
 				return
 			}
 			query := `SELECT id FROM information.ent_info WHERE company_name = ?`
-			rows, err := ent_util.ClickHouseConn.Query(context.Background(), query, company_name)
+			rows, err := ent_util.ClickHouseConn.Query(ctx, query, company_name)
 			if err != nil {
 				log.Println(err)
 			}
@@ -305,17 +436,11 @@ func getTQ() {
 		sess := ent_util.SpiMgo.GetMgoConn()
 		defer ent_util.SpiMgo.DestoryMongoConn(sess)
 		q := map[string]interface{}{}
-		it := sess.DB("mixdata").C(db).Find(&q).Select(map[string]interface{}{
-			"_id": 1, "use_flag": 1, "is_history": 1, "company_type": 1, "company_name": 1, "company_area": 1,
-			"company_city": 1, "company_district": 1, "company_address": 1, "company_code": 1, "credit_no": 1,
-			"org_code": 1, "tax_code": 1, "establish_date": 1, "legal_person": 1, "legal_person_caption": 1, "company_status": 1,
-			"authority": 1, "issue_date": 1, "operation_startdate": 1, "operation_enddate": 1, "capital": 1, "business_scope": 1,
-			"comeintime": 1, "updatetime": 1, "legal_person_type": 1, "real_capital": 1, "en_name": 1, "list_code": 1,
-			"employee_no": 1, "website": 1, "company_phone": 1, "company_email": 1,
-		}).Iter()
+		it := sess.DB("mixdata").C(db).Find(&q).Iter()
 		pool := make(chan bool, 10)
 		wg := &sync.WaitGroup{}
 		total := 0
+		ctx := context.Background()
 		for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
 			if total%1000 == 0 {
 				log.Println("cur index ", total)
@@ -333,7 +458,7 @@ func getTQ() {
 					return
 				}
 				query := `SELECT id FROM information.ent_info WHERE company_name = ?`
-				rows, err := ent_util.ClickHouseConn.Query(context.Background(), query, company_name)
+				rows, err := ent_util.ClickHouseConn.Query(ctx, query, company_name)
 				if err != nil {
 					log.Println(err)
 				}
@@ -438,8 +563,9 @@ func getMK() {
 	sess := ent_util.QyxyMgo.GetMgoConn()
 	defer ent_util.QyxyMgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{}
-	it := sess.DB("mixdata").C("qyxy_xzh").Find(&q).Sort("_id").Iter()
+	it := sess.DB("mixdata").C("qyxy_xzh").Find(&q).Iter()
 	pool := make(chan bool, 10)
+	ctx := context.Background()
 	wg := &sync.WaitGroup{}
 	total := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
@@ -458,7 +584,7 @@ func getMK() {
 				return
 			}
 			query := `SELECT id FROM information.ent_info WHERE company_name = ?`
-			rows, err := ent_util.ClickHouseConn.Query(context.Background(), query, company_name)
+			rows, err := ent_util.ClickHouseConn.Query(ctx, query, company_name)
 			if err != nil {
 				log.Println(err)
 			}
@@ -548,18 +674,16 @@ func getMK() {
 						"company_email":        company_email,
 					}
 					Ch <- data
-					ent_contact.InjectContactPingAnInfo(id, tmp)
+					go ent_contact.InjectContactPingAnInfo(id, tmp)
 				}
 			}
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
-	wg.Wait()
 	log.Println("make is over ~ ", total)
 }
 
 func getCompanyLabel(name string) ([]uint64, string) {
-	name = strings.TrimSpace(RegClean.ReplaceAllString(name, ""))
 	buyerclassArr, tag_subinformationArr, property_formArr := []string{}, []string{}, []string{}
 	bitMapCodeArr, bitMapStrArr := []uint64{}, []string{}
 	data := ent_util.ExtMgo.FindOne("label", map[string]interface{}{"name": name})
@@ -649,3 +773,26 @@ L:
 		}(&arr)
 	}
 }
+
+func gettime() []map[string]int64 {
+	startTime := int64(1630914780)
+	endTime := int64(1714348067)
+	numSegments := 200
+
+	duration := endTime - startTime
+	segmentDuration := duration / int64(numSegments)
+
+	result := make([]map[string]int64, numSegments)
+
+	for i := 0; i < numSegments; i++ {
+		segmentStart := startTime + (int64(i) * segmentDuration)
+		segmentEnd := segmentStart + segmentDuration
+
+		segment := make(map[string]int64)
+		segment["start"] = segmentStart
+		segment["end"] = segmentEnd
+
+		result[i] = segment
+	}
+	return result
+}

ent_util/global.go

@@ -2,12 +2,13 @@ package ent_util
 
 import (
 	"fmt"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"runtime"
 	"strconv"
 	"strings"
 	"unicode/utf8"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 )
 
 // 处理地域代码

ent_util/init.go

@@ -39,11 +39,11 @@ const (
 func InitGlobalVar() {
 	IsLocal = false
 	qu.ReadConfig(&SysConfig) //加载配置文件
-	//initMgo()
+	initMgo()
 	initMysql()
 	initClickHouse()
-	//initVCode()
-	//initEs()
+	initVCode()
+	initEs()
 }
 
 // 初始化mgo
@@ -57,7 +57,7 @@ func initMgo() {
 	SourceMgo = &MongodbSim{
 		MongodbAddr: qu.ObjToString(b_mgo["addr"]),
 		DbName:      qu.ObjToString(b_mgo["dbname"]),
-		Size:        10,
+		Size:        20,
 		UserName:    qu.ObjToString(b_mgo["username"]),
 		Password:    qu.ObjToString(b_mgo["password"]),
 	}
@@ -70,7 +70,7 @@ func initMgo() {
 	QyxyMgo = &MongodbSim{
 		MongodbAddr: qu.ObjToString(q_mgo["addr"]),
 		DbName:      qu.ObjToString(q_mgo["dbname"]),
-		Size:        10,
+		Size:        20,
 		UserName:    qu.ObjToString(q_mgo["username"]),
 		Password:    qu.ObjToString(q_mgo["password"]),
 	}
@@ -83,7 +83,7 @@ func initMgo() {
 	SpiMgo = &MongodbSim{
 		MongodbAddr: qu.ObjToString(s_mgo["addr"]),
 		DbName:      qu.ObjToString(s_mgo["dbname"]),
-		Size:        10,
+		Size:        20,
 		UserName:    qu.ObjToString(s_mgo["username"]),
 		Password:    qu.ObjToString(s_mgo["password"]),
 	}
@@ -96,7 +96,7 @@ func initMgo() {
 	ExtMgo = &MongodbSim{
 		MongodbAddr: qu.ObjToString(e_mgo["addr"]),
 		DbName:      qu.ObjToString(e_mgo["dbname"]),
-		Size:        10,
+		Size:        20,
 		UserName:    qu.ObjToString(e_mgo["username"]),
 		Password:    qu.ObjToString(e_mgo["password"]),
 	}
@@ -123,7 +123,6 @@ func initVCode() {
 	data_types := MysqlGlobalTool.Find("code_buyerclass", nil, "", "", -1, -1)
 	//先构建所有一级数据
 	for _, v := range *data_types {
-
 		name := qu.ObjToString(v["name"])
 		code := qu.ObjToString(v["code"])
 		BuyerClassData[name] = code

main.go

@@ -20,11 +20,11 @@ func init() {
 }
 
 func main() {
-	mode := flag.Int("m", 1, "")
+	mode := flag.Int("m", 2, "")
 	flag.Parse()
+	go ent_legal.Save()
 	if *mode == 1 {
 		//全量
-		go ent_legal.Save()
 		log.Println("全量任务")
 		// ent_contact.InjectContactTidbInfo()
 		ent_legal.LegalFull()
@@ -32,6 +32,7 @@ func main() {
 		select {}
 	} else {
 		//增量
+		// ent_legal.LegalAdd()
 		a := cron.New()
 		a.AddFunc("0 0 20 * * ?", func() {
 			ent_legal.LegalAdd()