vm.go 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227
  1. /**
  2. * 虚拟机
  3. */
  4. package main
  5. import (
  6. "context"
  7. "fmt"
  8. "log"
  9. "strings"
  10. "github.com/chromedp/chromedp"
  11. "github.com/yuin/gopher-lua"
  12. "github.com/yuin/gopher-lua/parse"
  13. )
  14. const (
  15. run_on_device_remote = iota
  16. run_on_device_local
  17. )
  18. type (
  19. //虚拟机
  20. VM struct {
  21. WsAddr string
  22. Headless bool
  23. ShowImage bool
  24. ProxyAddr string
  25. RunMode int
  26. DownloadPath string
  27. B *Browser
  28. S Storage
  29. }
  30. //浏览器,(不用之前封装的,这个更轻量)
  31. Browser struct {
  32. Ctx context.Context
  33. CancelFn context.CancelFunc
  34. }
  35. )
  36. // NewVM
  37. func NewRemoteVM(wsAddr string, s Storage) *VM {
  38. return &VM{WsAddr: wsAddr, RunMode: run_on_device_remote, S: s}
  39. }
  40. // NewVM
  41. func NewLocalVM(headless bool, showImage bool, proxyAddr, downloadPath string, s Storage) *VM {
  42. return &VM{Headless: headless, ProxyAddr: proxyAddr, ShowImage: showImage,
  43. RunMode: run_on_device_local,
  44. DownloadPath: downloadPath,
  45. S: s}
  46. }
  47. // Quit
  48. func (b *Browser) Quit() {
  49. if b != nil && b.CancelFn != nil {
  50. b.CancelFn()
  51. b.Ctx = nil
  52. b.CancelFn = nil
  53. }
  54. }
  55. // createRemoteBrowser 创建远程浏览器
  56. func createRemoteBrowser(wsAddr string) *Browser {
  57. allocCtx, cancelFn := chromedp.NewRemoteAllocator(context.TODO(),
  58. wsAddr)
  59. incCtx, _ := chromedp.NewContext(allocCtx)
  60. return &Browser{
  61. incCtx, cancelFn,
  62. }
  63. }
  64. // createLocalBrowser 创建本地浏览器
  65. func createLocalBrowser(headless,
  66. showImage bool,
  67. proxyAddr, downloadPath string) *Browser {
  68. baseCtx, _ := chromedp.NewContext(context.Background())
  69. chromeOptions := []chromedp.ExecAllocatorOption{
  70. chromedp.NoFirstRun,
  71. chromedp.NoDefaultBrowserCheck,
  72. chromedp.DisableGPU,
  73. chromedp.NoSandbox,
  74. chromedp.WindowSize(1920, 1080),
  75. chromedp.Flag("enable-automation", false), // 防止监测webdriver
  76. chromedp.Flag("disable-blink-features", "AutomationControlled"), //禁用 blink 特征 作者:知识货栈 https://www.bilibili.com/read/cv24371371/ 出处:bilibili
  77. chromedp.Flag("lang", "zh-CN"),
  78. chromedp.Flag("mixed-forms-disable-autofill", false), //从https转http不再检查
  79. chromedp.Flag("ignore-certificate-errors", true), //忽略错误
  80. chromedp.Flag("ignore-urlfetcher-cert-requests", true),
  81. chromedp.Flag("enable-automation", false), // 防止监测webdriver
  82. chromedp.Flag("disable-blink-features", "AutomationControlled"), //禁用 blink 特征
  83. chromedp.Flag("force-dev-mode-highlighting", true),
  84. chromedp.Flag("disable-extensions", false), //是否禁用扩展
  85. chromedp.Flag("headless", headless),
  86. chromedp.Flag("user-agent", "Chrome 9 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/93.0.4577.58 Safari/537.36 Edg/93.0.961.33"),
  87. chromedp.Flag("disable-keep-alive", true),
  88. chromedp.Flag("disable-dev-shm-usage", false),
  89. chromedp.Flag("default-browser-check", false),
  90. chromedp.Flag("disable-web-security", true), //禁用网络安全标志
  91. chromedp.Flag("mute-audio", false),
  92. chromedp.Flag("https-upgrades", "disabled"),
  93. chromedp.Flag("accept-language", `zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-TW;q=0.6`),
  94. //chromedp.Flag("blink-settings", "imagesEnabled=true"),
  95. //chromedp.Flag("incognito", true), //隐私模式
  96. chromedp.Flag("disable-cache", true), //不用缓存
  97. }
  98. if proxyAddr != "" {
  99. chromeOptions = append(chromeOptions,
  100. chromedp.ProxyServer(fmt.Sprintf("socks5://%s", proxyAddr)))
  101. }
  102. if downloadPath != "" {
  103. chromeOptions = append(chromeOptions,
  104. chromedp.Flag("download-path", downloadPath))
  105. }
  106. if showImage {
  107. chromeOptions = append(chromeOptions,
  108. chromedp.Flag("blink-settings", "imagesEnabled=true"),
  109. )
  110. } else {
  111. chromeOptions = append(chromeOptions,
  112. chromedp.Flag("blink-settings", "imagesEnabled=false"),
  113. )
  114. }
  115. allocCtx, _ := chromedp.NewExecAllocator(baseCtx, chromeOptions...)
  116. // 创建一个浏览器实例
  117. incCtx, incCancelFn := chromedp.NewContext(allocCtx,
  118. chromedp.WithLogf(log.Printf))
  119. return &Browser{
  120. incCtx, incCancelFn,
  121. }
  122. }
  123. // 重置浏览器
  124. func (vm *VM) ResetBrowser() {
  125. if vm.B != nil && vm.B.CancelFn != nil {
  126. vm.B.CancelFn()
  127. vm.B.Ctx = nil
  128. vm.B.CancelFn = nil
  129. }
  130. var b *Browser
  131. if vm.RunMode == run_on_device_local {
  132. b = createLocalBrowser(vm.Headless, vm.ShowImage, vm.ProxyAddr, vm.DownloadPath)
  133. } else {
  134. b = createRemoteBrowser(vm.WsAddr)
  135. }
  136. if vm.B == nil {
  137. vm.B = b
  138. } else {
  139. vm.B.Ctx, vm.B.CancelFn = b.Ctx, b.CancelFn
  140. }
  141. }
  142. // BindLuaState 绑定虚拟机函数
  143. func (vm *VM) BindLuaState(state *lua.LState) {
  144. state.SetGlobal("browser_reset", state.NewFunction(func(l *lua.LState) int {
  145. vm.ResetBrowser()
  146. return 0
  147. }))
  148. state.SetGlobal("browser_save", state.NewFunction(func(l *lua.LState) int {
  149. spiderCode := l.ToString(-5)
  150. siteName := l.ToString(-4)
  151. siteChannelName := l.ToString(-3)
  152. siteChannelUrl := l.ToString(-2)
  153. table := l.ToTable(-1)
  154. data := TableToMap(table)
  155. vm.S.Save(spiderCode, siteName, siteChannelName, siteChannelUrl, data)
  156. return 0
  157. }))
  158. state.SetGlobal("browser_url_last_segs", state.NewFunction(func(l *lua.LState) int {
  159. segs := l.ToInt(-2)
  160. href := l.ToString(-1)
  161. if segs == 0 {
  162. segs = 2
  163. }
  164. s := urlLastSegs(href, segs)
  165. l.Push(lua.LString(s))
  166. return 1
  167. }))
  168. //最多传10个string参数,不支持其他类型
  169. state.SetGlobal("browser_log", state.NewFunction(func(l *lua.LState) int {
  170. params := []string{}
  171. for i := -10; i < 0; i++ {
  172. p := l.ToString(i)
  173. if p != "" {
  174. params = append(params, p)
  175. }
  176. }
  177. if sl != nil {
  178. sl.Log(params...)
  179. }
  180. return 0
  181. }))
  182. }
  183. // runScript 执行lua代码
  184. func (vm *VM) RunScript(script string) error {
  185. defer Catch()
  186. var state *lua.LState = lua.NewState()
  187. defer state.Close()
  188. //方法绑定
  189. vm.ResetBrowser() //先创建浏览器对象
  190. vm.BindLuaState(state)
  191. vm.B.BindLuaState(state)
  192. defer func() {
  193. if vm.B != nil {
  194. vm.B.Quit()
  195. }
  196. }()
  197. reader := strings.NewReader(script)
  198. chunk, err := parse.Parse(reader, "code")
  199. if err != nil {
  200. return err
  201. }
  202. proto, err := lua.Compile(chunk, script)
  203. if err != nil {
  204. return err
  205. }
  206. lfunc := state.NewFunctionFromProto(proto)
  207. state.Push(lfunc)
  208. state.Call(0, 0)
  209. return nil
  210. }