browserSpider/program/spider/main.go

package main

import (
	"flag"
	"jycdp"
	"log"

	"os"
)

type (
	Config struct {
		RunModel               string `yaml:"run_model"`
		WebsocketURL           string `yaml:"websocket_url"` //仅远程浏览器有用
		ProxyAddr              string `yaml:"proxy_address"` //仅本地浏览器有用
		HeadLess               bool   `yaml:"headless"`
		ShowImage              bool   `yaml:"show_image"`
		StorageType            string `yaml:"storage"`
		StorageMongoURI        string `yaml:"mongo_address"`
		StorageMongoDB         string `yaml:"mongo_db"`
		StorageMongoCollection string `yaml:"mongo_colesstion"`
		StorageFile            string `yaml:"storage_file"`
		WeChatRobotURL         string `yaml:"wechat_robot_url"`
		DownloadPath           string `yaml:"download_path"`
	}
)

var (
	configFile = flag.String("cf", "./config.yaml", "配置文件")
	luaFile    = flag.String("sf", "./scripts/demo.lua", "lua文件")
	dataFile   = flag.String("df", "./data.dat", "数据文件")
	logFile    = flag.String("lf", "./log.dat", "日志文件")
	storage    Storage
	config     *Config = new(Config)
)

// init
func init() {
	flag.Parse()
	jycdp.LoadConfig(*configFile, config)
	var err error
	switch config.StorageType {
	case "console":
		storage = NewConsoleStorage()
	case "mongodb":
		storage = NewMongoStorage(config.StorageMongoURI,
			config.StorageMongoDB,
			config.StorageMongoCollection)
	case "wechatrobot":
		storage = NewWeChatMsgStorage(config.WeChatRobotURL)
	default:
		var theDataFile = *dataFile
		if theDataFile == "" {
			theDataFile = config.StorageFile
		}
		fi, err := os.OpenFile(theDataFile, os.O_CREATE|os.O_APPEND|os.O_RDWR|os.O_SYNC|os.O_TRUNC, 0777)
		if err != nil {
			panic(err)
		}
		storage = NewFileStorage(fi)
	}
	if *logFile != "" {
		sl, err = NewSpiderLog(*logFile)
		if err != nil {
			log.Fatal(err)
		}
	}
	//log.Println(config)
}

func main() {
	bs, err := os.ReadFile(*luaFile)
	if err != nil {
		log.Fatal(err)
	}

	script := string(bs)

	var vm *VM
	if config.RunModel == "local" {
		vm = NewLocalVM(config.HeadLess, config.ShowImage, config.ProxyAddr, config.DownloadPath, storage)
	} else {
		vm = NewRemoteVM(config.WebsocketURL, storage)
	}
	if err := vm.RunScript(script); err != nil {
		log.Println(err.Error())
	}
	//最后执行
	if sl != nil {
		sl.Close()
	}
}