spider_creater/app.go

package main

import (
	"context"
	"fmt"
	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"os"
	rt "runtime"
	bdb "spider_creator/backend/db"
	"spider_creator/backend/script"
	bvm "spider_creator/backend/vm"
	bws "spider_creator/backend/webservice"
)

func getDownloadDir() (string, error) {
	var downloadDir string
	var err error

	switch rt.GOOS {
	case "windows":
		downloadDir = os.Getenv("USERPROFILE") + "\\Downloads"
	case "darwin":
		downloadDir = os.Getenv("HOME") + "/Downloads"
	case "linux":
		downloadDir = os.Getenv("HOME") + "/Downloads"
	default:
		err = fmt.Errorf("unsupported operating system")
	}

	return downloadDir, err
}

// App struct
type App struct {
	ctx context.Context
}

// NewApp creates a new App application struct
func NewApp() *App {
	return &App{}
}

// startup is called when the app starts. The context is saved
// so we can call the runtime methods
func (a *App) startup(ctx context.Context) {
	a.ctx = ctx
	if rt.GOOS == "darwin" {
		baseDir = os.Getenv("HOME") + "/Downloads"
	} else {
		baseDir = "."
	}
	//创建附件下载目录
	attachesDir = baseDir + "/spider_attaches"
	if _, err := os.Stat(attachesDir); err != nil {
		os.MkdirAll(attachesDir, 0777)
	}
	//创建千里马数据目录
	//脚本目录
	qlmScriptDir := baseDir + "/special/script"
	qlmLogsDir := baseDir + "/special/logs"
	if _, err := os.Stat(qlmScriptDir); err != nil {
		os.MkdirAll(qlmScriptDir, 0777)
	}
	if _, err := os.Stat(qlmLogsDir); err != nil {
		os.MkdirAll(qlmLogsDir, 0777)
	}
	var dbfile = baseDir + "/spider.dat"
	qu.Debug("db file:", dbfile)

	db = bdb.NewSpiderDb(dbfile, a)
	bdb.Db = db
	vm = bvm.NewVM(attachesDir, a)
	glvm = script.NewGLVM(qlmScriptDir, qlmLogsDir, a)
	ws = bws.NewWebService(db, a, currentSpiderConfig)
	//
	go ws.RunHttpServe()
}

// destory
func (a *App) destory(ctx context.Context) {
	db.Close()
}

//var (
//	//db     *SpiderDb
//	exitCh chan bool
//)

// App struct
//type App struct {
//	ctx context.Context
//}

// NewApp creates a new App application struct
//func NewApp() *App {
//	return &App{}
//}

// startup is called when the app starts. The context is saved
// so we can call the runtime methods
//func (a *App) startup(ctx context.Context) {
//	a.ctx = ctx
//	db = NewSpiderDb("./data.db")
//}

// destory
//func (a *App) destory(ctx context.Context) {
//	db.Close()
//}

// Greet returns a greeting for the given name
//func (a *App) Greet(name string) string {
//	return fmt.Sprintf("Hello %s, It's show time!", name)
//}

//// LoadSpiderConfigAll，带分页
//func (a *App) LoadSpiderConfigAll(pageSize, pageNo int) []*SpiderConfig {
//	return db.LoadAll()
//}
//
//// LoadSpiderConfigAll，带分页
//func (a *App) SaveOrUpdateSpiderConfig(sc *SpiderConfig) string {
//	db.SaveOrUpdate(sc)
//	return "ok"
//}

// SwitchSpiderConfig
//func (a *App) SwitchSpiderConfig(code string) string {
//	qu.Debug("切换当前默认爬虫配置：", code)
//	db.Switch(code)
//	return "ok"
//}

// SwitchSpiderConfig
//func (a *App) ViewCurrentSpiderConfig() *SpiderConfig {
//	return currentSpiderConfig
//}

// SwitchSpiderConfig
//func (a *App) DeleteSpiderConfig(code string) string {
//	db.Delete(code)
//	return "ok"
//}

// 推送消息
//func (a *App) pushMessage(event string, data interface{}) {
//	runtime.EventsEmit(a.ctx, event, data)
//}

// 调试爬虫
//func (a *App) DebugSpider(url string, listDealy int64, contentDelay int64, headless bool, showImage bool, proxyServe string) {
//	exitCh = make(chan bool, 1)
//	RunSpider(url, listDealy, contentDelay, headless, showImage, proxyServe, exitCh)
//}

// 停止调试
//func (a *App) StopDebugSpider() string {
//	defer func() {
//		if err := recover(); err != nil {
//			qu.Debug(err)
//		}
//	}()
//	exitCh <- true
//	return "ok"
//}

// 查看所有结果
//func (a *App) ViewResultItemAll() ResultItems {
//	return currentResult
//}
//
//// ExportEpubFile
//func (a *App) ExportEpubFile(filepath string) string {
//	ExportEpubFile(filepath)
//	return "ok"
//}

// SelectSaveFilePath
//func (a *App) SelectSaveFilePath() string {
//	path, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{Filters: []runtime.FileFilter{
//		{Pattern: "*.epub", DisplayName: "epub file *.epub"},
//		{Pattern: "*.xlsx", DisplayName: "excel file *.xlsx"},
//		{Pattern: "*.json", DisplayName: "json file *.json"},
//	}})
//	if err != nil {
//		qu.Debug(err.Error())
//		return ""
//	}
//	return path
//}

// SelectOpenFilePath
//func (a *App) SelectOpenFilePath() string {
//	path, err := runtime.OpenFileDialog(a.ctx, runtime.OpenDialogOptions{Filters: []runtime.FileFilter{
//		{Pattern: "*.xlsx", DisplayName: "excel file *.xlsx"},
//	}})
//	if err != nil {
//		qu.Debug(err.Error())
//		return ""
//	}
//	return path
//}

// ImportSpiderConfigByExcelFile 通过excel文件导入爬虫配置
//func (a *App) ImportSpiderConfigByExcelFile(filepath string) string {
//	db.BatchImport(filepath)
//	return "ok"
//}

// 获取login状态
//func (a *App) GetLoginState() bool {
//	return loginState
//}
//
//func (a *App) PutLoginState(state bool) string {
//	loginState = state
//	return "ok"
//}

// CountYestodayArts
//func (a *App) CountYestodayArts(url string, listDealy int64,
//	trunPageDelay int64, headless bool, showImage bool) {
//	exitCh = make(chan bool, 1)
//	CountYestodayArts(url, listDealy, trunPageDelay, headless, showImage, exitCh)
//}