|
@@ -0,0 +1,130 @@
|
|
|
|
+// 爬虫调试绑定
|
|
|
|
+package main
|
|
|
|
+
|
|
|
|
+import (
|
|
|
|
+ "log"
|
|
|
|
+ "sort"
|
|
|
|
+ be "spidercreator/backend"
|
|
|
|
+ bdb "spidercreator/backend/db"
|
|
|
|
+)
|
|
|
|
+
|
|
|
|
+// DebugSpider 调试爬虫
|
|
|
|
+func (a *App) DebugSpider(url string, maxPages int, listDealy int64, trunPageDelay int64, contentDelay int64, headless bool,
|
|
|
|
+ showImage bool, proxyServe string, threads int) {
|
|
|
|
+ exitCh = make(chan bool, 1)
|
|
|
|
+ currentResults.Init()
|
|
|
|
+ if maxPages == 1 && threads == 1 {
|
|
|
|
+ vm.RunSpider(url, maxPages, listDealy, contentDelay, headless, showImage, proxyServe, exitCh,
|
|
|
|
+ currentSpiderConfig, currentResults)
|
|
|
|
+ } else { //多页下载强制使用多线程模式
|
|
|
|
+ vm.RunSpiderMulThreads(url, maxPages, listDealy, trunPageDelay, contentDelay, headless, showImage, proxyServe, threads, exitCh,
|
|
|
|
+ currentSpiderConfig, currentResults)
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// 停止调试
|
|
|
|
+func (a *App) StopDebugSpider() string {
|
|
|
|
+ defer func() {
|
|
|
|
+ if err := recover(); err != nil {
|
|
|
|
+ log.Println(err)
|
|
|
|
+ }
|
|
|
|
+ }()
|
|
|
|
+ exitCh <- true
|
|
|
|
+ return "ok"
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// ViewResultItemAll 查看所有结果,只显示最近的50条
|
|
|
|
+func (a *App) ViewResultItemAll() be.ResultItems {
|
|
|
|
+ ret := make(be.ResultItems, 0)
|
|
|
|
+ index := 0
|
|
|
|
+ for el := currentResults.Front(); el != nil; el = el.Next() {
|
|
|
|
+ if index > 50 {
|
|
|
|
+ break
|
|
|
|
+ }
|
|
|
|
+ index += 1
|
|
|
|
+ v, _ := el.Value.(*be.ResultItem)
|
|
|
|
+ ret = append(ret, v)
|
|
|
|
+ }
|
|
|
|
+ return ret
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// ExportEpubFile
|
|
|
|
+func (a *App) ExportEpubFile(bookname, filepath string) string {
|
|
|
|
+ log.Println("EPUB 文件存储:", bookname, filepath)
|
|
|
|
+ db.ExportEpubFile(bookname, filepath, currentResults)
|
|
|
|
+ return "ok"
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// ImportSpiderConfigByExcelFile 通过excel文件导入爬虫配置
|
|
|
|
+func (a *App) ImportSpiderConfigByExcelFile(filepath string) string {
|
|
|
|
+ db.BatchImport(filepath)
|
|
|
|
+ return "ok"
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// CountYestodayArts
|
|
|
|
+func (a *App) CountYestodayArts(url string, listDealy int64,
|
|
|
|
+ trunPageDelay int64, headless bool, showImage bool) {
|
|
|
|
+ exitCh = make(chan bool, 1)
|
|
|
|
+ vm.CountYestodayArts(url, listDealy, trunPageDelay, headless, showImage, exitCh, currentSpiderConfig)
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// ExportExcelFile
|
|
|
|
+func (a *App) ExportExcelFile(filepath string) string {
|
|
|
|
+ if err := db.ExportExcelFile(filepath, currentSpiderConfig.Site, currentSpiderConfig.Channel, currentResults); err == nil {
|
|
|
|
+ return "ok"
|
|
|
|
+ } else {
|
|
|
|
+ return err.Error()
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// LoadAllJobs 加载我的所有作业
|
|
|
|
+func (a *App) LoadAllJobs() be.Jobs {
|
|
|
|
+ rs, err := bdb.LoadEntities[be.Job]("jobs")
|
|
|
|
+ if err != nil {
|
|
|
|
+ return make(be.Jobs, 0)
|
|
|
|
+ }
|
|
|
|
+ jobs := be.Jobs(rs)
|
|
|
|
+ sort.Sort(jobs)
|
|
|
|
+ return jobs
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// SaveJob
|
|
|
|
+func (a *App) SaveJob(job *be.Job) string {
|
|
|
|
+ err := bdb.SaveEntity[be.Job]("jobs", job.Code, job)
|
|
|
|
+ if err != nil {
|
|
|
|
+ return err.Error()
|
|
|
|
+ }
|
|
|
|
+ return "ok"
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// DeleteJob
|
|
|
|
+func (a *App) DeleteJob(code string) string {
|
|
|
|
+ err := bdb.DeleteEntity[be.Job]("jobs", code)
|
|
|
|
+ if err != nil {
|
|
|
|
+ return err.Error()
|
|
|
|
+ }
|
|
|
|
+ return "ok"
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// LoadJob
|
|
|
|
+func (a *App) LoadJob(code string) *be.Job {
|
|
|
|
+ job, _ := bdb.LoadEntity[be.Job]("jobs", code)
|
|
|
|
+ return job
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// RunJob
|
|
|
|
+func (a *App) RunJob(code string) string {
|
|
|
|
+ go vm.RunJob(code)
|
|
|
|
+ return "ok"
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+func (a *App) StopJob(code string) string {
|
|
|
|
+ vm.StopJob(code)
|
|
|
|
+ return "ok"
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// ExportJobResult
|
|
|
|
+func (a *App) ExportJobResult(code string, filePath string) string {
|
|
|
|
+ vm.ExportJobResult(code, filePath)
|
|
|
|
+ return "ok"
|
|
|
|
+}
|