spider_creater/backend/types.go

package backend

import (
	"container/list"
	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
)

const (
	JOB_RUNNING_EVENT_PROGRESS = 1
	JOB_RUNNING_EVENT_DEBUG    = 0
)

type (
	//爬虫配置信息
	SpiderConfig struct {
		Site               string `json:"site"`
		Channel            string `json:"channel"`
		Author             string `json:"author"`
		Url                string `json:"url"`
		Code               string `json:"code"`
		ListBodyCss        string `json:"listBodyCss"`
		ListItemCss        string `json:"listItemCss"`
		ListLinkCss        string `json:"listLinkCss"`
		ListPubtimeCss     string `json:"listPublishTimeCss"`
		ListNextPageCss    string `json:"listNextPageCss"`
		TitleCss           string `json:"titleCss"`
		PublishUnitCss     string `json:"publishUnitCss"`
		PublishTimeCss     string `json:"publishTimeCss"`
		ContentCss         string `json:"contentCss"`
		AttachCss          string `json:"attachCss"`
		ListJSCode         string `json:"listJs"`
		ContentJSCode      string `json:"contentJs"`
		AttachJSCode       string `json:"attachJs"` //无效
		ListTrunPageJSCode string `json:"listTrunPageJs"`
	}
	//附件链接
	AttachLink struct {
		Title    string `json:"title"`
		Href     string `json:"href"`
		FileName string `json:"fileName"`
		FileType string `json:"fileType"`
		FileSize string `json:"fileSize"`
		FilePath string `json:"filePath"`
	}
	//爬取结果信息
	ResultItem struct {
		No      int    `json:"no"` //编号
		Site    string `json:"site"`
		Channel string `json:"channel"`

		Href        string        `json:"href"`
		ListTitle   string        `json:"listTitle"`
		ListPubTime string        `json:"listPubishTime"`
		Title       string        `json:"title"`
		PublishUnit string        `json:"publishUnit"`
		PublishTime string        `json:"publishTime"`
		Content     string        `json:"content"`
		ContentHtml string        `json:"contentHtml"`
		AttachLinks []*AttachLink `json:"attachLinks"` //存放附件的标题，链接
		AttachJson  string        `json:"attachJson"`  //存放附件的OSS元信息
	}
	//爬取结果信息清单
	ResultItems []*ResultItem

	SpiderConfiges []*SpiderConfig
	//
	JobItem struct {
		//code/site与爬虫配置一致
		SpiderCode string `json:"code"`
		SpiderSite string `json:"site"`
		//以下是运行参数(不配置时，默认使用通用配置）
		Channel              string `json:"channel"`
		Url                  string `json:"url"`
		ProxyServe           string `json:"proxyServe"`
		MaxPages             int    `json:"maxPages"`
		Threads              int    `json:"threads"`
		ListDelay            int64  `json:"listDelay"`
		TrunPageDelay        int64  `json:"trunPageDelay"`
		ContentDelay         int64  `json:"contentDelay"`
		NeedDownloadAttaches bool   `json:"needDownloadAttaches"`
	}
	//作业定义
	Job struct {
		Code  string     `json:"code"`
		Name  string     `json:"name"`
		Items []*JobItem `json:"items"`
		//通用参数
		ProxyServe           string `json:"proxyServe"`
		MaxPages             int    `json:"maxPages"`
		Threads              int    `json:"threads"`
		ListDelay            int64  `json:"listDelay"`
		TrunPageDelay        int64  `json:"trunPageDelay"`
		ContentDelay         int64  `json:"contentDelay"`
		State                int    `json:"state"`
		StateType            string `json:"stateType"`
		Progress             int    `json:"progress"`
		NeedDownloadAttaches bool   `json:"needDownloadAttaches"`
	}
	Jobs []*Job
	//推送事件
	JobRunningEvent struct {
		Progress int    `json:"progress"`
		Msg      string `json:"msg"`
		Act      int    `json:"act"`
		Code     string `json:"code"`
	}
	//Job运行时状态，中间结果
	JobRunningState struct {
		Code        string
		State       int
		Progress    int
		ResultCache *list.List //缓存
		ExitCh      chan bool
	}

	//爬虫配置验证结果
	SpiderConfigVerifyResult struct {
		Title        bool `json:"title"`
		PublishUnit  bool `json:"publishUnit"`
		PublishTime  bool `json:"publishTime"`
		Content      bool `json:"content"`
		Attaches     bool `json:"attaches"`
		ListItems    bool `json:"listItems"`
		ListTrunPage bool `json:"listTrunPage"`
	}
)

func (sc SpiderConfiges) Len() int {
	return len(sc)
}
func (sc SpiderConfiges) Swap(i, j int) {
	sc[i], sc[j] = sc[j], sc[i]
}
func (sc SpiderConfiges) Less(i, j int) bool {
	return sc[i].Code > sc[j].Code
}

func (js Jobs) Len() int {
	return len(js)
}
func (js Jobs) Swap(i, j int) {
	js[i], js[j] = js[j], js[i]
}
func (js Jobs) Less(i, j int) bool {
	return js[i].Code > js[j].Code
}

// CopyAttribute
func CopyAttribute(dst *string, value1, value2 string) {
	if value1 == "EMPTY" { //特殊符号，强制修改
		*dst = ""
	} else if value1 != "" {
		*dst = value1
	} else if value2 != "" {
		*dst = value2
	}
}

// NewSpiderConfig 生成css对象
func NewSpiderConfig(param map[string]interface{}) *SpiderConfig {
	param_common := param["param_common"].(map[string]interface{})
	css_list, _ := param["css_list"].(map[string]interface{})
	css_content, _ := param["css_content"].(map[string]interface{})
	js_list, _ := param["js_list"].(map[string]interface{})
	js_nextpage, _ := param["js_nextpage"].(map[string]interface{})
	js_content, _ := param["js_content"].(map[string]interface{})
	sc := &SpiderConfig{
		Site:               qu.ObjToString(param_common["site"]),
		Channel:            qu.ObjToString(param_common["channel"]),
		Author:             qu.ObjToString(param_common["modifyuser"]),
		Url:                qu.ObjToString(param_common["href"]),
		Code:               qu.ObjToString(param_common["code"]),
		ListBodyCss:        qu.ObjToString(css_list["body"]),
		ListItemCss:        qu.ObjToString(css_list["title"]),
		ListLinkCss:        qu.ObjToString(css_list["href"]),
		ListPubtimeCss:     qu.ObjToString(css_list["ptime"]),
		ListNextPageCss:    qu.ObjToString(css_list["nextpage"]),
		TitleCss:           qu.ObjToString(css_content["title"]),
		PublishUnitCss:     qu.ObjToString(css_content["source"]),
		PublishTimeCss:     qu.ObjToString(css_content["ptime"]),
		ContentCss:         qu.ObjToString(css_content["content"]),
		AttachCss:          qu.ObjToString(css_content["file"]),
		ListJSCode:         qu.ObjToString(js_list["js"]),
		ContentJSCode:      qu.ObjToString(js_content["js"]),
		ListTrunPageJSCode: qu.ObjToString(js_nextpage["js"]),
		//AttachJSCode    :     string `json:"attachJs"` //无效
	}
	return sc
}

// MergeSpiderConfig 合并
func MergeSpiderConfig(src1, src2 *SpiderConfig) *SpiderConfig {
	nsc := new(SpiderConfig)
	CopyAttribute(&nsc.Code, src2.Code, src1.Code)
	CopyAttribute(&nsc.Site, src2.Site, src1.Site)
	CopyAttribute(&nsc.Channel, src2.Channel, src1.Channel)
	CopyAttribute(&nsc.Url, src2.Url, src1.Url)
	CopyAttribute(&nsc.Author, src2.Author, src1.Author)
	CopyAttribute(&nsc.ListBodyCss, src2.ListBodyCss, src1.ListBodyCss)
	CopyAttribute(&nsc.ListItemCss, src2.ListItemCss, src1.ListItemCss)
	CopyAttribute(&nsc.ListLinkCss, src2.ListLinkCss, src1.ListLinkCss)
	CopyAttribute(&nsc.ListPubtimeCss, src2.ListPubtimeCss, src1.ListPubtimeCss)
	CopyAttribute(&nsc.ListNextPageCss, src2.ListNextPageCss, src1.ListNextPageCss)
	CopyAttribute(&nsc.TitleCss, src2.TitleCss, src1.TitleCss)
	CopyAttribute(&nsc.PublishTimeCss, src2.PublishTimeCss, src1.PublishTimeCss)
	CopyAttribute(&nsc.PublishUnitCss, src2.PublishUnitCss, src1.PublishUnitCss)
	CopyAttribute(&nsc.ContentCss, src2.ContentCss, src1.ContentCss)
	CopyAttribute(&nsc.AttachCss, src2.AttachCss, src1.AttachCss)
	CopyAttribute(&nsc.ListJSCode, src2.ListJSCode, src1.ListJSCode)
	CopyAttribute(&nsc.ContentJSCode, src2.ContentJSCode, src1.ContentJSCode)
	CopyAttribute(&nsc.AttachJSCode, src2.AttachJSCode, src1.AttachJSCode)
	CopyAttribute(&nsc.ListTrunPageJSCode, src2.ListTrunPageJSCode, src1.ListTrunPageJSCode)
	return nsc
}

// CopySpiderConfig 复制
func CopySpiderConfig(src1, src2 *SpiderConfig) {
	src1.Code = src2.Code
	src1.Site = src2.Site
	src1.Author = src2.Author
	src1.Channel = src2.Channel
	src1.Url = src2.Url
	src1.ListBodyCss = src2.ListBodyCss
	src1.ListItemCss = src2.ListItemCss
	src1.ListPubtimeCss = src2.ListPubtimeCss
	src1.ListNextPageCss = src2.ListNextPageCss
	src1.ListLinkCss = src2.ListLinkCss
	src1.TitleCss = src2.TitleCss
	src1.PublishTimeCss = src2.PublishTimeCss
	src1.PublishUnitCss = src2.PublishUnitCss
	src1.ContentCss = src2.ContentCss
	src1.AttachCss = src2.AttachCss
	src1.ListJSCode = src2.ListJSCode
	src1.ListTrunPageJSCode = src2.ListTrunPageJSCode
	src1.ContentJSCode = src2.ContentJSCode
	src1.AttachJSCode = src2.AttachJSCode
}