data_extract/src/jy/extract/extractflow.go

package extract

import (
	log "github.com/donnie4w/go-logger/logger"
	db "jy/mongodbutil"
	ju "jy/util"
	qu "qfw/util"
)

// 流模式

var ExtFlow *ExtractTask

func InitExtractFlowTask() {
	if ExtFlow == nil {
		ExtFlow = nil
		ExtFlow = &ExtractTask{}
		ExtFlow.Id = qu.ObjToString(ju.Config["udptaskid"])
		ExtFlow.InitTaskInfo()
		ExtFlow.TaskInfo.FDB = db.MgoFactory(3, 5, 600, ExtFlow.TaskInfo.FromDbAddr, ExtFlow.TaskInfo.FromDB)
		ExtFlow.TaskInfo.TDB = db.MgoFactory(3, 5, 600, ExtFlow.TaskInfo.ToDbAddr, ExtFlow.TaskInfo.ToDB)
		ExtFlow.InitSite()
		ExtFlow.InitRulePres()
		ExtFlow.InitRuleBacks(false)
		ExtFlow.InitRuleBacks(true)
		ExtFlow.InitRuleCore(false)
		ExtFlow.InitRuleCore(true)
		ExtFlow.InitBlockRule()
		ExtFlow.InitPkgCore()
		ExtFlow.InitTag(false)
		ExtFlow.InitTag(true)
		ExtFlow.InitClearFn(false)
		ExtFlow.InitClearFn(true)
		ExtFlow.Lock()
		if ExtFlow.IsExtractCity { //版本上控制是否开始城市抽取
			ExtFlow.InitCityInfo()
			ExtFlow.InitAreaCode()
			ExtFlow.InitPostCode()
		}
		ExtFlow.Unlock()
		//质量审核
		ExtFlow.InitAuditFields()
		ExtFlow.InitAuditRule()
		ExtFlow.InitAuditClass()
		ExtFlow.InitAuditRecogField()
		//品牌抽取是否开启
		ju.IsBrandGoods, _ = ju.Config["brandgoods"].(bool)
		ExtFlow.InitFile()
		ExtFlow.IsRun = true
		ExtFlow.BidTotal = 0
	}
}

func ExtractByExtFlow(v map[string]interface{}) map[string]interface{} {
	defer qu.Catch()
	if ju.IsUpdateRule {
		ju.IsUpdateRule = false
		log.Debug("每天更新一次规则......")
		//规则重置
		ExtFlow.InitSite()
		ExtFlow.InitRulePres()
		ExtFlow.InitRuleBacks(false)
		ExtFlow.InitRuleBacks(true)
		ExtFlow.InitRuleCore(false)
		ExtFlow.InitRuleCore(true)
		ExtFlow.InitBlockRule()
		ExtFlow.InitPkgCore()
		ExtFlow.InitTag(false)
		ExtFlow.InitTag(true)
		ExtFlow.InitClearFn(false)
		ExtFlow.InitClearFn(true)
		//地域重置
		ExtFlow.Lock()
		if ExtFlow.IsExtractCity { //版本上控制是否开始城市抽取
			ExtFlow.InitCityInfo()
			ExtFlow.InitAreaCode()
			ExtFlow.InitPostCode()
		}
		ExtFlow.Unlock()
	}
	if spidercode[qu.ObjToString(v["spidercode"])] { //临时开标记录
		log.Debug(qu.BsonIdToSId(v["_id"]), "//开标记录")
		return v
	}
	//......
	var j, jf *ju.Job
	var isSite bool
	if ExtFlow.IsFileField && (v["projectinfo"] != nil || v["attach_text"] != nil) {
		v["isextFile"] = true
		j, jf, isSite = ExtFlow.PreInfo(v)
	} else {
		j, _, isSite = ExtFlow.PreInfo(v)
	}
	return ExtFlow.ExtractProcessFlow(j, jf, isSite)
}