jy-data-extract/fullproject/src2/main.go

package main

import (
	"qfw/util"
	"qfw/util/mongodb"
	"qfw/util/redis"
	"sync"
	"time"

	"gopkg.in/mgo.v2/bson"
)

const (
	REDISIDS    = "ids"
	REDISKEYS   = "keys"
	INFOID      = "info"
	INFOTIMEOUT = 86400 * 30
)

var (
	Sysconfig                map[string]interface{}
	MQFW                     mongodb.MongodbSim
	extractColl, projectColl string
	lenprojectname           int
	MultiThread              chan bool
	IdLock                   = &sync.Mutex{}
	PncbMayLock              = &sync.Mutex{}
	MegerFieldsLen           *MegerFields
	//三组lock,对应的（PNKey）key为项目名称,值对应的是此项目名称对应的项目id数组
	PNKey, PCKey, PBKey          = NewKeyMap(), NewKeyMap(), NewKeyMap()
	PNKeyMap, PCKeyMap, PBKeyMap = sync.Map{}, sync.Map{}, sync.Map{}

	currentMegerTime  int64 //合并项目的时间位置，用来清理几个月之前的项目
	currentMegerCount int   //合并项目的计数，用来定时清理
)

type MegerFields struct {
	ProjectNamelen int
	ProjectCodelen int
}

type KeyMap struct {
	Lock sync.Mutex
	Map  map[string]*Key
}

type Key struct {
	Arr  *[]string
	Lock *sync.Mutex
}

func NewKeyMap() *KeyMap {
	return &KeyMap{
		Map: map[string]*Key{},
	}
}

func init() {
	initarea()

	util.ReadConfig(&Sysconfig)
	MultiThread = make(chan bool, util.IntAllDef(Sysconfig["thread"], 200))
	lenprojectname = util.IntAllDef(Sysconfig["lenprojectname"], 20) - 1
	megerfields, _ := Sysconfig["megerfields"].(map[string]interface{})
	MegerFieldsLen = &MegerFields{
		ProjectNamelen: util.IntAllDef(megerfields["projectlen"], 5),
		ProjectCodelen: util.IntAllDef(megerfields["projectcodelen"], 8),
	}
	redis.InitRedisBySize(Sysconfig["redisaddrs"].(string), util.IntAllDef(Sysconfig["redisPoolSize"], 100), 30, 300)
	MQFW = mongodb.MongodbSim{
		MongodbAddr: Sysconfig["mongodbServers"].(string),
		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
		DbName:      Sysconfig["mongodbName"].(string),
	}
	MQFW.InitPool()
	extractColl = Sysconfig["extractColl"].(string)
	projectColl = Sysconfig["projectColl"].(string)
}

func main() {
	RunFullData()
	time.Sleep(99999 * time.Hour)
}

func NewPushInfo(tmp map[string]interface{}) bson.M {
	return bson.M{
		"comeintime":  tmp["comeintime"],
		"publishtime": tmp["publishtime"],
		"title":       tmp["title"],
		"toptype":     tmp["toptype"],
		"subtype":     tmp["subtype"],
		"infoformat":  tmp["infoformat"],
		"infoid":      util.BsonIdToSId(tmp["_id"]),
		"href":        tmp["href"],
		"area":        tmp["area"],
		"city":        tmp["city"],

		"cresult": tmp["cresult"],
		"score":   tmp["score"],
	}
}