jy-data-extract/fullproject/src_dev1/main.go

package main

import (
	"qfw/util"
	"qfw/util/mongodb"
	"qfw/util/redis"
	"sync"
	"time"

	"gopkg.in/mgo.v2/bson"
)

const (
	REDISIDS    = "ids"
	REDISKEYS   = "keys"
	INFOID      = "info"
	INFOTIMEOUT = 86400 * 30
)

var (
	Sysconfig                map[string]interface{}
	MQFW                     mongodb.MongodbSim
	extractColl, projectColl string
	lenprojectname           int
	MultiThread              chan bool
	IdLock                   = &sync.Mutex{}
	PncbMayLock              = &sync.Mutex{}

	//三组lock,对应的（PNKey）key为项目名称,值对应的是此项目名称对应的项目id数组
	PNKey, PCKey, PBKey = NewKeyMap(), NewKeyMap(), NewKeyMap()

	currentMegerTime  int64 //合并项目的时间位置，用来清理几个月之前的项目
	currentMegerCount int   //合并项目的计数，用来定时清理
)

func init() {

	util.ReadConfig(&Sysconfig)
	MultiThread = make(chan bool, util.IntAllDef(Sysconfig["thread"], 200))
	lenprojectname = util.IntAllDef(Sysconfig["lenprojectname"], 20) - 1

	redis.InitRedisBySize(Sysconfig["redisaddrs"].(string), util.IntAllDef(Sysconfig["redisPoolSize"], 100), 30, 300)
	MQFW = mongodb.MongodbSim{
		MongodbAddr: Sysconfig["mongodbServers"].(string),
		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
		DbName:      Sysconfig["mongodbName"].(string),
	}
	MQFW.InitPool()
	extractColl = Sysconfig["extractColl"].(string)
	projectColl = Sysconfig["projectColl"].(string)
}

func main() {
	RunFullData()
	time.Sleep(99999 * time.Hour)
}

func NewPushInfo(tmp map[string]interface{}) bson.M {
	return bson.M{
		"comeintime":  tmp["comeintime"],
		"publishtime": tmp["publishtime"],
		"title":       tmp["title"],
		"toptype":     tmp["toptype"],
		"subtype":     tmp["subtype"],
		"infoformat":  tmp["infoformat"],
		"infoid":      util.BsonIdToSId(tmp["_id"]),
		"area":        tmp["area"],
		"city":        tmp["city"],
		"projectname": tmp["projectname"],
		"projectcode": tmp["projectcode"],
		"buyer":       tmp["buyer"],
		"href":        tmp["href"],
	}
}