|
@@ -76,7 +76,7 @@ func NewPT() *ProjectTask {
|
|
|
p := &ProjectTask{
|
|
p := &ProjectTask{
|
|
|
InitMinTime: int64(1325347200),
|
|
InitMinTime: int64(1325347200),
|
|
|
name: "全/增量对象",
|
|
name: "全/增量对象",
|
|
|
- thread: 4,
|
|
|
|
|
|
|
+ thread: Thread,
|
|
|
updatePool: make(chan []map[string]interface{}, 5000),
|
|
updatePool: make(chan []map[string]interface{}, 5000),
|
|
|
//savePool: make(chan map[string]interface{}, 2000),
|
|
//savePool: make(chan map[string]interface{}, 2000),
|
|
|
wg: sync.WaitGroup{},
|
|
wg: sync.WaitGroup{},
|
|
@@ -103,10 +103,12 @@ var sp = make(chan bool, 5)
|
|
|
|
|
|
|
|
//初始化全量合并对象
|
|
//初始化全量合并对象
|
|
|
func init() {
|
|
func init() {
|
|
|
|
|
+ util.Debug("task init...")
|
|
|
P_QL = NewPT()
|
|
P_QL = NewPT()
|
|
|
log.Println(len(P_QL.updatePool))
|
|
log.Println(len(P_QL.updatePool))
|
|
|
go P_QL.updateAllQueue()
|
|
go P_QL.updateAllQueue()
|
|
|
go P_QL.clearMem()
|
|
go P_QL.clearMem()
|
|
|
|
|
+ util.Debug("task init end")
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
func (p *ProjectTask) updateAllQueue() {
|
|
func (p *ProjectTask) updateAllQueue() {
|
|
@@ -585,7 +587,7 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
|
|
|
}()
|
|
}()
|
|
|
fields := map[string]interface{} {"area": 1, "city": 1, "district": 1, "comeintime": 1, "publishtime": 1, "bidopentime": 1, "title": 1, "projectname": 1, "href": 1,
|
|
fields := map[string]interface{} {"area": 1, "city": 1, "district": 1, "comeintime": 1, "publishtime": 1, "bidopentime": 1, "title": 1, "projectname": 1, "href": 1,
|
|
|
"projectcode": 1, "buyerclass": 1, "winner": 1, "s_winner": 1, "buyer": 1, "buyerperson": 1, "buyertel": 1, "infoformat": 1, "toptype": 1, "subtype": 1, "spidercode": 1,
|
|
"projectcode": 1, "buyerclass": 1, "winner": 1, "s_winner": 1, "buyer": 1, "buyerperson": 1, "buyertel": 1, "infoformat": 1, "toptype": 1, "subtype": 1, "spidercode": 1,
|
|
|
- "site": 1, "topscopeclass": 1, "subscopeclass": 1, "bidamount": 1, "budget": 1, "agency": 1, "package": 1}
|
|
|
|
|
|
|
+ "site": 1, "topscopeclass": 1, "subscopeclass": 1, "bidamount": 1, "budget": 1, "agency": 1, "package": 1, "jsondata": 1}
|
|
|
ms := sess.DB(db).C(coll).Find(q).Select(fields).Sort("publishtime")
|
|
ms := sess.DB(db).C(coll).Find(q).Select(fields).Sort("publishtime")
|
|
|
if Sysconfig["hints"] != nil {
|
|
if Sysconfig["hints"] != nil {
|
|
|
ms.Hint(Sysconfig["hints"])
|
|
ms.Hint(Sysconfig["hints"])
|
