|
@@ -426,6 +426,11 @@ func doIndex(tmp map[string]interface{}, bkey, stype string, ygsiteArr []map[str
|
|
update["dataprocess"] = 7
|
|
update["dataprocess"] = 7
|
|
MgoE.Save("bidding_yg", map[string]interface{}{"id": tid, "source": "is_yg_new"})
|
|
MgoE.Save("bidding_yg", map[string]interface{}{"id": tid, "source": "is_yg_new"})
|
|
log.Println("阳光采购平台新采集爬虫", tid)
|
|
log.Println("阳光采购平台新采集爬虫", tid)
|
|
|
|
+ } else if datatype == "4" && site == sites {
|
|
|
|
+ update["extracttype"] = -1
|
|
|
|
+ update["dataprocess"] = 7
|
|
|
|
+ MgoE.Save("bidding_private", map[string]interface{}{"id": tid, "source": site})
|
|
|
|
+ log.Println("私有采集爬虫", tid)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@@ -470,42 +475,20 @@ func doIndex(tmp map[string]interface{}, bkey, stype string, ygsiteArr []map[str
|
|
if (tmp["tag_topinformation"] != nil && (subtype == "合同" || subtype == "中标" || subtype == "成交" || subtype == "采购意向" || toptype == "招标")) || (tmp["tag_topinformation"] == nil && toptype == "拟建" && strings.Contains(s_topscopeclass, "建筑工程")) {
|
|
if (tmp["tag_topinformation"] != nil && (subtype == "合同" || subtype == "中标" || subtype == "成交" || subtype == "采购意向" || toptype == "招标")) || (tmp["tag_topinformation"] == nil && toptype == "拟建" && strings.Contains(s_topscopeclass, "建筑工程")) {
|
|
update["tag_set"] = getTagSet(tmp, compare)
|
|
update["tag_set"] = getTagSet(tmp, compare)
|
|
}
|
|
}
|
|
|
|
+ if toptype == "拟建" {
|
|
|
|
+ if update["owner"] != nil {
|
|
|
|
+ update["buyer"] = update["owner"]
|
|
|
|
+ }
|
|
|
|
+ }
|
|
if len(update) > 0 {
|
|
if len(update) > 0 {
|
|
log.Println("保存bidding区域 省", update["area"], " 市 ", update["city"], " 区 ", update["district"], " buyerclass ", update["buyerclass"], update["buyer_type"], " id ", tid)
|
|
log.Println("保存bidding区域 省", update["area"], " 市 ", update["city"], " 区 ", update["district"], " buyerclass ", update["buyerclass"], update["buyer_type"], " id ", tid)
|
|
if len(del) > 0 {
|
|
if len(del) > 0 {
|
|
bidUpdate = map[string]interface{}{"$set": update, "$unset": del}
|
|
bidUpdate = map[string]interface{}{"$set": update, "$unset": del}
|
|
- // bidUpdate = append(bidUpdate, []map[string]interface{}{{
|
|
|
|
- // "_id": tmp["_id"],
|
|
|
|
- // },
|
|
|
|
- // {"$set": update, "$unset": del},
|
|
|
|
- // })
|
|
|
|
} else {
|
|
} else {
|
|
bidUpdate = map[string]interface{}{"$set": update}
|
|
bidUpdate = map[string]interface{}{"$set": update}
|
|
}
|
|
}
|
|
- // if len(bidUpdate) >= MgoBulkSize {
|
|
|
|
- // tmps := bidUpdate
|
|
|
|
- // MgoB.UpdateBulk(config.Conf.DB.MongoB.Coll, tmps...)
|
|
|
|
- // bidUpdate = [][]map[string]interface{}{}
|
|
|
|
- // }
|
|
|
|
MgoB.UpdateById(config.Conf.DB.MongoB.Coll, tid, bidUpdate)
|
|
MgoB.UpdateById(config.Conf.DB.MongoB.Coll, tid, bidUpdate)
|
|
}
|
|
}
|
|
- // if n%500 == 0 {
|
|
|
|
- // log.Println("biddingTask current ", n)
|
|
|
|
- // }
|
|
|
|
- // tmp = make(map[string]interface{})
|
|
|
|
- // }
|
|
|
|
- //SaveEsLock.Lock()
|
|
|
|
- // if len(bidUpdate) > 0 {
|
|
|
|
- // tmps := bidUpdate
|
|
|
|
- // MgoB.UpdateBulk(config.Conf.DB.MongoB.Coll, tmps...)
|
|
|
|
- // bidUpdate = [][]map[string]interface{}{}
|
|
|
|
- // }
|
|
|
|
- // if len(extUpdate) > 0 {
|
|
|
|
- // tmps := extUpdate
|
|
|
|
- // MgoE.UpdateBulk(config.Conf.DB.MongoE.Coll, tmps...)
|
|
|
|
- // extUpdate = [][]map[string]interface{}{}
|
|
|
|
- // }
|
|
|
|
- //SaveEsLock.Unlock()
|
|
|
|
return syncNo
|
|
return syncNo
|
|
}
|
|
}
|
|
|
|
|