|
@@ -33,7 +33,7 @@ var (
|
|
ClearTaskList map[string]*ClearTask //清理任务列表
|
|
ClearTaskList map[string]*ClearTask //清理任务列表
|
|
saveLimit = 100 //抽取日志批量保存
|
|
saveLimit = 100 //抽取日志批量保存
|
|
PageSize = 5000 //查询分页
|
|
PageSize = 5000 //查询分页
|
|
- Fields = `{"title":1,"summary":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"bidstatus":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1,"href":1,"infoformat":1,"attach_text":1,"dataging":1,"review_experts":1,"purchasing":1}`
|
|
|
|
|
|
+ Fields = `{"jyfb_data":1,"title":1,"summary":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"bidstatus":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1,"href":1,"infoformat":1,"attach_text":1,"dataging":1,"review_experts":1,"purchasing":1}`
|
|
//Fields = `{"title":1,"summary":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"bidstatus":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1,"href":1,"infoformat":1,"attach_text":1,"dataging":1,"new_attach_text":1,"createtime":1,"currency":1,"id":1,"company_email":1,"buyerclass":1,"tagname":1,"company_phone":1,"appid":1,"industry":1,"projectscope":1,"item":1,"s_subscopeclass":1,"matchkey":1,"jybxhref":1,"legal_person":1,"matchtype":1,"review_experts":1,"purchasing":1}`
|
|
//Fields = `{"title":1,"summary":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"bidstatus":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1,"href":1,"infoformat":1,"attach_text":1,"dataging":1,"new_attach_text":1,"createtime":1,"currency":1,"id":1,"company_email":1,"buyerclass":1,"tagname":1,"company_phone":1,"appid":1,"industry":1,"projectscope":1,"item":1,"s_subscopeclass":1,"matchkey":1,"jybxhref":1,"legal_person":1,"matchtype":1,"review_experts":1,"purchasing":1}`
|
|
Fields2 = `{"budget":1,"bidamount":1,"title":1,"projectname":1,"winner":1}`
|
|
Fields2 = `{"budget":1,"bidamount":1,"title":1,"projectname":1,"winner":1}`
|
|
spidercode = map[string]bool{
|
|
spidercode = map[string]bool{
|
|
@@ -2120,17 +2120,13 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if qu.Float64All(tmp["budget"]) < tmpBudget {
|
|
if qu.Float64All(tmp["budget"]) < tmpBudget {
|
|
- if tmpBudget == qu.Float64All(tmp["budget"])*float64(10000) &&
|
|
|
|
- tmpBudget>=1000000000 && qu.Float64All(tmp["budget"])>0{
|
|
|
|
- tmp["is_dif_ratioMoney"] = true
|
|
|
|
- }else {
|
|
|
|
- fieldSource["budget"] = map[string]interface{}{
|
|
|
|
- "ext_type":"",
|
|
|
|
- "ext_from":"package",
|
|
|
|
- }
|
|
|
|
- tmp["budget"] = tmpBudget
|
|
|
|
|
|
+ fieldSource["budget"] = map[string]interface{}{
|
|
|
|
+ "ext_type":"",
|
|
|
|
+ "ext_from":"package",
|
|
}
|
|
}
|
|
|
|
+ tmp["budget"] = tmpBudget
|
|
}
|
|
}
|
|
|
|
+
|
|
if qu.Float64All(tmp["agencyfee"]) < tmpAgencyfee {
|
|
if qu.Float64All(tmp["agencyfee"]) < tmpAgencyfee {
|
|
fieldSource["agencyfee"] = map[string]interface{}{
|
|
fieldSource["agencyfee"] = map[string]interface{}{
|
|
"ext_type":"",
|
|
"ext_type":"",
|
|
@@ -2138,6 +2134,7 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
|
|
}
|
|
}
|
|
tmp["agencyfee"] = tmpAgencyfee
|
|
tmp["agencyfee"] = tmpAgencyfee
|
|
}
|
|
}
|
|
|
|
+
|
|
if qu.Float64All(tmp["bidamount"]) > 0 && qu.Float64All(tmp["budget"]) > 0 && (qu.Float64All(tmp["bidamount"])/10 > qu.Float64All(tmp["budget"])) {
|
|
if qu.Float64All(tmp["bidamount"]) > 0 && qu.Float64All(tmp["budget"]) > 0 && (qu.Float64All(tmp["bidamount"])/10 > qu.Float64All(tmp["budget"])) {
|
|
fieldSource["bidamount"] = map[string]interface{}{
|
|
fieldSource["bidamount"] = map[string]interface{}{
|
|
"ext_type":"",
|
|
"ext_type":"",
|
|
@@ -2145,17 +2142,31 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
|
|
}
|
|
}
|
|
tmp["bidamount"] = tmpBidamount
|
|
tmp["bidamount"] = tmpBidamount
|
|
} else if qu.Float64All(tmp["bidamount"]) < tmpBidamount {
|
|
} else if qu.Float64All(tmp["bidamount"]) < tmpBidamount {
|
|
- if tmpBidamount == qu.Float64All(tmp["bidamount"])*float64(10000) &&
|
|
|
|
- tmpBidamount>=1000000000 && qu.Float64All(tmp["bidamount"])>0{
|
|
|
|
- tmp["is_dif_ratioMoney"] = true
|
|
|
|
- }else {
|
|
|
|
- fieldSource["bidamount"] = map[string]interface{}{
|
|
|
|
- "ext_type":"",
|
|
|
|
- "ext_from":"package",
|
|
|
|
- }
|
|
|
|
- tmp["bidamount"] = tmpBidamount
|
|
|
|
|
|
+ fieldSource["bidamount"] = map[string]interface{}{
|
|
|
|
+ "ext_type": "",
|
|
|
|
+ "ext_from": "package",
|
|
}
|
|
}
|
|
|
|
+ tmp["bidamount"] = tmpBidamount
|
|
}
|
|
}
|
|
|
|
+
|
|
|
|
+ //if qu.Float64All(tmp["bidamount"]) > 0 && qu.Float64All(tmp["budget"]) > 0 && (qu.Float64All(tmp["bidamount"])/10 > qu.Float64All(tmp["budget"])) {
|
|
|
|
+ // fieldSource["bidamount"] = map[string]interface{}{
|
|
|
|
+ // "ext_type":"",
|
|
|
|
+ // "ext_from":"package",
|
|
|
|
+ // }
|
|
|
|
+ // tmp["bidamount"] = tmpBidamount
|
|
|
|
+ //} else if qu.Float64All(tmp["bidamount"]) < tmpBidamount {
|
|
|
|
+ // if tmpBidamount == qu.Float64All(tmp["bidamount"])*float64(10000) &&
|
|
|
|
+ // tmpBidamount>=1000000000 && qu.Float64All(tmp["bidamount"])>0{
|
|
|
|
+ // tmp["is_dif_ratioMoney"] = true
|
|
|
|
+ // }else {
|
|
|
|
+ // fieldSource["bidamount"] = map[string]interface{}{
|
|
|
|
+ // "ext_type":"",
|
|
|
|
+ // "ext_from":"package",
|
|
|
|
+ // }
|
|
|
|
+ // tmp["bidamount"] = tmpBidamount
|
|
|
|
+ // }
|
|
|
|
+ //}
|
|
} else {
|
|
} else {
|
|
//包数等于1,tmp没有值取包里的值
|
|
//包数等于1,tmp没有值取包里的值
|
|
if tmp["budget"] == nil || tmp["budget"] == 0 {
|
|
if tmp["budget"] == nil || tmp["budget"] == 0 {
|
|
@@ -2412,7 +2423,12 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
|
|
}
|
|
}
|
|
}*/
|
|
}*/
|
|
//检查字段
|
|
//检查字段
|
|
- tmp = checkFields(tmp)
|
|
|
|
|
|
+ tmp = checkFields(tmp,*j.Data)
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+
|
|
if tmp["projectname"] == nil || tmp["projectname"] == "" {
|
|
if tmp["projectname"] == nil || tmp["projectname"] == "" {
|
|
tmp["projectname"] = j.Title
|
|
tmp["projectname"] = j.Title
|
|
}
|
|
}
|
|
@@ -2477,7 +2493,7 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
|
|
}
|
|
}
|
|
|
|
|
|
//检查字段-
|
|
//检查字段-
|
|
-func checkFields(tmp map[string]interface{}) map[string]interface{} {
|
|
|
|
|
|
+func checkFields(tmp map[string]interface{},j_data map[string]interface{}) map[string]interface{} {
|
|
delete(tmp, "contenthtml")
|
|
delete(tmp, "contenthtml")
|
|
delete(tmp, "detail")
|
|
delete(tmp, "detail")
|
|
tmp["repeat"] = 0
|
|
tmp["repeat"] = 0
|
|
@@ -2595,8 +2611,6 @@ func checkFields(tmp map[string]interface{}) map[string]interface{} {
|
|
delete(tmp, "bidway")
|
|
delete(tmp, "bidway")
|
|
}
|
|
}
|
|
|
|
|
|
-
|
|
|
|
-
|
|
|
|
//折扣系数
|
|
//折扣系数
|
|
discount := dealWithDiscountBid(tmp)
|
|
discount := dealWithDiscountBid(tmp)
|
|
if discount >0.0 {
|
|
if discount >0.0 {
|
|
@@ -2607,28 +2621,38 @@ func checkFields(tmp map[string]interface{}) map[string]interface{} {
|
|
delete(tmp, "biddiscount_up")
|
|
delete(tmp, "biddiscount_up")
|
|
delete(tmp, "biddiscount_down")
|
|
delete(tmp, "biddiscount_down")
|
|
|
|
|
|
-
|
|
|
|
-
|
|
|
|
//临时
|
|
//临时
|
|
- docstarttime := qu.Int64All(tmp["docstarttime"])
|
|
|
|
- docendtime := qu.Int64All(tmp["docendtime"])
|
|
|
|
- timeLayout := "2006-01-02 15:04:05"
|
|
|
|
|
|
+ //bidstarttime := qu.Int64All(tmp["bidstarttime"])
|
|
|
|
+ //docendtime := qu.Int64All(tmp["docendtime"])
|
|
|
|
+ //timeLayout := "2006-01-02 15:04:05"
|
|
|
|
|
|
- if docstarttime>0 {
|
|
|
|
- time_1 := time.Unix(docstarttime, 0).Format(timeLayout) //设置时间戳 使用模板格式化为日期字符串
|
|
|
|
- tmp["docstarttime"] = time_1
|
|
|
|
- }
|
|
|
|
- if docendtime>0 {
|
|
|
|
- time_2 := time.Unix(docendtime, 0).Format(timeLayout) //设置时间戳 使用模板格式化为日期字符串
|
|
|
|
- tmp["docendtime"] = time_2
|
|
|
|
- }
|
|
|
|
|
|
+ //if bidstarttime>0 {
|
|
|
|
+ // time_1 := time.Unix(bidstarttime, 0).Format(timeLayout) //设置时间戳 使用模板格式化为日期字符串
|
|
|
|
+ // tmp["bidstarttime"] = time_1
|
|
|
|
+ //}
|
|
|
|
+ //if docendtime>0 {
|
|
|
|
+ // time_2 := time.Unix(docendtime, 0).Format(timeLayout) //设置时间戳 使用模板格式化为日期字符串
|
|
|
|
+ // tmp["docendtime"] = time_2
|
|
|
|
+ //}
|
|
|
|
|
|
jyhref:= fmt.Sprintf(JYUrl, qu.CommonEncodeArticle("content", qu.BsonIdToSId(tmp["_id"])))
|
|
jyhref:= fmt.Sprintf(JYUrl, qu.CommonEncodeArticle("content", qu.BsonIdToSId(tmp["_id"])))
|
|
tmp["jytest_href"] = jyhref
|
|
tmp["jytest_href"] = jyhref
|
|
|
|
|
|
|
|
+
|
|
|
|
+ //检查剑鱼发布-爬虫
|
|
|
|
+ jyfb_data := *qu.ObjToMap(j_data["jyfb_data"])
|
|
|
|
+ if jyfb_data!=nil {
|
|
|
|
+ for k,v := range jyfb_data{
|
|
|
|
+ if k=="area" {
|
|
|
|
+ delete(tmp,"district")
|
|
|
|
+ }
|
|
|
|
+ tmp[k] = v
|
|
|
|
+ }
|
|
|
|
+ }
|
|
return tmp
|
|
return tmp
|
|
}
|
|
}
|
|
-//处理折扣系数
|
|
|
|
|
|
+
|
|
|
|
+//处理折扣系数-
|
|
func dealWithDiscountBid(tmp map[string]interface{}) float64 {
|
|
func dealWithDiscountBid(tmp map[string]interface{}) float64 {
|
|
biddiscount := qu.Float64All(tmp["biddiscount"])
|
|
biddiscount := qu.Float64All(tmp["biddiscount"])
|
|
biddiscount_up := qu.Float64All(tmp["biddiscount_up"])
|
|
biddiscount_up := qu.Float64All(tmp["biddiscount_up"])
|