预处理调整

src/jy/extract/extractudp.go

@@ -347,7 +347,15 @@ func ExtractByUdpPre(n_sid, n_eid string, f_sid, f_eid string) {
 	log.Debug("待抽取数量：", count1+count2)
 	list1, _ := ext.TaskInfo.FDB.Find("bidding_nomal", query1, nil, Fields, false, -1, -1)
 	list2, _ := ext.TaskInfo.FDB.Find("bidding_file", query2, nil, Fields, false, -1, -1)
-	new_list := append(*list1, *list2...)
+	new_list := []map[string]interface{}{}
+	for _, v := range *list1 {
+		v["fromtable"] = "bidding_nomal"
+		new_list = append(new_list, v)
+	}
+	for _, v := range *list2 {
+		v["fromtable"] = "bidding_file"
+		new_list = append(new_list, v)
+	}
 	now_time := time.Now().Unix()
 	total := 0
 	wg_mgo := sync.WaitGroup{}

src/jy/extract/extraxtmethod.go

@@ -36,7 +36,7 @@ var (
 	ClearTaskList      map[string]*ClearTask                         //清理任务列表
 	saveLimit                                                 = 100  //抽取日志批量保存
 	PageSize                                                  = 5000 //查询分页
-	Fields                                                    = `{"jyfb_data":1,"approvecode":1,"approvenumber":1,"projecttype":1,"approvestatus":1,"total_investment":1,"funds":1,"owner":1,"projectaddr":1,"projectperiod":1,"project_scale":1,"project_person":1,"project_phone":1,"project_startdate":1,"project_completedate":1,"construction_area":1,"floor_area":1,"title":1,"summary":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"bidstatus":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1,"href":1,"infoformat":1,"attach_text":1,"dataging":1,"review_experts":1,"purchasing":1}`
+	Fields                                                    = `{"jyfb_data":1,"approvecode":1,"approvenumber":1,"projecttype":1,"approvestatus":1,"total_investment":1,"funds":1,"owner":1,"projectaddr":1,"projectperiod":1,"project_scale":1,"project_person":1,"project_phone":1,"project_startdate":1,"project_completedate":1,"construction_area":1,"floor_area":1,"title":1,"summary":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"bidstatus":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1,"href":1,"infoformat":1,"attach_text":1,"dataging":1,"review_experts":1,"purchasing":1,"extracttype":1}`
 	BiddingFields                                             = map[string]interface{}{
 		"_id":         1,
 		"title":       1,
@@ -46,6 +46,10 @@ var (
 		"publishtime": 1,
 		"href":        1,
 		"detail":      1,
+		"sensitive":   1,
+		"infoformat":  1,
+		"extracttype": 1,
+		"fromtable":   1,
 	}
 	Fields2     = `{"budget":1,"bidamount":1,"title":1,"projectname":1,"winner":1}`
 	NiJianField = []string{