重复源适配支持

src/jy/extract/extractcity_way.go

@@ -637,9 +637,9 @@ func LinkSpecialQyxyStep(buyer string, area *string, city *string, district *str
 	if qyxy_arr != nil && len(qyxy_arr) > 0 {
 
 	} else {
-		qyxy_arr, _ = ju.Qyxy_Mgo.Find("qyxy_std", map[string]interface{}{
-			"history_name": buyer,
-		}, nil, nil)
+		//qyxy_arr, _ = ju.Qyxy_Mgo.Find("qyxy_std", map[string]interface{}{
+		//	"history_name": buyer,
+		//}, nil, nil)
 	}
 	if qyxy_arr != nil && len(qyxy_arr) > 0 {
 		for _, v := range qyxy_arr {

src/jy/extract/extractsave.go

@@ -462,6 +462,11 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 			tmp["projectname"] = j.Title
 		}
 		tmp["repeat"] = 0
+		if qu.IntAll((*j.Data)["extracttype"]) == -1 {
+			tmp["repeat"] = 1
+			tmp["repeat_reason"] = "采集源重复"
+		}
+
 		if ju.Ffield {
 			if len(ffield) > 0 {
 				tmp["ffield"] = ffield

src/jy/extract/extraxtmethod.go

@@ -36,7 +36,7 @@ var (
 	ClearTaskList      map[string]*ClearTask                         //清理任务列表
 	saveLimit                                                 = 100  //抽取日志批量保存
 	PageSize                                                  = 5000 //查询分页
-	Fields                                                    = `{"jyfb_data":1,"approvecode":1,"approvenumber":1,"projecttype":1,"approvestatus":1,"total_investment":1,"funds":1,"owner":1,"projectaddr":1,"projectperiod":1,"project_scale":1,"project_person":1,"project_phone":1,"project_startdate":1,"project_completedate":1,"construction_area":1,"floor_area":1,"title":1,"summary":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"bidstatus":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1,"href":1,"infoformat":1,"attach_text":1,"dataging":1,"review_experts":1,"purchasing":1}`
+	Fields                                                    = `{"jyfb_data":1,"extracttype":1,"approvecode":1,"approvenumber":1,"projecttype":1,"approvestatus":1,"total_investment":1,"funds":1,"owner":1,"projectaddr":1,"projectperiod":1,"project_scale":1,"project_person":1,"project_phone":1,"project_startdate":1,"project_completedate":1,"construction_area":1,"floor_area":1,"title":1,"summary":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"bidstatus":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1,"href":1,"infoformat":1,"attach_text":1,"dataging":1,"review_experts":1,"purchasing":1}`
 	BiddingFields                                             = map[string]interface{}{
 		"_id":         1,
 		"title":       1,
@@ -181,6 +181,12 @@ func file2text(doc *map[string]interface{}) {
 						if clearStrReg.MatchString(ffname) {
 							continue
 						}
+						//根据类型过滤不同附件
+						/*
+							{0: '承诺书', 1: '委托书', 2: '文件格式', 3: '中标公告', 4: '招标公告', 5: '招标文件', 6: '更正公告', 7: '合同', 8: '评审',
+							 9: '工程量清单', 10: '验收', 11: '报价', 12: '中小企业声明', 13: '采购内容', 14: '审批', 15: '其他', 16: '其他公告'}
+						*/
+
 						mname[ffname] = true
 						murl[ffname] = attach_url
 						if sortStrReg.MatchString(ffname) {