|
|
@@ -24,7 +24,7 @@ var endOfPunctuationClrear = regexp.MustCompile("[,，.。?？;；]+$")
|
|
|
var keysClrear = regexp.MustCompile("(详见|公告|X|内文|某单位|某部|文件|\\*|暂无|？|\\?)")
|
|
|
|
|
|
//jsondata清理
|
|
|
-func clearJd(jd *map[string]interface{}, e *ExtractTask, spiderCode string) {
|
|
|
+func clearJd(jd *map[string]interface{}, e *ExtractTask, spiderCode,IsClearnMoney string) {
|
|
|
for k, v := range *jd {
|
|
|
if k == "buyer" || k == "winner" || k == "agency" || k == "projectcode" || k == "projectname" {
|
|
|
vstring := util2.ObjToString(v)
|
|
|
@@ -37,7 +37,7 @@ func clearJd(jd *map[string]interface{}, e *ExtractTask, spiderCode string) {
|
|
|
cfn := e.ClearFn[k]
|
|
|
lockclear.Unlock()
|
|
|
if len(cfn) > 0 {
|
|
|
- data := clear.DoClearFn(cfn, []interface{}{vstring, ""}, spiderCode)
|
|
|
+ data := clear.DoClearFn(cfn, []interface{}{vstring, ""}, spiderCode,IsClearnMoney)
|
|
|
lockclear.Lock()
|
|
|
if clear.AsyField[k] != nil || clear.SymField[k] != nil || clear.MesField[k] != nil {
|
|
|
vstring = clear.OtherClean(k, util2.ObjToString(data[0]))
|
|
|
@@ -85,7 +85,7 @@ func JsonDataMergeProcessing(j *util.Job, e *ExtractTask) map[string][]*util.Ext
|
|
|
if len(cfn) == 0 {
|
|
|
continue
|
|
|
}
|
|
|
- newNum := clear.DoClearFn(cfn, []interface{}{(*j.Jsondata)[v], ""}, j.SpiderCode)
|
|
|
+ newNum := clear.DoClearFn(cfn, []interface{}{(*j.Jsondata)[v], ""}, j.SpiderCode,j.IsClearnMoney)
|
|
|
//if util2.IntAll(newNum[0]) != 0 {
|
|
|
extFields := make([]*util.ExtField, 0)
|
|
|
extFields = append(extFields, &util.ExtField{Code: "JsonData_" + v + "_" + fmt.Sprint(jdextweight), Field: v, ExtFrom: "JsonData_" + v + "_" + fmt.Sprint(jdextweight), SourceValue: (*j.Jsondata)[v], Value: newNum[0], Score: 0.1, IsTrue: newNum[len(newNum)-1].(bool)})
|
|
|
@@ -203,7 +203,7 @@ func JsonDataMergeProcessing(j *util.Job, e *ExtractTask) map[string][]*util.Ext
|
|
|
if len(cfn) == 0 {
|
|
|
continue
|
|
|
}
|
|
|
- newNum := clear.DoClearFn(cfn, []interface{}{(*j.Jsondata)[v], ""}, j.SpiderCode)
|
|
|
+ newNum := clear.DoClearFn(cfn, []interface{}{(*j.Jsondata)[v], ""}, j.SpiderCode,j.IsClearnMoney)
|
|
|
//if util2.IntAll(newNum[0]) != 0 {
|
|
|
extFields := make([]*util.ExtField, 0)
|
|
|
if jdextweight > 1 {
|
fengweiqiang