Przeglądaj źródła

Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

fengweiqiang 5 lat temu
rodzic
commit
a68b166663

+ 7 - 7
fullproject/src_v1/init.go

@@ -298,17 +298,17 @@ type ProjectInfo struct {
 	score         int
 	score         int
 	comStr        string
 	comStr        string
 	resVal, pjVal int
 	resVal, pjVal int
-	InfoFiled	map[string]InfoFiled
+	InfoFiled	map[string]InfoFiled	`json:"infofiled"`		//逻辑处理需要的info字段
 }
 }
 
 
 //存储部分招标信息字段,业务逻辑处理需要
 //存储部分招标信息字段,业务逻辑处理需要
 type InfoFiled struct {
 type InfoFiled struct {
-	budget			float64
-	bidamount		float64
-	contractCode	string
-	projectName		string
-	projectCode		string
-	bidstatus		string
+	Budget			float64				`json:"budget"`
+	Bidamount		float64				`json:"bidamount"`
+	ContractCode	string				`json:"contractcode"`
+	ProjectName		string				`json:"projectname"`
+	ProjectCode		string				`json:"projectcode"`
+	Bidstatus		string				`json:"bidstatus"`
 }
 }
 
 
 //站点信息
 //站点信息

+ 51 - 26
fullproject/src_v1/project.go

@@ -1,6 +1,7 @@
 package main
 package main
 
 
 import (
 import (
+	"encoding/json"
 	"log"
 	"log"
 	//	"log"
 	//	"log"
 	"math"
 	"math"
@@ -534,14 +535,18 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	}
 	}
 
 
 	p1.InfoFiled = make(map[string]InfoFiled)
 	p1.InfoFiled = make(map[string]InfoFiled)
-	p1.InfoFiled[thisinfo.Id] =
-		InfoFiled{
-			budget: thisinfo.Budget,
-			bidamount: thisinfo.Bidamount,
-			projectName: thisinfo.ProjectName,
-			contractCode: thisinfo.ContractCode,
-			projectCode: thisinfo.ProjectCode,
-			bidstatus: bs,
+	infofiled := InfoFiled{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    bs,
+	}
+	p1.InfoFiled[thisinfo.Id] = infofiled
+	res := StructToMap(infofiled)
+	set["infofield"] = map[string]interface{}{
+		thisinfo.Id: res,
 	}
 	}
 
 
 	push := p.PushListInfo(tmp, thisinfo.Id)
 	push := p.PushListInfo(tmp, thisinfo.Id)
@@ -816,6 +821,21 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	CountAmount(pInfo, thisinfo)
 	CountAmount(pInfo, thisinfo)
 	set["budget"] = pInfo.Budget
 	set["budget"] = pInfo.Budget
 	set["bidamount"] = pInfo.Bidamount
 	set["bidamount"] = pInfo.Bidamount
+	infofiled := InfoFiled{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    bs,
+	}
+	pInfo.InfoFiled[thisinfo.Id] = infofiled
+	tmpMap := make(map[string]interface{})
+	for k, v := range pInfo.InfoFiled{
+		tmpMap[k] = StructToMap(v)
+	}
+	tmpMap[thisinfo.Id] = StructToMap(infofiled)
+	set["infofield"] = tmpMap
 
 
 	set["mpn"] = pInfo.MPN
 	set["mpn"] = pInfo.MPN
 	set["mpc"] = pInfo.MPC
 	set["mpc"] = pInfo.MPC
@@ -1022,9 +1042,9 @@ func CountAmount(project *ProjectInfo, info *Info) {
 			flag := false
 			flag := false
 			if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
 			if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
 				for _, res := range project.InfoFiled {
 				for _, res := range project.InfoFiled {
-					if res.projectName == info.ProjectName {
-						if res.budget < info.Budget {
-							project.Budget = project.Budget - res.budget + info.Budget
+					if res.ProjectName == info.ProjectName {
+						if res.Budget < info.Budget {
+							project.Budget = project.Budget - res.Budget + info.Budget
 						}
 						}
 						flag = true
 						flag = true
 						break
 						break
@@ -1078,18 +1098,18 @@ func CountAmount(project *ProjectInfo, info *Info) {
 				flag := false
 				flag := false
 				if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
 				if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
 					for _, res := range project.InfoFiled {
 					for _, res := range project.InfoFiled {
-						if res.bidstatus == "合同" && res.contractCode != "" && info.SubType == "合同" && info.ContractCode != "" {
-							if res.contractCode == info.ContractCode {
-								if res.bidamount < info.Bidamount {
-									project.Bidamount = project.Bidamount - res.bidamount + info.Bidamount
+						if res.Bidstatus == "合同" && res.ContractCode != "" && info.SubType == "合同" && info.ContractCode != "" {
+							if res.ContractCode == info.ContractCode {
+								if res.Bidamount < info.Bidamount {
+									project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
 								}
 								}
 								flag = true
 								flag = true
 								break
 								break
 							}
 							}
 						}else {
 						}else {
-							if res.projectName == info.ProjectName {
-								if res.bidamount < info.Bidamount {
-									project.Bidamount = project.Bidamount - res.bidamount + info.Bidamount
+							if res.ProjectName == info.ProjectName {
+								if res.Bidamount < info.Bidamount {
+									project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
 								}
 								}
 								flag = true
 								flag = true
 								break
 								break
@@ -1104,12 +1124,17 @@ func CountAmount(project *ProjectInfo, info *Info) {
 			}
 			}
 		}
 		}
 	}
 	}
-	project.InfoFiled[info.Id] = InfoFiled{
-		budget:       info.Budget,
-		bidamount:    info.Bidamount,
-		contractCode: info.ContractCode,
-		projectName:  info.ProjectName,
-		projectCode:  info.ProjectCode,
-		bidstatus: 	  info.SubType,
-	}
+}
+
+//结构体转map
+func StructToMap(filed InfoFiled) map[string]interface{} {
+	//先转json
+	result, err := json.Marshal(filed)
+	if err != nil {
+		return nil
+	}
+	//json转map
+	res := make(map[string]interface{})
+	err = json.Unmarshal(result, &res)
+	return res
 }
 }

+ 11 - 18
fullproject/src_v1/task.go

@@ -96,7 +96,6 @@ func NewPT() *ProjectTask {
 
 
 var P_QL *ProjectTask
 var P_QL *ProjectTask
 var sp = make(chan bool, 5)
 var sp = make(chan bool, 5)
-var updatalock sync.RWMutex
 //初始化全量合并对象
 //初始化全量合并对象
 func init() {
 func init() {
 	P_QL = NewPT()
 	P_QL = NewPT()
@@ -114,32 +113,26 @@ func (p *ProjectTask) updateAllQueue() {
 			arru[indexu] = v
 			arru[indexu] = v
 			indexu++
 			indexu++
 			if indexu == p.saveSize {
 			if indexu == p.saveSize {
-				updatalock.Lock()
-				tmparr := arru
-				arru = make([][]map[string]interface{}, p.saveSize)
-				updatalock.Unlock()
 				sp <- true
 				sp <- true
-				go func(tmparr [][]map[string]interface{}) {
+				go func(arru [][]map[string]interface{}) {
 					defer func() {
 					defer func() {
 						<-sp
 						<-sp
 					}()
 					}()
-					MongoTool.UpSertBulk(p.coll, tmparr...)
-				}(tmparr)
+					MongoTool.UpSertBulk(p.coll, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, p.saveSize)
 				indexu = 0
 				indexu = 0
 			}
 			}
 		case <-time.After(1000 * time.Millisecond):
 		case <-time.After(1000 * time.Millisecond):
 			if indexu > 0 {
 			if indexu > 0 {
-				updatalock.Lock()
-				tmparr := arru
-				arru = make([][]map[string]interface{}, p.saveSize)
-				updatalock.Unlock()
 				sp <- true
 				sp <- true
-				go func(tmparr [][]map[string]interface{}) {
+				go func(arru [][]map[string]interface{}) {
 					defer func() {
 					defer func() {
 						<-sp
 						<-sp
 					}()
 					}()
-					MongoTool.UpSertBulk(p.coll, tmparr...)
-				}(tmparr[:indexu])
+					MongoTool.UpSertBulk(p.coll, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, p.saveSize)
 				indexu = 0
 				indexu = 0
 			}
 			}
 		}
 		}
@@ -671,15 +664,15 @@ var Elements = []string{
 */
 */
 func modifyEle(tmpPro map[string]interface{}, tmp map[string]interface{}) (map[string]interface{}, bool) {
 func modifyEle(tmpPro map[string]interface{}, tmp map[string]interface{}) (map[string]interface{}, bool) {
 	modifyMap := map[string]interface{}{}
 	modifyMap := map[string]interface{}{}
-	for k, _ := range tmpPro {
-		for k1, _ := range tmp {
+	for k := range tmpPro {
+		for k1 := range tmp {
 			if k == k1 && tmpPro[k] != tmp[k1] {
 			if k == k1 && tmpPro[k] != tmp[k1] {
 				modifyMap[k] = tmp[k1]
 				modifyMap[k] = tmp[k1]
 				break
 				break
 			}
 			}
 		}
 		}
 	}
 	}
-	for k, _ := range modifyMap {
+	for k := range modifyMap {
 		for _, str := range Elements {
 		for _, str := range Elements {
 			if k == str {
 			if k == str {
 				return modifyMap, true
 				return modifyMap, true

+ 24 - 1
fullproject/src_v1/update.go

@@ -15,6 +15,16 @@ func (p *ProjectTask) modifyUpdate(pInfoId string, index int, info *Info, tmp ma
 	infoList := []interface{}(tmpPro["list"].(primitive.A))
 	infoList := []interface{}(tmpPro["list"].(primitive.A))
 	infoMap := infoList[index].(map[string]interface{})
 	infoMap := infoList[index].(map[string]interface{})
 	infoList[index] = updateValue(infoMap, modifyMap)
 	infoList[index] = updateValue(infoMap, modifyMap)
+	infofiled := InfoFiled{
+		Budget:       info.Budget,
+		Bidamount:    info.Bidamount,
+		ContractCode: info.ContractCode,
+		ProjectName:  info.ProjectName,
+		ProjectCode:  info.ProjectCode,
+		Bidstatus:    info.SubType,
+	}
+	mapTmp, _ := tmpPro["infofiled"].(map[string]interface{})
+	mapTmp[info.Id] = StructToMap(infofiled)
 	set := map[string]interface{}{
 	set := map[string]interface{}{
 		"$set": tmpPro,
 		"$set": tmpPro,
 	}
 	}
@@ -41,6 +51,7 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index int, info *Info, tmp
 				continue
 				continue
 			}
 			}
 			tmp1 := MongoTool.FindById(ExtractColl, infoId)
 			tmp1 := MongoTool.FindById(ExtractColl, infoId)
+			p.fillInPlace(tmp)
 			info1 := ParseInfo(tmp1)
 			info1 := ParseInfo(tmp1)
 			p.CommonMerge(tmp1, info1)
 			p.CommonMerge(tmp1, info1)
 		}
 		}
@@ -59,6 +70,16 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index int, info *Info, tmp
 				infoList := []interface{}(tmpPro["list"].(primitive.A))
 				infoList := []interface{}(tmpPro["list"].(primitive.A))
 				infoMap := infoList[index].(map[string]interface{})
 				infoMap := infoList[index].(map[string]interface{})
 				infoList[index] = updateValue(infoMap, modifyMap)
 				infoList[index] = updateValue(infoMap, modifyMap)
+				infofiled := InfoFiled{
+					Budget:       info.Budget,
+					Bidamount:    info.Bidamount,
+					ContractCode: info.ContractCode,
+					ProjectName:  info.ProjectName,
+					ProjectCode:  info.ProjectCode,
+					Bidstatus:    info.SubType,
+				}
+				mapTmp, _ := tmpPro["infofiled"].(map[string]interface{})
+				mapTmp[info.Id] = StructToMap(infofiled)
 				set := map[string]interface{}{
 				set := map[string]interface{}{
 					"$set": tmpPro,
 					"$set": tmpPro,
 				}
 				}
@@ -67,7 +88,7 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index int, info *Info, tmp
 				//合并到的其它项目里边
 				//合并到的其它项目里边
 				//更新其它的项目
 				//更新其它的项目
 				pro := MongoTool.FindById(ProjectColl, mergePro.Id.Hex())
 				pro := MongoTool.FindById(ProjectColl, mergePro.Id.Hex())
-				backupPro(pro, )
+				backupPro(pro)
 				choose, ex := p.CompareStatus(mergePro, info)
 				choose, ex := p.CompareStatus(mergePro, info)
 				if !choose {
 				if !choose {
 					p.UpdateProject(tmp, info, mergePro, i, comStr, ex)
 					p.UpdateProject(tmp, info, mergePro, i, comStr, ex)
@@ -250,6 +271,8 @@ func (p *ProjectTask) innerMerge(pInfo *ProjectInfo, info *Info, tmp map[string]
 		}
 		}
 		mergeProject(p, pInfo, info, tmpPro, index)
 		mergeProject(p, pInfo, info, tmpPro, index)
 	}
 	}
+	mapTmp, _ := tmpPro["infofiled"].(map[string]interface{})
+	delete(mapTmp, info.Id)
 	set := map[string]interface{}{
 	set := map[string]interface{}{
 		"$set": tmpPro,
 		"$set": tmpPro,
 	}
 	}

+ 4 - 3
src/config.json

@@ -2,7 +2,7 @@
     "port": "9090",
     "port": "9090",
     "mgodb": "192.168.3.207:27092",
     "mgodb": "192.168.3.207:27092",
     "dbsize": 10,
     "dbsize": 10,
-    "dbname": "extract_kf",
+    "dbname": "extract_dev32",
     "redis": "buyer=192.168.3.207:1679,winner=192.168.3.207:1679,agency=192.168.3.207:1679",
     "redis": "buyer=192.168.3.207:1679,winner=192.168.3.207:1679,agency=192.168.3.207:1679",
     "elasticsearch": "http://127.0.0.1:9200",
     "elasticsearch": "http://127.0.0.1:9200",
     "elasticsearch_index": "extract_kf",
     "elasticsearch_index": "extract_kf",
@@ -11,6 +11,7 @@
     "mergetable": "projectset",
     "mergetable": "projectset",
     "mergetablealias": "projectset_v1",
     "mergetablealias": "projectset_v1",
     "saveresult": true,
     "saveresult": true,
+    "fieldsfind": false,
     "qualityaudit": false,
     "qualityaudit": false,
     "saveblock": false,
     "saveblock": false,
     "filelength": 100000,
     "filelength": 100000,
@@ -58,9 +59,9 @@
             }
             }
         ]
         ]
     },
     },
-	"isSaveTag":false,
+    "isSaveTag": false,
     "tomail": "zhangjinkun@topnet.net.cn,chenmingzhu@topnet.net.cn,zhaolongyue@topnet.net.cn",
     "tomail": "zhangjinkun@topnet.net.cn,chenmingzhu@topnet.net.cn,zhaolongyue@topnet.net.cn",
     "api": "http://10.171.112.160:19281/_send/_mail",
     "api": "http://10.171.112.160:19281/_send/_mail",
-    "deleteInstanceTimeHour":1,
+    "deleteInstanceTimeHour": 1,
     "jsondata_extweight": 1
     "jsondata_extweight": 1
 } 
 } 

+ 9 - 5
src/jy/extract/extract.go

@@ -262,7 +262,7 @@ func (e *ExtractTask) PreInfo(doc map[string]interface{}) (j, jf *ju.Job, isSite
 	if isextFile {
 	if isextFile {
 		file2text(&doc) //附件文本堆一起(后期可以考虑,分开处理),方法里修改了doc["detailfile"]结果
 		file2text(&doc) //附件文本堆一起(后期可以考虑,分开处理),方法里修改了doc["detailfile"]结果
 	}
 	}
-	//正文小于50个字,有附件把附件内容加到正文
+	//正文小于200个字,有附件把附件内容加到正文
 	tmpDeatil := detail
 	tmpDeatil := detail
 	tmpdocument, err := goquery.NewDocumentFromReader(strings.NewReader(tmpDeatil))
 	tmpdocument, err := goquery.NewDocumentFromReader(strings.NewReader(tmpDeatil))
 	if err == nil {
 	if err == nil {
@@ -289,6 +289,10 @@ func (e *ExtractTask) PreInfo(doc map[string]interface{}) (j, jf *ju.Job, isSite
 	if subtype == "" {
 	if subtype == "" {
 		subtype = "all"
 		subtype = "all"
 	}
 	}
+	if toptype == "其它" || subtype == "其它" || subtype == "其他" || subtype == "结果变更" {
+		toptype = "all"
+		subtype = "all"
+	}
 	toMap := qu.ObjToMap(doc["jsondata"])
 	toMap := qu.ObjToMap(doc["jsondata"])
 	//log.Debug("toMap", toMap)
 	//log.Debug("toMap", toMap)
 	if (*toMap) != nil {
 	if (*toMap) != nil {
@@ -1554,7 +1558,7 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 		//重新取出清理过后的中标候选人
 		//重新取出清理过后的中标候选人
 		resetWinnerorder(j)
 		resetWinnerorder(j)
 		doc, result, _id := funcAnalysis(j, e)
 		doc, result, _id := funcAnalysis(j, e)
-		if isSaveTag, _ := ju.Config["isSaveTag"].(bool); isSaveTag {
+		if ju.IsSaveTag {
 			go otherNeedSave(j, result, e)
 			go otherNeedSave(j, result, e)
 		}
 		}
 		auxinfo := auxInfo(j)
 		auxinfo := auxInfo(j)
@@ -1614,7 +1618,7 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 			}
 			}
 		}
 		}
 		//质量审核
 		//质量审核
-		if ok, _ := ju.Config["qualityaudit"].(bool); ok {
+		if ju.QualityAudit {
 			e.QualityAudit(tmp)
 			e.QualityAudit(tmp)
 		}
 		}
 		if e.IsExtractCity { //城市抽取
 		if e.IsExtractCity { //城市抽取
@@ -1646,7 +1650,7 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 		blocks := make([]ju.BlockAndTag, 0)
 		blocks := make([]ju.BlockAndTag, 0)
 		for _, v := range j.Block {
 		for _, v := range j.Block {
 			//分包和标签
 			//分包和标签
-			if ju.Config["saveblock"].(bool) {
+			if ju.SaveBlock {
 				xx, _ := json.Marshal(v)
 				xx, _ := json.Marshal(v)
 				tmpblock := new(ju.TmpBlock)
 				tmpblock := new(ju.TmpBlock)
 				err := json.Unmarshal(xx, &tmpblock)
 				err := json.Unmarshal(xx, &tmpblock)
@@ -1712,7 +1716,7 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 				e.BidTotal++
 				e.BidTotal++
 				e.RWMutex.Unlock()
 				e.RWMutex.Unlock()
 			}
 			}
-			if b, ok := ju.Config["saveresult"].(bool); ok && b {
+			if ju.SaveResult {
 				id := tmp["_id"]
 				id := tmp["_id"]
 				tmp["result"] = result
 				tmp["result"] = result
 				tmp["resultf"] = resultf
 				tmp["resultf"] = resultf

+ 29 - 25
src/jy/extract/extractInit.go

@@ -80,7 +80,7 @@ type ExtractTask struct {
 	SiteClearFn   map[string][]string               //站点清理函数
 	SiteClearFn   map[string][]string               //站点清理函数
 	IsExtractCity bool                              //是否开启城市抽取
 	IsExtractCity bool                              //是否开启城市抽取
 	Fields        map[string]int                    //抽取属性组
 	Fields        map[string]int                    //抽取属性组
-	SiteFields        map[string]int                    //抽取站点属性组
+	SiteFields    map[string]int                    //抽取站点属性组
 
 
 	IsFileField bool      //是否开启附件抽取
 	IsFileField bool      //是否开启附件抽取
 	FileFields  *sync.Map //抽取附件属性组
 	FileFields  *sync.Map //抽取附件属性组
@@ -516,9 +516,9 @@ func (e *ExtractTask) InitRuleCore(isSite bool) {
 					continue
 					continue
 				}
 				}
 				field := qu.ObjToString(v["s_field"])
 				field := qu.ObjToString(v["s_field"])
-				if isSite{
+				if isSite {
 					e.SiteFields[field] = 1
 					e.SiteFields[field] = 1
-				}else {
+				} else {
 					e.Fields[field] = 1 //加入抽取属性组备用
 					e.Fields[field] = 1 //加入抽取属性组备用
 				}
 				}
 				rinfo := &RegLuaInfo{
 				rinfo := &RegLuaInfo{
@@ -573,9 +573,9 @@ func (e *ExtractTask) InitRuleCore(isSite bool) {
 					continue
 					continue
 				}
 				}
 				field := qu.ObjToString(v["s_field"])
 				field := qu.ObjToString(v["s_field"])
-				if isSite{
+				if isSite {
 					e.SiteFields[field] = 1
 					e.SiteFields[field] = 1
-				}else {
+				} else {
 					e.Fields[field] = 1 //加入抽取属性组备用
 					e.Fields[field] = 1 //加入抽取属性组备用
 				}
 				}
 				rinfo := &RegLuaInfo{
 				rinfo := &RegLuaInfo{
@@ -1520,33 +1520,37 @@ func getFieldAllAndBlocks(a [][]map[string]interface{}) (arr [][]map[string]inte
 	for _, v := range a {
 	for _, v := range a {
 		_id, _ := v[0]["_id"]
 		_id, _ := v[0]["_id"]
 		if tmp, ok := v[1]["$set"].(map[string]interface{}); ok {
 		if tmp, ok := v[1]["$set"].(map[string]interface{}); ok {
-			if tmp["blocks"] != nil {
-				block := map[string]interface{}{
-					"_id":    _id,
-					"blocks": tmp["blocks"],
+			if ju.SaveBlock {
+				if tmp["blocks"] != nil {
+					block := map[string]interface{}{
+						"_id":    _id,
+						"blocks": tmp["blocks"],
+					}
+					blocks = append(blocks, block)
 				}
 				}
-				blocks = append(blocks, block)
 			}
 			}
 			delete(tmp, "blocks")
 			delete(tmp, "blocks")
-			if f, ok := tmp["fieldall"].(map[string][]map[string]interface{}); ok {
-				fieldall := map[string]interface{}{
-					"_id": _id,
+			if ju.FieldsFind {
+				if f, ok := tmp["fieldall"].(map[string][]map[string]interface{}); ok {
+					fieldall := map[string]interface{}{
+						"_id": _id,
+					}
+					for k, v := range f {
+						fieldall[k] = v
+					}
+					fieldalls = append(fieldalls, fieldall)
 				}
 				}
-				for k, v := range f {
-					fieldall[k] = v
+				if ff, ok := tmp["fieldallf"].(map[string][]map[string]interface{}); ok {
+					fieldallf := map[string]interface{}{
+						"_id": _id,
+					}
+					for k, v := range ff {
+						fieldallf[k] = v
+					}
+					fieldallsf = append(fieldalls, fieldallf)
 				}
 				}
-				fieldalls = append(fieldalls, fieldall)
 			}
 			}
 			delete(tmp, "fieldall")
 			delete(tmp, "fieldall")
-			if ff, ok := tmp["fieldallf"].(map[string][]map[string]interface{}); ok {
-				fieldallf := map[string]interface{}{
-					"_id": _id,
-				}
-				for k, v := range ff {
-					fieldallf[k] = v
-				}
-				fieldallsf = append(fieldalls, fieldallf)
-			}
 			delete(tmp, "fieldallf")
 			delete(tmp, "fieldallf")
 			v[1] = tmp
 			v[1] = tmp
 		}
 		}

+ 1 - 1
src/jy/pretreated/analytable.go

@@ -114,7 +114,7 @@ var (
 	regHz                       = regexp.MustCompile("[\u4e00-\u9fa5]")
 	regHz                       = regexp.MustCompile("[\u4e00-\u9fa5]")
 	winnerOrderAndBidResult     = regexp.MustCompile("((中标)?候选人|(中标|评标)结果)")
 	winnerOrderAndBidResult     = regexp.MustCompile("((中标)?候选人|(中标|评标)结果)")
 	WinnerOrderStr              = regexp.MustCompile(`(集团|公司|学校|中心|家具城|门诊|\[大中小\]学|部|院|局|厂|店|所|队|社|室|厅|段|会|场|行)$`)
 	WinnerOrderStr              = regexp.MustCompile(`(集团|公司|学校|中心|家具城|门诊|\[大中小\]学|部|院|局|厂|店|所|队|社|室|厅|段|会|场|行)$`)
-	DoubtReg                    = regexp.MustCompile("((我|交易)中心|有(疑问|质疑|异议)|(书面)?提出|不再受理|投诉|质疑|监督|交易中心|公示期(限)?)")
+	DoubtReg                    = regexp.MustCompile("((我|交易)中心|有(疑问|质疑|异议|意见)|(书面)?提出|不再受理|投诉|质疑|书面形式|监督|交易中心|公示期(限)?|招标|采购)")
 )
 )
 
 
 //在解析时,判断表格元素是否隐藏
 //在解析时,判断表格元素是否隐藏

+ 9 - 1
src/jy/util/util.go

@@ -32,15 +32,23 @@ var GoodsGet *DFA     //商品
 var BrandGet *DFA     //品牌
 var BrandGet *DFA     //品牌
 var IsBrandGoods bool //是否开启品牌抽取
 var IsBrandGoods bool //是否开启品牌抽取
 
 
+var SaveResult, FieldsFind, IsSaveTag, SaveBlock, QualityAudit bool
+
 func init() {
 func init() {
 	syncint = make(chan bool, 1)
 	syncint = make(chan bool, 1)
 }
 }
 
 
-func InitMgoPool() {
+func UtilInit() {
 	initCap := qu.IntAll(Config["dbsize"])
 	initCap := qu.IntAll(Config["dbsize"])
 	addr := qu.ObjToString(Config["mgodb"])
 	addr := qu.ObjToString(Config["mgodb"])
 	dbname := qu.ObjToString(Config["dbname"])
 	dbname := qu.ObjToString(Config["dbname"])
 	Mgo = MgoFactory(initCap, initCap*3, 120, addr, dbname)
 	Mgo = MgoFactory(initCap, initCap*3, 120, addr, dbname)
+
+	SaveResult, _ = Config["saveresult"].(bool)
+	FieldsFind, _ = Config["fieldsfind"].(bool)
+	IsSaveTag, _ = Config["iscltlog"].(bool)
+	SaveBlock, _ = Config["saveblock"].(bool)
+	QualityAudit, _ = Config["qualityaudit"].(bool)
 }
 }
 
 
 func GetSyncIndex(code string) string {
 func GetSyncIndex(code string) string {

+ 6 - 4
src/main.go

@@ -1,7 +1,6 @@
 package main
 package main
 
 
 import (
 import (
-	log "github.com/donnie4w/go-logger/logger"
 	_ "jy/admin"
 	_ "jy/admin"
 	_ "jy/admin/audit"
 	_ "jy/admin/audit"
 	_ "jy/admin/distribution"
 	_ "jy/admin/distribution"
@@ -13,9 +12,12 @@ import (
 	"net/http"
 	"net/http"
 	_ "net/http/pprof"
 	_ "net/http/pprof"
 	qu "qfw/util"
 	qu "qfw/util"
+
+	log "github.com/donnie4w/go-logger/logger"
 	//"qfw/util/elastic"
 	//"qfw/util/elastic"
-	"gopkg.in/olivere/elastic.v5"
 	"qfw/util/redis"
 	"qfw/util/redis"
+
+	"gopkg.in/olivere/elastic.v5"
 )
 )
 
 
 func init() {
 func init() {
@@ -29,8 +31,8 @@ func init() {
 	//初始化品牌和商品
 	//初始化品牌和商品
 	util.InitBrand()
 	util.InitBrand()
 	util.InitGoods()
 	util.InitGoods()
-	//初始化mongo连接
-	util.InitMgoPool()
+	//初始化util
+	util.UtilInit()
 	//初始化redis
 	//初始化redis
 	redis.InitRedisBySize(qu.ObjToString(util.Config["redis"]), 50, 30, 240)
 	redis.InitRedisBySize(qu.ObjToString(util.Config["redis"]), 50, 30, 240)
 	//初始化elastic连接
 	//初始化elastic连接

+ 1 - 1
src/main_blocktest.go

@@ -51,7 +51,7 @@ func all() {
 }
 }
 func one() {
 func one() {
 	m := mongodbutil.MgoFactory(3, 3, 120, "192.168.3.207:27081", "qfw")
 	m := mongodbutil.MgoFactory(3, 3, 120, "192.168.3.207:27081", "qfw")
-	d, _ := m.FindById("bidding", "5dedb8d7e9d1f601e4b9cf7b", extract.Fields)
+	d, _ := m.FindById("bidding", "59e47b5a40d2d9bbe82296bf", extract.Fields)
 	com(*d)
 	com(*d)
 }
 }
 func com(doc map[string]interface{}) {
 func com(doc map[string]interface{}) {

+ 1 - 1
src/main_test.go

@@ -28,7 +28,7 @@ func Test_han(t *testing.T) {
 func Test_task(t *testing.T) {
 func Test_task(t *testing.T) {
 	Mgo = MgoFactory(1, 3, 120, "192.168.3.207:27092", "extract_kf")
 	Mgo = MgoFactory(1, 3, 120, "192.168.3.207:27092", "extract_kf")
 	//extract.StartExtractTaskId("5b8f804025e29a290415aee1")5c528686698414055c47b115
 	//extract.StartExtractTaskId("5b8f804025e29a290415aee1")5c528686698414055c47b115
-	extract.StartExtractTestTask("5cdd3025698414032c8322b1", "5ded053fe9d1f601e4c9e3ee", "1", "result_mxs", "result_mxs")
+	extract.StartExtractTestTask("5cdd3025698414032c8322b1", "57f9e23961a0721f15428be4", "1", "result_mxs", "result_mxs")
 	//extract.StartExtractTestTask("5c3d75c96984142998eb00e1", "5c2a3d28a5cb26b9b76144dd", "100", "mxs_v3", "mxs_v3")
 	//extract.StartExtractTestTask("5c3d75c96984142998eb00e1", "5c2a3d28a5cb26b9b76144dd", "100", "mxs_v3", "mxs_v3")
 	time.Sleep(5 * time.Second)
 	time.Sleep(5 * time.Second)
 }
 }

+ 7 - 4
versioncomparison/config.json

@@ -1,17 +1,20 @@
 {
 {
-    "premgo": "192.168.3.207:27082",
+    "premgo": "192.168.3.207:27092",
     "predb": "extract_kf",
     "predb": "extract_kf",
-    "prec": "result_v3",
-    "newmgo": "192.168.3.207:27082",
+    "prec": "datainfo_dev3.2",
+    "newmgo": "192.168.3.207:27092",
     "newdb": "extract_kf",
     "newdb": "extract_kf",
-    "newc": "result_data",
+    "newc": "datainfo_dev3.4",
     "fields": [
     "fields": [
         "projectname",
         "projectname",
         "projectcode",
         "projectcode",
+        "contractcode",
         "buyer",
         "buyer",
         "bidamount",
         "bidamount",
         "budget",
         "budget",
         "winner",
         "winner",
+        "winnertel",
+        "winnerperson",
         "agency",
         "agency",
         "buyerperson",
         "buyerperson",
         "buyertel",
         "buyertel",

+ 6 - 7
versioncomparison/main.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"fmt"
 	"jy/mongodbutil"
 	"jy/mongodbutil"
 	"log"
 	"log"
-	"qfw/common/src/qfw/util"
 	qu "qfw/util"
 	qu "qfw/util"
 	"strings"
 	"strings"
 
 
@@ -41,8 +40,8 @@ type Data struct {
 }
 }
 
 
 func init() {
 func init() {
-	flag.StringVar(&Sid, "sid", "5cc87219a5cb26b9b719a6be", "开始id")
-	flag.StringVar(&Eid, "eid", "5d0c876aa5cb26b9b7078df8", "结束id")
+	flag.StringVar(&Sid, "sid", "5e17deb150b5ea296ec939d3", "开始id")
+	flag.StringVar(&Eid, "eid", "5e17e1e685a9271abf08616d", "结束id")
 	flag.Parse()
 	flag.Parse()
 	qu.ReadConfig(&SysConfig)
 	qu.ReadConfig(&SysConfig)
 	Premgo = mongodbutil.MgoFactory(1, 3, 120, qu.ObjToString(SysConfig["premgo"]), qu.ObjToString(SysConfig["predb"]))
 	Premgo = mongodbutil.MgoFactory(1, 3, 120, qu.ObjToString(SysConfig["premgo"]), qu.ObjToString(SysConfig["predb"]))
@@ -83,12 +82,12 @@ func createXlsx() {
 				(*row).Cells[2].SetStyle(style)
 				(*row).Cells[2].SetStyle(style)
 				(*row).Cells[3].SetStyle(style)
 				(*row).Cells[3].SetStyle(style)
 				(*row).Cells[4].SetStyle(style)
 				(*row).Cells[4].SetStyle(style)
-				if Compares[field] == nil{
+				if Compares[field] == nil {
 					(*row).Cells[1].SetInt(0)
 					(*row).Cells[1].SetInt(0)
 					(*row).Cells[2].SetInt(0)
 					(*row).Cells[2].SetInt(0)
 					(*row).Cells[3].SetInt(0)
 					(*row).Cells[3].SetInt(0)
 					(*row).Cells[4].SetInt(0)
 					(*row).Cells[4].SetInt(0)
-				}else {
+				} else {
 					(*row).Cells[1].SetInt(Compares[field].PreExtNum)
 					(*row).Cells[1].SetInt(Compares[field].PreExtNum)
 					(*row).Cells[2].SetInt(Compares[field].NewExtNum)
 					(*row).Cells[2].SetInt(Compares[field].NewExtNum)
 					(*row).Cells[3].SetInt(Compares[field].EqNum)
 					(*row).Cells[3].SetInt(Compares[field].EqNum)
@@ -135,7 +134,7 @@ func getVersionData() {
 			if rd == nil {
 			if rd == nil {
 				rd = map[string]*Data{}
 				rd = map[string]*Data{}
 			}
 			}
-			if v[key] == nil && strings.TrimSpace(util.ObjToString(v[key])) == ""{
+			if v[key] == nil && strings.TrimSpace(qu.ObjToString(v[key])) == "" {
 				continue
 				continue
 			}
 			}
 			rd[qu.BsonIdToSId(v["_id"])] = &Data{
 			rd[qu.BsonIdToSId(v["_id"])] = &Data{
@@ -154,7 +153,7 @@ func getVersionData() {
 			if rd == nil {
 			if rd == nil {
 				rd = map[string]*Data{}
 				rd = map[string]*Data{}
 			}
 			}
-			if v[field] == nil && strings.TrimSpace(util.ObjToString(v[field])) == ""{
+			if v[field] == nil && strings.TrimSpace(qu.ObjToString(v[field])) == "" {
 				continue
 				continue
 			}
 			}
 			_id := qu.BsonIdToSId(v["_id"])
 			_id := qu.BsonIdToSId(v["_id"])

BIN
versioncomparison/template.xlsx