Merge branch 'master' of http://192.168.3.207:10080/qmx/jy-data-extract

src/jy/cluster/aliecs.go

@@ -41,10 +41,10 @@ func RunInstances(TaskName string, num, hours int) {
 				//[]string{"SecurityGroupId", "sg-bp16x3td2evrejhkshp7"},
 				[]string{"VSwitchId", qu.ObjToString(esconfig["VSwitchId"])},
 				[]string{"InternetMaxBandwidthIn", "50"},
-				[]string{"InternetMaxBandwidthOut", "5"},
+				[]string{"InternetMaxBandwidthOut", "0"},
 				[]string{"InstanceChargeType", "PostPaid"},
 				[]string{"SpotStrategy", "SpotWithPriceLimit"},
-				[]string{"SpotPriceLimit", "2.99"},
+				[]string{"SpotPriceLimit", "1.99"},
 				[]string{"InstanceName", "extract"},
 				[]string{"UniqueSuffix", "true"},
 				[]string{"Password", Password},

src/jy/cluster/distributed.go

@@ -91,7 +91,7 @@ func RangeIdsByDate(table string, escnum int, start time.Time) map[string][]stri
 			count := DB.Count(table, query)
 			start = end
 			log.Printf("i:%d count:%d", i, count)
-			if count >= 500000 || start.Unix() > time.Now().Unix() {
+			if count >= 200000 || start.Unix() > time.Now().Unix() {
 				idsnum = count
 				break
 			}

src/jy/extract/extract.go

@@ -1,6 +1,7 @@
 package extract
 
 import (
+	"encoding/json"
 	"fmt"
 	"jy/clear"
 	db "jy/mongodbutil"
@@ -466,10 +467,16 @@ func getKvByLuaFields(extfrom string, j *ju.Job, in *RegLuaInfo, t map[string][]
 				for k, v := range tkv.Kv {
 					if k == fieldname {
 						if len(tags) > -tkv.KvIndex[fieldname] {
+							ruletext := ""
+							if fieldname == "项目名称" && -tkv.KvIndex[fieldname] == -100 {
+								ruletext = "项目名称"
+							} else {
+								ruletext = tags[-tkv.KvIndex[fieldname]].Key
+							}
 							kvmap[field] = append(kvmap[field], map[string]interface{}{
 								"field":     field,
 								"code":      in.Code,
-								"ruletext":  tags[-tkv.KvIndex[fieldname]].Key,
+								"ruletext":  ruletext,
 								"extfrom":   "table",
 								"value":     v,
 								"type":      "table",
@@ -719,7 +726,7 @@ func AnalysisSaveResult(j *ju.Job, e *ExtractTask) {
 	_id := qu.BsonIdToSId((*doc)["_id"])
 	iscore, _ := ju.Config["fieldscore"].(bool)
 	if iscore { //打分
-		result = ScoreFields(result)
+		result = ScoreFields(j)
 	}
 	//结果排序
 	values := map[string][]*ju.SortObject{}
@@ -765,32 +772,32 @@ func AnalysisSaveResult(j *ju.Job, e *ExtractTask) {
 			}
 		}
 	}
-	//resulttmp := tmp
-	resulttmp, _ := ju.DeepCopy(tmp).(map[string]interface{}) //保存结果
-	resulttmp["result"] = result
 	if len(j.PackageInfo) > 0 { //分包信息
-		resulttmp["package"] = j.PackageInfo
+		tmp["package"] = j.PackageInfo
 	}
 	if len(j.Winnerorder) > 0 { //候选人信息
-		resulttmp["winnerorder"] = j.Winnerorder
+		tmp["winnerorder"] = j.Winnerorder
 	}
 	for k, v := range *doc {
-		if resulttmp[k] == nil { //&& (k != "detail" || k != "contenthtml") {
-			resulttmp[k] = v
+		if k == "detail" || k == "contenthtml" {
+			continue
+		}
+		if tmp[k] == nil {
+			tmp[k] = v
 		}
 	}
 
 	//质量审核
 	if ju.Config["qualityaudit"].(bool) {
-		e.QualityAudit(resulttmp)
+		e.QualityAudit(tmp)
 	}
 	if e.IsExtractCity { //城市抽取
-		b, p, c, d := e.TransmitData(resulttmp, _id) //抽取省份城市
+		b, p, c, d := e.TransmitData(tmp, _id) //抽取省份城市
 		//log.Println("省份---", p, "城市---", c, "区---", d)
-		resulttmp["district"] = d
+		tmp["district"] = d
 		if b {
-			resulttmp["city"] = c
-			resulttmp["area"] = p
+			tmp["city"] = c
+			tmp["area"] = p
 		}
 	}
 
@@ -805,21 +812,24 @@ func AnalysisSaveResult(j *ju.Job, e *ExtractTask) {
 			e.BidArr = append(e.BidArr, tmparr)
 		}
 		if b, ok := ju.Config["saveresult"].(bool); ok && b {
-			id := resulttmp["_id"]
-			delete(resulttmp, "_id")
+			id := tmp["_id"]
+			tmp["result"] = result
+			delete(tmp, "_id")
 			tmparr := []map[string]interface{}{
 				map[string]interface{}{
 					"_id": id,
 				},
-				map[string]interface{}{"$set": resulttmp},
+				map[string]interface{}{"$set": tmp},
 			}
 			e.ResultArr = append(e.ResultArr, tmparr)
 		}
 	} else { //测试结果
+		delete(tmp, "_id")
 		if len(j.BlockPackage) > 0 { //分包详情
-			resulttmp["epackage"] = j.BlockPackage
+			bs, _ := json.Marshal(j.BlockPackage)
+			tmp["epackage"] = string(bs)
 		}
-		b := db.Mgo.Update(e.TaskInfo.TestColl, `{"_id":"`+_id+`"}`, map[string]interface{}{"$set": resulttmp}, true, false)
+		b := db.Mgo.Update(e.TaskInfo.TestColl, `{"_id":"`+_id+`"}`, map[string]interface{}{"$set": tmp}, true, false)
 		if !b {
 			log.Println(e.TaskInfo.TestColl, _id)
 		}

src/jy/extract/score.go

@@ -33,7 +33,29 @@ func init() {
 }
 
 //结果打分
-func ScoreFields(result map[string][]*ju.ExtField) map[string][]*ju.ExtField {
+func ScoreFields(j *ju.Job) map[string][]*ju.ExtField {
+	result := j.Result
+	if len(j.Winnerorder) > 0 { //候选人加入打分集合
+		winner := &ju.ExtField{
+			Field:     "winner",
+			Code:      "",
+			RuleText:  "",
+			Type:      "winnerorder",
+			MatchType: "winnerorder",
+			ExtFrom:   "",
+			Value:     j.Winnerorder[0]["entname"],
+			Score:     0,
+		}
+		winners := result["winner"]
+		if winners != nil {
+			winners = append(winners, winner)
+		} else {
+			winners = []*ju.ExtField{}
+			winners = append(winners, winner)
+		}
+		result["winner"] = winners
+	}
+	//打分
 	for field, tmps := range result {
 		scoreRule := SoreConfig[field]
 		if scoreRule == nil {
@@ -57,6 +79,8 @@ func ScoreFields(result map[string][]*ju.ExtField) map[string][]*ju.ExtField {
 					v.Score += qu.IntAll(extractype["space"])
 				} else if strings.Contains(v.Type, "regexp") {
 					v.Score += qu.IntAll(extractype["regexp"])
+				} else if strings.Contains(v.Type, "winnerorder") {
+					v.Score += qu.IntAll(extractype["winnerorder"])
 				}
 			}
 			//字符型打分

src/jy/pretreated/analytable.go

@@ -77,6 +77,8 @@ var (
 	filter_zbdw_kn = regexp.MustCompile("第[2二3三4四5五]|得分|地址")
 	//且值包含以下字眼
 	filter_zbdw_v = regexp.MustCompile("(公司|集团|研究院|设计院|局|厂|部|站|城|店|市|所|处)$|([^购]中心|办公|用品)")
+	//且值包含以下字眼
+	filter_zbdw_v2 = regexp.MustCompile("(公司|集团|研究院|设计院|局|厂|部|站|城|店|市|所|处)$")
 
 	//Tg = map[string]interface{}{}
 	//一些表格没有表头，是空的，对值是排序的做处理对应 NullTxBid
@@ -1402,10 +1404,68 @@ func (table *Table) FindKV() {
 //获取中标人顺序
 //direct 0默认 1横向 2纵向
 func GetBidOrder(td *TD, direct, n int) (d int, res bool) {
-	if td.Valtype == "BO" {
+	if td.Valtype != "BO" {
+		return
+	}
+	if td.Rowspan > 1 {
+		for i := 0; i < td.Rowspan; i++ {
+			nextcol := 1
+		L1:
+			for {
+				vtd := td.TR.Table.GetTdByRCNo(td.StartRow+i, td.EndCol+nextcol)
+				if vtd == nil {
+					break L1
+				}
+				nextcol += vtd.Colspan
+				if filter_zbdw_v2.MatchString(vtd.Val) {
+					arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
+					if arrbo == nil {
+						arrbo = []map[string]interface{}{}
+						td.TR.Table.SortKV.AddKey(NullTxtBid, arrbo)
+					}
+					a1 := arrbo.([]map[string]interface{})
+					a1 = append(a1, map[string]interface{}{
+						"entname": vtd.Val,
+						"sortstr": td.Val,
+						"sort":    GetBidSort(td.Val, n),
+					})
+					res = true
+					td.TR.Table.SortKV.Map[NullTxtBid] = a1
+				}
+			}
+		}
+	} else if td.Colspan > 1 {
+		for i := 1; i < td.Colspan; i++ {
+			nextcol := 0
+		L2:
+			for {
+				vtd := td.TR.Table.GetTdByRCNo(td.StartRow+i, td.StartCol+nextcol)
+				if vtd == nil || vtd.Colspan >= td.Colspan {
+					break L2
+				}
+				nextcol += vtd.Colspan
+				if filter_zbdw_v2.MatchString(vtd.Val) {
+					arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
+					if arrbo == nil {
+						arrbo = []map[string]interface{}{}
+						td.TR.Table.SortKV.AddKey(NullTxtBid, arrbo)
+					}
+					a1 := arrbo.([]map[string]interface{})
+					a1 = append(a1, map[string]interface{}{
+						"entname": vtd.Val,
+						"sortstr": td.Val,
+						"sort":    GetBidSort(td.Val, n),
+					})
+					res = true
+					td.TR.Table.SortKV.Map[NullTxtBid] = a1
+				}
+			}
+		}
+	} else {
 		rtd := td.TR.Table.GetTdByRCNo(td.StartRow, td.EndCol+1)
 		btd := td.TR.Table.GetTdByRCNo(td.EndRow+1, td.StartCol)
-		if ((rtd != nil && !rtd.BH && rtd.Valtype == "BO") || direct == 1) && btd != nil && filter_zbdw_v.MatchString(btd.Val) {
+		//if ((rtd != nil && !rtd.BH && rtd.Valtype == "BO") || direct == 1) && btd != nil && filter_zbdw_v.MatchString(btd.Val) {
+		if ((rtd != nil && !rtd.BH) || direct == 1) && btd != nil && filter_zbdw_v2.MatchString(btd.Val) {
 			d = 1
 			arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
 			if arrbo == nil {
@@ -1420,7 +1480,8 @@ func GetBidOrder(td *TD, direct, n int) (d int, res bool) {
 			})
 			res = true
 			td.TR.Table.SortKV.Map[NullTxtBid] = a1
-		} else if ((btd != nil && !btd.BH && btd.Valtype == "BO") || direct == 2) && rtd != nil && filter_zbdw_v.MatchString(rtd.Val) {
+			//} else if ((btd != nil && !btd.BH && btd.Valtype == "BO") || direct == 2) && rtd != nil && filter_zbdw_v.MatchString(rtd.Val) {
+		} else if ((btd != nil && !btd.BH) || direct == 2) && rtd != nil && filter_zbdw_v2.MatchString(rtd.Val) {
 			d = 2
 			arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
 			if arrbo == nil {
@@ -1995,7 +2056,7 @@ func (tn *Table) assemblePackage(k1, v1, key string) {
 		k2, w1, v2, _, bf := CommonDataAnaly(k1, "中标情况", "", v1)
 		if bf {
 			for pos, k3 := range k2 {
-				if bp.TableKV.Kv[k3] == "" || w1[pos] > bp.TableKV.KvTag[k3].Weight {
+				if bp.TableKV.Kv != nil && (bp.TableKV.Kv[k3] == "" || w1[pos] > bp.TableKV.KvTag[k3].Weight) {
 					bp.TableKV.Kv[k3] = v2
 					bp.TableKV.KvTag[k3] = &u.Tag{Value: v2, Weight: w1[pos]}
 				}

src/jy/pretreated/tablev2.go

@@ -162,7 +162,7 @@ func NewTD(Goquery *goquery.Selection, tr *TR, table *Table) *TD {
 						bp := tb1.Map[k].(*u.BlockPackage)
 						if v1.TableKV != nil && v1.TableKV.Kv != nil {
 							for k2, v2 := range v1.TableKV.Kv {
-								if bp.TableKV.Kv[k2] == "" {
+								if bp.TableKV.Kv != nil && bp.TableKV.Kv[k2] == "" {
 									bp.TableKV.Kv[k2] = v2
 									bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
 								}

src/jy/pretreated/winnerorder.go

@@ -236,7 +236,7 @@ func (wo *WinnerOrderEntity) clear(typ, v string) interface{} {
 	if typ == "中标单位" && regDivision.MatchString(v) {
 		v = findCompanyReg.FindString(v)
 	}
-	return nil //clear.ClearResult(typ, v)
+	return v //clear.ClearResult(typ, v)
 }
 
 //

src/res/fieldscore.json

@@ -5,7 +5,8 @@
         "table": 5,
         "colon": 3,
         "space": 3,
-        "regexp": 2
+        "regexp": 2,
+        "winnerorder": 3
     },
     "projectname": {
         "type": "string",

src/web/res/js/com.js

@@ -99,4 +99,26 @@ com.pushArry=function(arr1,arr2){
 	  }
 	}
 	return arr1
+}
+
+com.maskShow=function(title){
+	var change=function(){
+		if(title){
+			$("#maskTitle").text(title);
+		}else{
+			$("#maskTitle").text("加载中");
+		}
+		var width=$(window).width();
+		var height=$("body").height();
+		$(".mask>.box").css("left",width/4);
+		$(".mask").removeClass("hide").css("height",height);
+	}
+	$(window).resize(function(){
+		change();
+	});
+	change();
+}
+com.maskHide=function(){
+	$(".mask").addClass("hide");
+	$(window).unbind("resize");
 }

src/web/templates/admin/com_mask.html

@@ -0,0 +1,36 @@
+{{ define "mask" }}
+<style> 
+.mask {
+  width: 100%;
+  top: 0;
+  bottom: 0;
+  left: 0;
+  right: 0;
+  z-index: 99999;
+  position: absolute;
+  background: rgba(0, 0, 0, 0.5);
+}
+.mask > .box {
+  position: absolute;
+  width: 50%;
+  top: 35%;
+}
+</style> 
+<div class="mask hide" >
+	<div class="box box-danger">
+	  <div class="box-header">
+	        <h3 class="box-title" id="maskTitle">加载中</h3>
+		    <div onclick="com.maskHide();" style="cursor:pointer;float:right;font-weight:600;z-index: 9999;position: relative;">X</div>
+	  </div>
+      <div class="box-body">
+        正在加载数据,请稍后......
+      </div>
+      <!-- /.box-body -->
+      <!-- Loading (remove the following to stop the loading)-->
+      <div class="overlay">
+        <i class="fa fa-refresh fa-spin"></i>
+      </div>
+      <!-- end loading -->
+    </div>
+</div>
+{{ end }}

src/web/templates/admin/distribution.html

@@ -50,6 +50,7 @@
 </div>
 	
 <!-- footer -->
+{{template "mask"}}
 {{template "dialog"}}
 {{template "footer"}}
 
@@ -172,7 +173,7 @@ $(function () {
 })
 function del(_id){
 	showConfirm("确定释放实例?", function() {
-		$.ajax({
+        $.ajax({
 			url:"/admin/distribution/deleteInstance",
 			type:"post",
 			data:{"InstanceId":_id},
@@ -188,14 +189,14 @@ function del(_id){
 }
 function upInstanceIds(){
     showConfirm("更新实例?", function() {
-    	$.ajax({
+    	com.maskShow("正在更新实例...");
+		$.ajax({
     		url:"/admin/distribution/upstatus",
     		type:"post",
     		success:function(r){
-    			if(r.rep){				
+    			if(r.rep){			
     				ttable.ajax.reload();
-    			}else{
-    				showTip("更新实例", 1000, function() {});
+                    com.maskHide();
     			}
     		}
     	})
@@ -204,12 +205,14 @@ function upInstanceIds(){
 
 function deploy(){
     showConfirm("部署应用?", function() {
+       com.maskShow("正在部署应用...");
        $.ajax({
     		url:"/admin/distribution/deploy",
     		type:"post",
     		success:function(r){
     			if(r.rep){				
     				ttable.ajax.reload();
+                    com.maskHide();
     			}else{
     				showTip("部署失败", 1000, function() {});
     			}
@@ -220,12 +223,14 @@ function deploy(){
 
 function rangetask(){
     showConfirm("启动抽取?", function() {
-       $.ajax({
+        com.maskShow("正在部署应用...");
+        $.ajax({
     		url:"/admin/distribution/rangetask",
     		type:"post",
     		success:function(r){
     			if(r.rep){				
     				ttable.ajax.reload();
+                    com.maskHide();
     			}else{
     				showTip("启动抽取失败", 1000, function() {});
     			}