qyxy

qyxy/qyxy_es/config.json

@@ -2,14 +2,14 @@
   "mgodb": "192.168.3.207:27092",
   "dbsize": 12,
   "dbname": "wjh",
-  "dbcoll": "qyxy_oprd",
-  "uname": "SJZY_RWMIX_Other",
-  "upwd": "SJZY@M34I6x7D9ata",
+  "dbcoll": "qyxy_0902",
+  "uname": "",
+  "upwd": "",
   "tasktime": 0,
   "updatetime": 0,
   "elastic": {
     "addr": "http://192.168.3.206:9800",
-    "index": "oprd_qyxy_v1",
+    "index": "qyxy",
     "itype": "qyxy",
     "pool": 12,
     "esfields": [
@@ -54,7 +54,8 @@
       "website_url",
       "partners",
       "employees",
-      "tag_business"
+      "tag_business",
+      "employee_num"
     ]
   }
 }

qyxy/qyxy_es/task.go

@@ -96,6 +96,14 @@ func StdAdd() {
 					} else {
 						esMap["establish_date"] = location.Unix()
 					}
+				} else if field == "cancel_date" {
+					esMap["cancel_date"] = tmp["cancel_date"]
+					location, err := time.ParseInLocation(util.Date_Short_Layout, util.ObjToString(tmp["cancel_date"]), time.Local)
+					if err != nil {
+						util.Debug(err)
+					} else {
+						esMap["cancel_date_unix"] = location.Unix()
+					}
 				} else if field == "lastupdatetime" {
 					esMap["lastupdatetime"] = tmp["update_time_msql"]
 				} else if field == "bid_projectname" {
@@ -196,7 +204,7 @@ func StdAdd() {
 					esMap["company_type_int"] = 22
 				}
 			} else {
-				if company_type == "内资分公司" {
+				if company_type == "内资分公司" || company_type == "内资非法人企业、非公司私营企业及内资非公司企业分支机构" {
 					esMap["company_type_int"] = 12
 				} else if len([]rune(company_name)) >= 4 {
 					esMap["company_type_int"] = 11
@@ -263,6 +271,14 @@ func StdAll() {
 					}
 				} else if field == "lastupdatetime" {
 					esMap["lastupdatetime"] = tmp["update_time_msql"]
+				} else if field == "cancel_date" {
+					esMap["cancel_date"] = tmp["cancel_date"]
+					location, err := time.ParseInLocation(util.Date_Short_Layout, util.ObjToString(tmp["cancel_date"]), time.Local)
+					if err != nil {
+						util.Debug(err)
+					} else {
+						esMap["cancel_date_unix"] = location.Unix()
+					}
 				} else if field == "bid_projectname" {
 					if pname, ok := tmp["bid_projectname"].([]interface{}); ok {
 						p1 := util.ObjArrToStringArr(pname)
@@ -365,7 +381,7 @@ func StdAll() {
 					esMap["company_type_int"] = 22
 				}
 			} else {
-				if company_type == "内资分公司" {
+				if company_type == "内资分公司" || company_type == "内资非法人企业、非公司私营企业及内资非公司企业分支机构" {
 					esMap["company_type_int"] = 12
 				} else if len([]rune(company_name)) >= 4 {
 					esMap["company_type_int"] = 11

qyxy/qyxy_std/init.go

@@ -58,11 +58,11 @@ func init() {
 	//}
 	//MysqlTool.Init()
 
-	Es = &elastic.Elastic{
-		S_esurl: "http://127.0.0.1:9800", //http://172.17.145.170:9800
-		I_size:  10,
-	}
-	Es.InitElasticSize()
+	//Es = &elastic.Elastic{
+	//	S_esurl: "http://127.0.0.1:9800", //http://172.17.145.170:9800
+	//	I_size:  10,
+	//}
+	//Es.InitElasticSize()
 
 	saveSize = 200
 	updatePool = make(chan []map[string]interface{}, 5000)

qyxy/qyxy_std/task.go

@@ -9,6 +9,7 @@ import (
 	"qfw/util"
 	"qfw/util/redis"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -322,6 +323,7 @@ func IncStd(tmp map[string]interface{}) {
 			}
 			year := 0
 			phone, email := "", ""
+			employeeNum := 0
 			for _, v := range reports {
 				if util.ObjToString(v["_operation_type"]) == "insert" {
 					exp := make(map[string]interface{})
@@ -329,6 +331,7 @@ func IncStd(tmp map[string]interface{}) {
 					exp["report_year"] = v["report_year"]
 					exp["zip_code"] = v["zip_code"]
 					exp["employee_no"] = v["employee_no"]
+					exp["member_no"] = v["member_no"]
 					exp["company_phone"] = v["company_phone"]
 					exp["company_email"] = v["company_email"]
 					exp["_id"] = util.IntAll(v["id"])
@@ -337,6 +340,13 @@ func IncStd(tmp map[string]interface{}) {
 						year = util.IntAll(v["report_year"])
 						phone = util.ObjToString(v["company_phone"])
 						email = util.ObjToString(v["company_email"])
+						employeeNo := DealMemberNo(util.ObjToString(v["employee_no"]))
+						memberNo := DealMemberNo(util.ObjToString(v["member_no"]))
+						if employeeNo > 0 {
+							employeeNum = employeeNo
+						} else if memberNo > 0 {
+							employeeNum = memberNo
+						}
 					}
 				} else {
 					eqFlag := true
@@ -347,6 +357,7 @@ func IncStd(tmp map[string]interface{}) {
 							m["report_year"] = v["report_year"]
 							m["zip_code"] = v["zip_code"]
 							m["employee_no"] = v["employee_no"]
+							m["member_no"] = v["member_no"]
 							m["company_phone"] = v["company_phone"]
 							m["company_email"] = v["company_email"]
 							break
@@ -354,20 +365,26 @@ func IncStd(tmp map[string]interface{}) {
 					}
 					if eqFlag {
 						exp := make(map[string]interface{})
-						exp["stock_capital"] = v["stock_capital"]
-						exp["stock_name"] = v["stock_name"]
-						exp["identify_no"] = v["identify_no"]
-						exp["stock_realcapital"] = v["stock_realcapital"]
-						exp["is_history"] = v["is_history"]
-						exp["is_personal"] = v["is_personal"]
-						exp["stock_type"] = v["stock_type"]
-						exp["identify_type"] = v["identify_type"]
+						exp["operator_name"] = v["operator_name"]
+						exp["report_year"] = v["report_year"]
+						exp["zip_code"] = v["zip_code"]
+						exp["employee_no"] = v["employee_no"]
+						exp["member_no"] = v["member_no"]
+						exp["company_phone"] = v["company_phone"]
+						exp["company_email"] = v["company_email"]
 						exp["_id"] = util.IntAll(v["id"])
 						arr = append(arr, exp)
 						if year < util.IntAll(v["report_year"]) {
 							year = util.IntAll(v["report_year"])
 							phone = util.ObjToString(v["company_phone"])
 							email = util.ObjToString(v["company_email"])
+							employeeNo := DealMemberNo(util.ObjToString(v["employee_no"]))
+							memberNo := DealMemberNo(util.ObjToString(v["member_no"]))
+							if employeeNo > 0 {
+								employeeNum = employeeNo
+							} else if memberNo > 0 {
+								employeeNum = memberNo
+							}
 						}
 					}
 				}
@@ -378,6 +395,8 @@ func IncStd(tmp map[string]interface{}) {
 			if year != 0 {
 				save["company_phone"] = phone
 				save["company_email"] = email
+				save["employee_num"] = employeeNum
+
 			}
 		}
 		// website_url
@@ -746,7 +765,7 @@ func partnerFun(tmp map[string]interface{}) {
 
 // annual_report_base	年报信息
 func reportFunMysql(tmp map[string]interface{}) {
-	query := "SELECT report_year, company_phone, company_email, zip_code, employee_no, operator_name FROM annual_report_base WHERE company_id=?"
+	query := "SELECT report_year, company_phone, company_email, zip_code, employee_no, member_no, operator_name FROM annual_report_base WHERE company_id=?"
 	info := MysqlTool.SelectBySql(query, util.ObjToString(tmp["_id"]))
 	if len(*info) > 0 {
 		tmp["annual_reports"] = *info
@@ -766,7 +785,7 @@ func reportFunMysql(tmp map[string]interface{}) {
 	}
 }
 func reportFun(tmp map[string]interface{}) {
-	field := bson.M{"report_year": 1, "company_phone": 1, "company_email": 1, "zip_code": 1, "employee_no": 1, "operator_name": 1}
+	field := bson.M{"report_year": 1, "company_phone": 1, "company_email": 1, "zip_code": 1, "employee_no": 1, "member_no": 1, "operator_name": 1}
 	info, b := MongoTool1.Find("annual_report_base", bson.M{"company_id": tmp["_id"]}, nil, field, false, -1, -1)
 	if b && len(*info) > 0 {
 		tmp["annual_reports"] = *info
@@ -901,3 +920,21 @@ func taskinfo1() {
 	//}
 	//updatePool <- updataInfo
 }
+
+func DealMemberNo(memberNo string) (no int) {
+	memberNo = strings.Trim(memberNo, " ")
+	if memberNo == "" {
+		return no
+	}
+	if memberNo == "企业选择不公示" {
+		return no
+	}
+	if strings.HasSuffix(memberNo, "人") {
+		cou := strings.Split(memberNo, "人")[0]
+		count, _ := strconv.Atoi(cou)
+		return count
+	}
+
+	no, _ = strconv.Atoi(memberNo)
+	return no
+}

qyxy/save_mgo/main.go

@@ -16,23 +16,27 @@ import (
 var (
 	MongoTool *mongodb.MongodbSim
 	rdb       *redis.Client
+
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp   = make(chan bool, 5)
 )
 
 func init() {
-	//MongoTool = &mongodb.MongodbSim{
-	//	MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
-	//	Size:        10,
-	//	DbName:      "mixdata",
-	//	UserName:    "SJZY_RWESBid_Other",
-	//	Password:    "SJZY@O17t8herB3B",
-	//}
-	//MongoTool.InitPool()
 	MongoTool = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.4.85:27080",
+		MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
 		Size:        10,
-		DbName:      "qfw",
+		DbName:      "mixdata",
+		UserName:    "SJZY_RWESBid_Other",
+		Password:    "SJZY@O17t8herB3B",
 	}
 	MongoTool.InitPool()
+	//MongoTool = &mongodb.MongodbSim{
+	//	MongodbAddr: "172.17.4.85:27080",
+	//	Size:        10,
+	//	DbName:      "qfw",
+	//}
+	//MongoTool.InitPool()
+
 }
 
 func initRedis() (err error) {
@@ -41,7 +45,7 @@ func initRedis() (err error) {
 	rdb = redis.NewClient(&redis.Options{
 		Addr:     "127.0.0.1:8379",
 		PoolSize: 200,
-		DB:       5,
+		DB:       2,
 	})
 	_, err = rdb.Ping(ctx).Result()
 	if err != nil {
@@ -52,6 +56,9 @@ func initRedis() (err error) {
 }
 
 func main() {
+
+	go updateMethod()
+
 	err := initRedis()
 	if err != nil {
 		fmt.Println("init redis failed err :", err)
@@ -67,7 +74,6 @@ func main() {
 			fmt.Println("scan keys failed err:", err)
 			return
 		}
-		util.Debug("---keys---", len(keys))
 		n += len(keys)
 		for _, key := range keys {
 			val, err := rdb.Get(ctx, key).Result()
@@ -75,10 +81,17 @@ func main() {
 				fmt.Println("get key values failed err:", err)
 				return
 			}
+			arr := strings.Split(val, "-")
+			if len(arr) > 1 {
+				num, _ := strconv.Atoi(strings.ReplaceAll(arr[1], "\"", ""))
 
-			val, _ = strconv.Unquote(val)
-			arr := strings.Split(key, ",")
-			MongoTool.Save("first_cooperation", map[string]interface{}{"buyer": arr[0], "winner": arr[1], "project_id": val})
+				save := []map[string]interface{}{{
+					"_id": key,
+				},
+					{"$set": map[string]interface{}{"employee_num": num}},
+				}
+				updatePool <- save
+			}
 
 			//val, _ = strconv.Unquote(val) //  处理json字符串带转义符号
 			//maps := make(map[string]interface{})
@@ -125,3 +138,38 @@ func Duplicate(a interface{}) (ret []interface{}) {
 	}
 	return ret
 }
+
+func updateMethod() {
+	arru := make([][]map[string]interface{}, 500)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 500 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoTool.UpdateBulk("qyxy_std", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 500)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoTool.UpdateBulk("qyxy_std", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 500)
+				indexu = 0
+			}
+		}
+	}
+}