更新 std ,添加特殊企业表更新 qyxy_std

qyxy_std_new/config.toml

@@ -1,15 +1,15 @@
 [env]
 #    addr = "172.17.145.163:27083"
-    addr = "127.0.0.1:27017"
+    addr = "127.0.0.1:27083"
     dbname = "mixdata"
     dbsize =10
-    dbsave = "qyxy_std"
-    username = ""
-    password = ""
+    dbsave = "qyxy_std_wcc_0319"
+    username = "SJZY_RWbid_ES"
+    password = "SJZY@B4i4D5e6S"
     lastid = 0
     localport =  ":18889"
     esport  =18890
     path = "" ## 路径默认为空，使用udp 传递的参数
     targetip = "127.0.0.1"
-    autoid = 338653106 ## 上次结束的 id
+#    autoid = 339275348 ## 上次结束的 id
     seoid = 35728211634 ## 上次结束的 seo_id 数字

qyxy_std_new/init.go

@@ -39,9 +39,16 @@ func Init() {
 	}
 
 	//qyxy_std 需要处理的数据表
-	CollArr = []string{"company_base", "company_employee", "company_history_name", "company_partner", "annual_report_base", "annual_report_website"}
-
+	CollArr = []string{"company_base", "company_employee", "company_history_name", "company_partner", "annual_report_base", "annual_report_website",
+		"special_enterprise", "special_foundation", "special_gov_unit", "special_hongkong_company", "special_law_office", "special_social_organ", "special_trade_union"}
+
+	db := DB()
+	defer db.Close()
+	// 打印数据对象
+	badgerDB := &bdb{db}
+	fmt.Println("seoid---", badgerDB.Get("seoid"))
 }
+
 func InitMgo() {
 	MongoTool = &mongodb.MongodbSim{
 		MongodbAddr: GF.Env.Addr,

qyxy_std_new/main.go

@@ -270,6 +270,9 @@ func hookfn(c string, line []byte) {
 
 		case "annual_report_website":
 			dealAnnualReportWebsite(tmp)
+			//处理特使企业信息
+		case "special_enterprise", "special_foundation", "special_gov_unit", "special_hongkong_company", "special_law_office", "special_social_organ", "special_trade_union":
+			dealSpecial(c, tmp)
 		default:
 			fmt.Println("CurrentColl =>", c)
 
@@ -441,6 +444,194 @@ func dealCompanyBase(data map[string]interface{}) {
 
 }
 
+// dealSpecial 处理特殊企业数据
+func dealSpecial(coll string, data map[string]interface{}) {
+	update := make(map[string]interface{})
+	save := make(map[string]interface{})
+
+	companyID := util.ObjToString(data["company_id"])
+	if companyID == "" {
+		return
+	}
+
+	for _, v := range company_base {
+		if data[v] == nil {
+			continue
+		}
+		// company_type 公司类型处理
+		if v == "company_type" {
+			save["company_type_old"] = data[v]
+			if text := util.ObjToString(data["company_type"]); text != "" {
+				if strings.Contains(text, "个体") || strings.Contains(text, "非公司") {
+					save["company_type"] = "个体工商户"
+				} else {
+					text = strings.ReplaceAll(text, "（", "(")
+					text = strings.ReplaceAll(text, "）", ")")
+					if stype := QyStypeMap[text]; stype != "" {
+						save["company_type"] = stype
+					} else {
+						save["company_type"] = "其他"
+					}
+				}
+			}
+		} else if v == "company_status" {
+			save["company_status_old"] = data[v]
+			if text := util.ObjToString(data["company_status"]); text != "" {
+				text = strings.ReplaceAll(text, "（", "(")
+				text = strings.ReplaceAll(text, "）", ")")
+				if status := CompanyStatusMap[text]; status != "" {
+					save["company_status"] = status
+				} else {
+					save["company_status"] = "其他"
+				}
+			}
+		} else if v == "capital" {
+			// capital/currency
+			text := util.ObjToString(data[v])
+			if currency := GetCurrency(text); currency != "" {
+				save["currency"] = currency //币种
+			}
+			capital := ObjToMoney(text)
+			capital = capital / 10000
+			if capital != 0 {
+				save[v] = capital
+			}
+		} else if v == "use_flag" {
+			save[v] = util.IntAll(data[v])
+		} else {
+			save[v] = data[v]
+		}
+	}
+
+	// mysql create_time/update_time
+	save["create_time_msql"] = data["create_time"]
+	save["update_time_msql"] = data["update_time"]
+	save["_id"] = data["company_id"]
+	save["createtime"] = time.Now().Unix()
+	save["updatetime"] = time.Now().Unix()
+
+	// company_area/company_city/company_district
+	pshort := util.ObjToString(data["province_short"])
+	save["company_area"] = province_map[pshort]
+
+	//company_city,company_district
+	for i, field := range AreaFiled {
+		if data[field] == nil {
+			continue
+		}
+		if code := fmt.Sprint(data[field]); code != "" {
+			if i == 0 && len(code) >= 8 { //credit_no企业信用代码
+				code = code[2:8]
+			} else if i == 1 && len(code) >= 6 { //company_code注册号
+				code = code[:6]
+			}
+			if city := AddressMap[code]; city != nil { //未作废中取
+				if city.Province != "" && city.Province == util.ObjToString(save["company_area"]) {
+					if city.City != "" {
+						save["company_city"] = city.City //市
+					}
+					if city.District != "" {
+						save["company_district"] = city.District //县
+					}
+					break
+				}
+			} else { //作废中取
+				if city := AddressOldMap[code]; city != nil {
+					if city.Province != "" && city.Province == util.ObjToString(save["company_area"]) {
+						if city.City != "" {
+							save["company_city"] = city.City //市
+						}
+						if city.District != "" {
+							save["company_district"] = city.District //县
+						}
+					}
+					break
+				}
+			}
+		}
+	}
+
+	// search_type
+	if t := util.ObjToString(save["company_type"]); t != "" {
+		if t != "个体工商户" && t != "其他" {
+			t1 := util.ObjToString(save["company_type_old"])
+			name := util.ObjToString(save["company_name"])
+			if strings.Contains(t1, "有限合伙") {
+				save["search_type"] = "有限合伙"
+			} else if strings.Contains(t1, "合伙") {
+				save["search_type"] = "普通合伙"
+			} else if strings.Contains(name, "股份") ||
+				(strings.Contains(t1, "上市") && !strings.Contains(t1, "非上市")) {
+				save["search_type"] = "股份有限公司"
+			} else {
+				save["search_type"] = "有限责任公司"
+			}
+		}
+	}
+
+	// company_shortname
+	if m := getStName(util.ObjToString(save["company_name"])); m != "" {
+		save["company_shortname"] = m
+	}
+
+	// bid_unittype
+	flag := false
+	for _, v := range WordsArr {
+		if strings.Contains(util.ObjToString(save["business_scope"]), v) {
+			flag = true
+			break
+		}
+	}
+	if flag {
+		if save["bid_unittype"] != nil {
+			save["bid_unittype"] = append(util.ObjArrToStringArr(save["bid_unittype"].([]interface{})), "厂商")
+		} else {
+			save["bid_unittype"] = []string{"厂商"}
+		}
+	}
+
+	oldInfo, _ := MongoTool.FindOne(GF.Env.Dbsave, map[string]interface{}{"_id": companyID})
+	if len(*oldInfo) > 0 && util.Int64All((*oldInfo)["autoid"]) > 0 {
+		save["autoid"] = (*oldInfo)["autoid"]
+	} else {
+		id := util.Int64All(data["id"])
+		/**
+		special_enterprise,起始ID 从2000000000 开始+data["id"]
+		special_foundation,起始ID 从3000000000 开始+data["id"]
+		special_gov_unit,起始ID 从4000000000 开始+data["id"]
+		special_hongkong_company,起始ID 从5000000000 开始+data["id"]
+		special_law_office,起始ID 从6000000000 开始+data["id"]
+		special_social_organ,起始ID 从7000000000 开始+data["id"]
+		special_trade_union,起始ID 从8000000000 开始+data["id"]
+		*/
+		startID := 0
+		if coll == "special_enterprise" {
+			startID = 2000000000
+		} else if coll == "special_foundation" {
+			startID = 3000000000
+		} else if coll == "special_gov_unit" {
+			startID = 4000000000
+		} else if coll == "special_hongkong_company" {
+			startID = 5000000000
+		} else if coll == "special_law_office" {
+			startID = 6000000000
+		} else if coll == "special_social_organ" {
+			startID = 7000000000
+		} else if coll == "special_trade_union" {
+			startID = 8000000000
+		}
+		autoid := startID + int(id)
+		save["autoid"] = autoid
+	}
+
+	update["$set"] = save
+	updataInfo := []map[string]interface{}{
+		{"_id": save["_id"]},
+		update,
+	}
+	MongoTool.UpSertBulk(GF.Env.Dbsave, updataInfo)
+}
+
 // dealCompanyEmployee company_employee
 func dealCompanyEmployee(data map[string]interface{}) {
 	save := make(map[string]interface{})

qyxy_std_new/task.go

@@ -20,10 +20,10 @@ var (
 	AreaFiled = []string{"credit_no", "company_code"}
 	WordsArr  = []string{"研发", "研制", "开发", "生产", "制造", "制作", "加工", "种植"}
 
-	seg        jiebago.Segmenter
-	regPre, _  = regexp.Compile(`^(.+[省|市|区|县|州])?(.+)`)
-	seed       = 188
-	startSeoId int64
+	seg       jiebago.Segmenter
+	regPre, _ = regexp.Compile(`^(.+[省|市|区|县|州])?(.+)`)
+	seed      = 188
+	//startSeoId int64
 )
 
 // 企业基本信息
@@ -89,20 +89,8 @@ func updateStd() {
 	// 创建数据库操作对象
 	badgerDB := &bdb{db}
 	var id int64
-	// 获取 autoid
-	auotidOld := badgerDB.Get("autoid")
-	if auotidOld == "" {
-		id = int64(GF.Env.Autoid) //起始 autoid
-		badgerDB.Set("autoid", strconv.Itoa(GF.Env.Autoid))
-	} else {
-		aotoid, _ := strconv.Atoi(auotidOld)
-		id = int64(aotoid)
-	}
+	var startSeoId int64
 
-	log.Println("autoid:", id)
-	if id == 0 {
-		log.Fatalln("autoid 为 0,请检查配置")
-	}
 	//获取 seo_id
 	seoIdOld := badgerDB.Get("seoid")
 	if seoIdOld == "" {
@@ -112,7 +100,7 @@ func updateStd() {
 		seoid, _ := strconv.Atoi(seoIdOld)
 		startSeoId = int64(seoid)
 	}
-	log.Println("seoid", startSeoId)
+	log.Println("seoid--------", startSeoId)
 	if startSeoId == 0 {
 		log.Fatalln("seoid 为 0,请检查配置")
 	}
@@ -126,8 +114,8 @@ func updateStd() {
 
 	ctx, _ := context.WithTimeout(context.Background(), 99999*time.Hour)
 	coll := sess.M.C.Database("mixdata").Collection("qyxy_std")
-	find := options.Find().SetBatchSize(200).SetSort(bson.D{bson.E{"autoid", 1}}).SetProjection(bson.M{"_id": 1, "autoid": 1, "establish_date": 1, "create_time_msql": 1, "nseo_id": 1})
-	cur, err := coll.Find(ctx, bson.M{"autoid": bson.M{"$gt": id}}, find)
+	find := options.Find().SetBatchSize(200).SetSort(bson.D{bson.E{"updatetime", 1}}).SetProjection(bson.M{"_id": 1, "autoid": 1, "establish_date": 1, "create_time_msql": 1, "nseo_id": 1})
+	cur, err := coll.Find(ctx, bson.M{"updatetime": bson.M{"$gt": startTime}}, find)
 
 	if err != nil {
 		log.Println("mgo find err", err.Error())
@@ -140,7 +128,7 @@ func updateStd() {
 			cur.Decode(&tmp)
 			id = util.Int64All(tmp["autoid"])
 			//存在就不处理
-			if _, ok := tmp["nseo_id"]; ok {
+			if seoid, ok := tmp["nseo_id"]; ok && seoid != nil {
 				continue
 			}