|
|
@@ -1,14 +1,20 @@
|
|
|
package main
|
|
|
|
|
|
import (
|
|
|
+ "bytes"
|
|
|
+ "encoding/json"
|
|
|
"fmt"
|
|
|
"github.com/xuri/excelize/v2"
|
|
|
"gorm.io/driver/clickhouse"
|
|
|
"gorm.io/gorm"
|
|
|
"gorm.io/gorm/logger"
|
|
|
+ "io"
|
|
|
+ "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
|
|
|
"log"
|
|
|
+ "net/http"
|
|
|
"net/url"
|
|
|
"sync"
|
|
|
+ "time"
|
|
|
)
|
|
|
|
|
|
// EntMapCode 映射关系表
|
|
|
@@ -34,6 +40,21 @@ func clickhouseData() {
|
|
|
username := "biservice"
|
|
|
password := "Bi_top95215#"
|
|
|
|
|
|
+ Mgo := &mongodb.MongodbSim{
|
|
|
+ MongodbAddr: "172.17.189.140:27080",
|
|
|
+ //MongodbAddr: "127.0.0.1:27083",
|
|
|
+ Size: 10,
|
|
|
+ DbName: "qfw",
|
|
|
+ UserName: "SJZY_RWbid_ES",
|
|
|
+ Password: "SJZY@B4i4D5e6S",
|
|
|
+ //Direct: true,
|
|
|
+ }
|
|
|
+ Mgo.InitPool()
|
|
|
+
|
|
|
+ //host := "192.168.3.207:19000"
|
|
|
+ //username := "jytop"
|
|
|
+ //password := "pwdTopJy123"
|
|
|
+
|
|
|
encodedPassword := url.QueryEscape(password)
|
|
|
dn := fmt.Sprintf("clickhouse://%s:%s@%s/information?dial_timeout=10s&read_timeout=20s", username, encodedPassword, host)
|
|
|
db, err := gorm.Open(clickhouse.Open(dn), &gorm.Config{
|
|
|
@@ -55,47 +76,71 @@ func clickhouseData() {
|
|
|
fmt.Println(err)
|
|
|
}
|
|
|
}()
|
|
|
-
|
|
|
- rows, err := f.GetRows("单位")
|
|
|
+ Sheet := "单位"
|
|
|
+ rows, err := f.GetRows(Sheet)
|
|
|
if err != nil {
|
|
|
fmt.Println(err)
|
|
|
return
|
|
|
}
|
|
|
|
|
|
+ defer func() {
|
|
|
+ if err := f.Close(); err != nil {
|
|
|
+ log.Println("f.close,", err)
|
|
|
+ }
|
|
|
+ }()
|
|
|
var wg sync.WaitGroup
|
|
|
sem := make(chan bool, 15) // 控制同时最多只有10个协程
|
|
|
for i := 1; i < len(rows); i++ {
|
|
|
if len(rows[i]) > 1 && rows[i][1] != "" {
|
|
|
+ insert := make(map[string]interface{})
|
|
|
+ company := rows[i][0]
|
|
|
+ log.Println(i, "----", company)
|
|
|
+ insert["bname"] = rows[i][0]
|
|
|
+ insert["aname"] = rows[i][1]
|
|
|
+ if len(rows[i]) > 2 {
|
|
|
+ insert["type"] = rows[i][2]
|
|
|
+ }
|
|
|
+ Mgo.Save("wcc_henan_0603", insert)
|
|
|
continue
|
|
|
}
|
|
|
-
|
|
|
+ //
|
|
|
+ //
|
|
|
wg.Add(1)
|
|
|
sem <- true
|
|
|
go func(i int) {
|
|
|
defer wg.Done()
|
|
|
company := rows[i][0]
|
|
|
+ insert := make(map[string]interface{})
|
|
|
log.Println(i, "----", company)
|
|
|
+ insert["bname"] = rows[i][0]
|
|
|
exist := EntMapCode{}
|
|
|
db.Where(&EntMapCode{BName: company}).First(&exist)
|
|
|
if exist.AName != "" {
|
|
|
- f.SetCellValue("单位", fmt.Sprintf("%s%d", "B", i+1), exist.AName)
|
|
|
+ insert["aname"] = exist.AName
|
|
|
}
|
|
|
|
|
|
if exist.Code == "0101" {
|
|
|
- f.SetCellValue("单位", fmt.Sprintf("%s%d", "C", i+1), "管辖关系")
|
|
|
+ insert["type"] = "管辖关系"
|
|
|
+ //f.SetCellValue(Sheet, fmt.Sprintf("%s%d", "C", i+1), "管辖关系")
|
|
|
} else if exist.Code == "0102" {
|
|
|
- f.SetCellValue("单位", fmt.Sprintf("%s%d", "C", i+1), "直属关系")
|
|
|
+ insert["type"] = "直属关系"
|
|
|
+ //f.SetCellValue(Sheet, fmt.Sprintf("%s%d", "C", i+1), "直属关系")
|
|
|
} else if exist.Code == "0103" {
|
|
|
- f.SetCellValue("单位", fmt.Sprintf("%s%d", "C", i+1), "组成关系")
|
|
|
+ insert["type"] = "组成关系"
|
|
|
+ //f.SetCellValue(Sheet, fmt.Sprintf("%s%d", "C", i+1), "组成关系")
|
|
|
}
|
|
|
|
|
|
+ Mgo.Save("wcc_henan_0603", insert)
|
|
|
<-sem // 释放一个位置，允许其他协程开始执行
|
|
|
}(i)
|
|
|
}
|
|
|
|
|
|
wg.Wait() // 等待所有协程结束
|
|
|
|
|
|
- f.Save()
|
|
|
+ err = f.Save()
|
|
|
+ if err != nil {
|
|
|
+ log.Println("f save err", err)
|
|
|
+ }
|
|
|
log.Println("结束")
|
|
|
//for i := 1; i < len(rows); i++ {
|
|
|
// company := rows[i][0]
|
|
|
@@ -120,3 +165,112 @@ func clickhouseData() {
|
|
|
//log.Println("结束")
|
|
|
|
|
|
}
|
|
|
+
|
|
|
+// getName 更新表格数据
|
|
|
+func getName() {
|
|
|
+ Mgo := &mongodb.MongodbSim{
|
|
|
+ MongodbAddr: "172.17.189.140:27080",
|
|
|
+ //MongodbAddr: "127.0.0.1:27083",
|
|
|
+ Size: 10,
|
|
|
+ DbName: "qfw",
|
|
|
+ UserName: "SJZY_RWbid_ES",
|
|
|
+ Password: "SJZY@B4i4D5e6S",
|
|
|
+ //Direct: true,
|
|
|
+ }
|
|
|
+ Mgo.InitPool()
|
|
|
+
|
|
|
+ //181
|
|
|
+ Mgo2 := &mongodb.MongodbSim{
|
|
|
+ MongodbAddr: "172.17.4.181:27001",
|
|
|
+ //MongodbAddr: "127.0.0.1:27001",
|
|
|
+ DbName: "mixdata",
|
|
|
+ Size: 10,
|
|
|
+ UserName: "",
|
|
|
+ Password: "",
|
|
|
+ //Direct: true,
|
|
|
+ }
|
|
|
+ Mgo2.InitPool()
|
|
|
+
|
|
|
+ f, err := excelize.OpenFile("./河南单位.xlsx")
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println(err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer func() {
|
|
|
+ if err := f.Close(); err != nil {
|
|
|
+ fmt.Println(err)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+
|
|
|
+ rows, err := f.GetRows("单位")
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println(err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ for i := 1; i < len(rows); i++ {
|
|
|
+ name := rows[i][0]
|
|
|
+ log.Println(i, "----", name)
|
|
|
+ //res, _ := Mgo2.FindOne("special_enterprise", map[string]interface{}{"company_name": name})
|
|
|
+ //if res != nil && len(*res) > 0 {
|
|
|
+ // f.SetCellValue("单位", fmt.Sprintf("%s%d", "E", i+1), "special_enterprise")
|
|
|
+ // f.SetCellValue("单位", fmt.Sprintf("%s%d", "F", i+1), (*res)["company_status"])
|
|
|
+ // f.SetCellValue("单位", fmt.Sprintf("%s%d", "G", i+1), (*res)["company_type"])
|
|
|
+ //} else {
|
|
|
+ // ra, _ := Mgo2.FindOne("special_gov_unit", map[string]interface{}{"company_name": name})
|
|
|
+ // if ra != nil && len(*ra) > 0 {
|
|
|
+ // f.SetCellValue("单位", fmt.Sprintf("%s%d", "E", i+1), "special_gov_unit")
|
|
|
+ // f.SetCellValue("单位", fmt.Sprintf("%s%d", "G", i+1), (*ra)["company_type"])
|
|
|
+ // if util.ObjToString((*ra)["company_status"]) != "" {
|
|
|
+ // f.SetCellValue("单位", fmt.Sprintf("%s%d", "F", i+1), (*ra)["company_status"])
|
|
|
+ // } else {
|
|
|
+ // if util.IntAll((*ra)["use_flag"]) < 5 {
|
|
|
+ // f.SetCellValue("单位", fmt.Sprintf("%s%d", "F", i+1), "正常")
|
|
|
+ // } else {
|
|
|
+ // f.SetCellValue("单位", fmt.Sprintf("%s%d", "F", i+1), "废弃")
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ //}
|
|
|
+ // 更新D 列
|
|
|
+ //rb, _ := Mgo.FindOne("wcc_zhengfujigou", map[string]interface{}{"bname": name})
|
|
|
+ //if rb != nil && len(*rb) > 0 {
|
|
|
+ // f.SetCellValue("单位", fmt.Sprintf("%s%d", "D", i+1), (*rb)["aname"])
|
|
|
+ //}
|
|
|
+ //
|
|
|
+ data := map[string]interface{}{
|
|
|
+ "detail": name,
|
|
|
+ }
|
|
|
+
|
|
|
+ res := getAreaInfo(data)
|
|
|
+ if res != nil {
|
|
|
+ city := res["city"]
|
|
|
+ district := res["district"]
|
|
|
+ f.SetCellValue("单位", fmt.Sprintf("%s%d", "B", i+1), city)
|
|
|
+ f.SetCellValue("单位", fmt.Sprintf("%s%d", "C", i+1), district)
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ f.Save()
|
|
|
+ log.Println("结束")
|
|
|
+}
|
|
|
+
|
|
|
+// getAreaInfo 调用抽取接口，获取省市区
|
|
|
+func getAreaInfo(data map[string]interface{}) map[string]interface{} {
|
|
|
+ info := map[string]interface{}{}
|
|
|
+ client := &http.Client{Timeout: 2 * time.Second}
|
|
|
+ jsonStr, _ := json.Marshal(data)
|
|
|
+ resp, err := client.Post("http://127.0.0.1:9996/service/region", "application/json", bytes.NewBuffer(jsonStr))
|
|
|
+ if err != nil {
|
|
|
+ return info
|
|
|
+ }
|
|
|
+ res, err := io.ReadAll(resp.Body)
|
|
|
+ if err != nil {
|
|
|
+ return info
|
|
|
+ }
|
|
|
+ err = json.Unmarshal(res, &info)
|
|
|
+ if err != nil {
|
|
|
+ return info
|
|
|
+ }
|
|
|
+ return info
|
|
|
+}
|
wcc