data_monitor/data_tower/src/read.go

package main

import (
	log "github.com/donnie4w/go-logger/logger"
	"github.com/tealeg/xlsx"
	"strings"
	u "ul"
)

var keyBxArr = []string{"infoid", "rulename", "projectId", "sourceinfoid", "isOptimization", "matchkey", "tagname", "bidtype", "bidway", "toptype", "subtype", "title", "area", "city", "district", "publishtime", "projectcode", "projectname", "buyer", "buyerclass", "tagname2", "budget", "buyerperson", "buyertel", "s_winner", "tagname3", "bidamount", "winnerperson", "winnertel", "agency", "agencyperson", "agencytel", "href", "jybxhref", "docstarttime", "docendtime", "bidstarttime", "bidendtime", "bidopentime", "isCompetitors"}
var indexBxArr = []int{0, -1, 1, 2, 4, -1, 3, -1, -1, 6, 7, 5, 13, 14, 15, 16, 17, -1, 8, 9, 10, 18, 20, 21, 11, 12, 19, 22, 23, 24, 25, 26, 27, -1, -1, -1, -1, -1, -1, -1}
var keyNhArr = []string{"projectId", "sourceinfoid", "tagname", "projectname", "buyer", "s_winner", "area", "city", "district", "projectcode", "zb_infoid", "tagname2", "buyerclass", "buyerperson", "buyertel", "zb_updatetime", "zb_type", "budget", "agency", "agencyperson", "agencytel", "zb_href", "infoid", "tagname3", "winnerperson", "winnertel", "updatetime", "jg_type", "bidamount", "href"}
var indexNhArr = []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29}

// 获取铁塔标讯数据
func readTowerBxInfo() {
	log.Debug(len(keyBxArr), "~", len(indexBxArr))
	ff, _ := xlsx.OpenFile("res/qlm.xlsx")
	total := 0
	for _, sheet := range ff.Sheets {
		for k, row := range sheet.Rows {
			if k == 0 {
				log.Debug("长度~", len(row.Cells))
				continue
			}
			if total%10000 == 0 {
				log.Debug("cur index ", total)
			}
			var arr []string
			for _, cell := range row.Cells {
				arr = append(arr, cell.String())
			}
			splitBxData(arr)
			total++
		}
		log.Debug("is over ...", total)
	}
}

func splitBxData(arr []string) {
	if arr[0] == "" {
		return
	}
	info := map[string]interface{}{}
	for k, v := range indexBxArr {
		if v <= -1 {
			continue
		}
		if arr[v] != "" {
			key := keyBxArr[k]
			if key == "publishtime" {
				nv := strings.ReplaceAll(arr[v], ";@", "")
				nv = strings.ReplaceAll(nv, "/", "-")
				info[key] = nv
			} else {
				info[key] = arr[v]
			}
		}
	}

	u.SaveMgo.Save("zktest_tower_info", info)
}

// 获取铁塔捏合数据
func readTowerNhInfo() {
	log.Debug(len(keyNhArr), "~", len(indexNhArr))
	ff, _ := xlsx.OpenFile("res/qlm_nh.xlsx")
	total := 0
	for _, sheet := range ff.Sheets {
		for k, row := range sheet.Rows {
			total++
			if k == 0 {
				log.Debug("长度~", len(row.Cells))
				continue
			}
			if total%10000 == 0 {
				log.Debug("cur index ", total)
			}
			var arr []string
			for _, cell := range row.Cells {
				arr = append(arr, cell.String())
			}
			splitNhData(arr)
		}
		log.Debug("is over ...", total)
	}
}

func splitNhData(arr []string) {
	info := map[string]interface{}{}
	for k, v := range indexNhArr {
		if v <= -1 {
			continue
		}
		if arr[v] != "" {
			key := keyNhArr[k]
			if key == "zb_updatetime" || key == "jg_updatetime" {
				nv := strings.ReplaceAll(arr[v], ";@", "")
				nv = strings.ReplaceAll(nv, "/", "-")
				info[key] = nv
			} else {
				info[key] = arr[v]
			}
		}
	}
	u.SaveMgo.Save("zktest_tower_nh_info", info)
}