oprd_data/import_execl/main.go

package main

import (
	"github.com/tealeg/xlsx"
	"mongodb"
	"qfw/util"
	"time"
)

var (
	Sysconfig map[string]interface{}
	Mgo       *mongodb.MongodbSim
	DbColl    string
	FilePath  string
	UpdateMap map[string]interface{}
	AddMap    map[string]interface{}
	keyIndex  map[int]interface{}

	updatePool chan []map[string]interface{}
	updateSp   chan bool
	SE         = util.SimpleEncrypt{Key: "topJYBX2019"}
)

type RowData struct {
	row   int
	id    string
	field map[string]interface{}
}

func init() {
	util.ReadConfig(&Sysconfig)
	Mgo = &mongodb.MongodbSim{
		MongodbAddr: Sysconfig["mongodb"].(string),
		Size:        util.IntAllDef(Sysconfig["dbsize"], 5),
		DbName:      Sysconfig["dbname"].(string),
	}
	Mgo.InitPool()
	DbColl = Sysconfig["dbcoll"].(string)

	FilePath = Sysconfig["filepath"].(string)
	UpdateMap = Sysconfig["update_field"].(map[string]interface{})
	AddMap = Sysconfig["add_field"].(map[string]interface{})
	keyIndex = make(map[int]interface{}) // 记录需要保存的字段/表格列索引

	updatePool = make(chan []map[string]interface{}, 5000)
	updateSp = make(chan bool, 5)
}

func main() {
	go updateMethod()

	xlFile, err := xlsx.OpenFile(FilePath)
	if err != nil {
		return
	}
	sheet := xlFile.Sheets[0]
	for rowIndex, row := range sheet.Rows {
		fieldMap := make(map[string]interface{})
		for cellIndex, cell := range row.Cells {
			util.Debug(rowIndex, cellIndex, cell.String())
			if rowIndex == 0 {
				//根据列标题获取列索引
				if cell.String() == "唯一标识" {
					keyIndex[cellIndex] = "id"
				} else {
					for k, v := range AddMap {
						if cell.String() == k {
							keyIndex[cellIndex] = v
						}
					}
					for k, v := range UpdateMap {
						if cell.String() == k {
							keyIndex[cellIndex] = v
						}
					}
				}
			} else {
				util.Debug(keyIndex)
				if field := util.ObjToString(keyIndex[cellIndex]); field != "" {
					if field == "id" {
						fieldMap[field] = SE.DecodeString(cell.String())
					} else if field == "budget" || field == "bidamount" {
						fieldMap[field], _ = cell.Float()
					} else {
						fieldMap[field] = cell.String()
					}
				}
			}
		}
		if rowIndex != 0 && len(fieldMap) > 0 {
			saveInfo := []map[string]interface{}{
				{
					"_id": mongodb.StringTOBsonId(util.ObjToString(fieldMap["id"])),
				},
				{"$set": fieldMap},
			}
			updatePool <- saveInfo
		}
	}

	c := make(chan bool, 1)
	<-c
}

func updateMethod() {
	arru := make([][]map[string]interface{}, 200)
	indexu := 0
	for {
		select {
		case v := <-updatePool:
			arru[indexu] = v
			indexu++
			if indexu == 200 {
				updateSp <- true
				go func(arru [][]map[string]interface{}) {
					defer func() {
						<-updateSp
					}()
					Mgo.UpdateBulk(DbColl, arru...)
				}(arru)
				arru = make([][]map[string]interface{}, 200)
				indexu = 0
			}
		case <-time.After(1000 * time.Millisecond):
			if indexu > 0 {
				updateSp <- true
				go func(arru [][]map[string]interface{}) {
					defer func() {
						<-updateSp
					}()
					Mgo.UpdateBulk(DbColl, arru...)
				}(arru[:indexu])
				arru = make([][]map[string]interface{}, 200)
				indexu = 0
			}
		}
	}
}