yusuan/fileUpload/main.go

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"app.yhyue.com/moapp/jybase/common"
	"github.com/studio-b12/gowebdav"
)

/*
@所有人  71服务上的预算网盘数据，准备5.11号起使用云服务网盘。涉及程序及数据调整内容如下：
1、网盘系统及文件迁移 张金坤
2、预算结构化数据迁移，数据提取程序、上传工具适配调整@张鑫磊
3、解析相关程序适配调整 @刘康佳
@所有人
云盘服务器地址：172.17.162.36
访问地址：http://yunpan.spdata.jianyu360.com/
线下连服务器172.17.162.36，需要申请代理，请提前准备。
*/
//正式环境
var (
	webdav = "http://yunpan.spdata.jianyu360.com/dav"
	user   = "jichunling@topnet.net.cn"
	// secert = "imq6aZfwvWdu9XRsd368uywaalD5Jgwf" //作废
	secert   = "WFDoZThbvx5TsKRQAOhTcgyjpQdNs0C1"
	filename = "./upload.txt"
	// waibao   = "华软"
	// waibao = "机电学院"
	waibao = "张金石"
)

//测试环境
// var (
// 	webdav = "https://cloudreve.jydev.jianyu360.com/dav"
// 	user   = "jichunling@topnet.net.cn"
// 	// secert = "imq6aZfwvWdu9XRsd368uywaalD5Jgwf" //作废
// 	secert   = "WFDoZThbvx5TsKRQAOhTcgyjpQdNs0C1"
// 	filename = "./upload.txt"
// 	// waibao   = "华软"
// 	// waibao = "中电科"
// 	waibao = "机电学院"
// )

func SpitHref(path string) string {
	result := filepath.Dir(path)
	result = strings.Replace(result, `\`, `/`, -1)
	return result
}

var (
	localPath = flag.String("localPath", "", "本地路径")
	cloudPath = flag.String("cloudPath", "", "云盘路径")

	date = flag.String("date", "", "年月日，例如，2024_04")
)

//日志上传
func logUpload(client *gowebdav.Client) {
	c := "/云盘管理/日志/" + waibao
	filepath.WalkDir("./jylog", func(path string, info os.DirEntry, err error) error {
		if err != nil {
			return nil
		}

		if !info.IsDir() {
			l := path
			c := c + "/" + strings.ReplaceAll(path, `\`, `/`)
			UploadYunPan(client, l, c, nil)
		}

		return nil
	})
	return
}

func main() {

	flag.Parse()

	defer common.Catch()

	Init()
	time.Sleep(500 * time.Millisecond)

	txt := `请悉知：

	该文件调用需要两个参数：localPath 、cloudPath
		参数说明： 
		localPath：本地需要上传的路径 ，必须精确到年份  例： D:/jianyu/data/2024年
		cloudPath：云盘需要上传的路径 ，必须精确到年份  例： 2024年 
		
	本地上传路径必须满足相关目录树结构：
	
	年份/省份/城市/区县/类型/文件
	年份/省份/城市/类型/文件
	
	类型为：学校、政府、医院等
	`
	fmt.Println(txt)

	if *localPath == "" && *cloudPath == "" {
		log.Println("参数错误")
		return
	} else if *localPath == "" {
		log.Println("本地路径设置错误")
		return
	} else if *cloudPath == "" {
		log.Println("云盘路径设置错误")
		return
	}

	log.Println("本地目录:", *localPath)
	log.Println("远程目录：", *cloudPath)

	// init
	client := gowebdav.NewAuthClient(webdav, gowebdav.NewAutoAuth(user, secert))
	client.Connect()

	m := LoadingPath(client, webdav, user, secert, "")

	// localPath := `D:/jianyu/data/202/2021_test1`
	// localPath := `D:/jianyu/data/202/2021_test1/2024年`
	// webdavPath := `2021_test`
	lp := *localPath
	cp := *cloudPath

	lp = strings.ReplaceAll(lp, `\`, `/`)
	cp = strings.ReplaceAll(cp, `\`, `/`)

	//文件上传
	uploadDirectory_new(client, lp, cp, m)

	//日志上传
	logUpload(client)

	fmt.Println("结束")

}

//加载目录树
func LoadingPath(client *gowebdav.Client, webdav, user, secert, remote string) map[string]bool {
	m := map[string]bool{}

	// 获取文件内容
	filePath := "/云盘管理/预算目录（千万不要删）.txt"
	content, err := client.Read(filePath)
	if err != nil {
		log.Println("未找到文件路径", err)
		return m
	}

	// 将文件内容转换为字符串
	contentStr := string(content)
	// fmt.Println("File content:")
	arr := strings.Split(contentStr, "\n")
	if len(arr) > 1 {
		year := strings.Split(Replace_hh(arr[0]), "、")
		hrefType := strings.Split(Replace_hh(arr[1]), "、")
		// log.Println(year)
		// log.Println(hrefType)
		for k, v := range arr {
			for _, vv := range year {
				for _, vvv := range hrefType {
					if v == "" || k < 2 {
						continue
					}
					href := Replace_hh(fmt.Sprintf(v, vv)) + Replace_hh(vvv)
					m[href] = true
				}
			}
		}
	}

	log.Println("路径载入完毕")
	return m
}

//上传
func uploadDirectory_new(client *gowebdav.Client, localPath string, webdavPath string, pathMap map[string]bool) error {
	//已上传的
	alreadyUpload, _ := readKeysFromFile(filename)

	fileUploadTxt, _ := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)

	defer fileUploadTxt.Close()

	// failPathMap := map[string]bool{}

	var err error
	// m := map[string]bool{}
	// city := ""
	// 遍历本地目录下的文件和子目录
	err = filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		//目录
		if !info.IsDir() {
			//文件
			// 如果是文件，则构建远程文件路径并上传文件
			relativePath, _ := filepath.Rel(localPath, path)
			// log.Println("~~~~~~~~", webdavPath, "==", relativePath)
			remotePath := filepath.Join(webdavPath, relativePath)
			// log.Println("~~~~~~~~", remotePath)
			remotePath = Replace_path(remotePath)
			path = Replace_path(path)

			//判断是否是规定路径
			spitpath := SpitHref(remotePath)

			if !pathMap[spitpath] {

				log.Println("上传的文件不符合规定路径，请检查：", spitpath)
				fmt.Println("上传的文件不符合规定路径，请检查：", spitpath)

			} else {
				pth := SpitFilePath(remotePath)
				if !alreadyUpload[pth] {
					//
					time.Sleep(time.Millisecond * 200)
					UploadYunPan(client, path, remotePath, fileUploadTxt)

					alreadyUpload[pth] = true
				} else {
					log.Println("已经上传过，过滤", pth)
					fmt.Println("已经上传过，过滤", pth)
				}
			}
		}

		return nil
	})

	return err
}

//切割path
func SpitFilePath(path string) string {
	index := strings.Index(path, "/")
	if index != -1 && index < len(path)-1 {
		return path[index+1:]
	}
	return path
}

//读取已上传的文件
func readKeysFromFile(filename string) (map[string]bool, error) {
	result := map[string]bool{}

	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return result, err
	}

	lines := strings.Split(string(content), "\n")
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line != "" {
			result[line] = true
		}
	}

	return result, nil
}

func UploadYunPan(client *gowebdav.Client, localFilePath, cloudFilePath string, fileUploadTxt *os.File) {

	// log.Println("localFilePath:", localFilePath)
	// log.Println("cloudFilePath:", cloudFilePath)
	localFilePath = strings.ReplaceAll(localFilePath, `\`, `/`)
	cloudFilePath = strings.ReplaceAll(cloudFilePath, `\`, `/`)

	// log.Println("---cloudFilePath", cloudFilePath)
	// return

	// 打开本地文件并读取文件内容
	file, err := os.Open(localFilePath)
	if err != nil {
		log.Println("打开本地文件失败:", err)
		return
	}
	defer file.Close()

	data, err := ioutil.ReadAll(file)
	if err != nil {
		log.Println("读取本地文件内容失败:", err)
		return
	}

	// 将文件上传到云盘
	err = client.Write(cloudFilePath, data, 0644)
	if err != nil {
		time.Sleep(time.Second * 2)
		for i := 1; i <= 5; i++ {
			err = client.Write(cloudFilePath, data, 0644)
			if err == nil {
				break
			}
		}
		if err != nil {
			log.Println("上传文件到云盘失败:", err)
		}
		return
	}

	//监控日志过滤，不打印
	if !strings.Contains(cloudFilePath, "云盘管理") {
		log.Println("文件上传成功！", cloudFilePath)
		fmt.Println("文件上传成功！", cloudFilePath)
	}

	if fileUploadTxt != nil {
		fileUploadTxt.WriteString(SpitFilePath(cloudFilePath) + "\n")
	}
}