spider_util/src/spiderutil/upload.go

//上传文件到网络服务器
package spiderutil

import (
	"bytes"
	"fmt"
	"io"
	"path"
	"qfw/util"
	weedcl "qfw/util/weedcl"
	"regexp"
	"strings"
	"time"

	"github.com/donnie4w/go-logger/logger"
)

var weedclClient *weedcl.Client
var fileTypeReg = regexp.MustCompile("^(bmp|mdb|docx|gif|avi|chm|dbx|jpg|mp4|bat|bmp|psd|eml|rtf|mpg|ini|wpd|pwl|flv|doc|pdf|wmv|mid|mxp|qdf|ps|mp3|wav|sql|rar|torrent|png|bmp|jar|pst|css|xml|rmvb|tif|dwg|zip|mov|mf|properties|js|gz|ram|exe)$")
var commUsedReg = regexp.MustCompile("^(docx|gif|jpg|doc|pdf|rar|png|zip|gz|swf|xlsx|xls)$")
var filterTypeReg = regexp.MustCompile("^(html|htm|xml|json)$")

func InitWeedcl() {
	weedclClient = weedcl.NewClient(Config.FileServer)
}
func Upload(code, fileName, url string, bt []byte) (string, string, string) {
	if bt == nil || len(bt) == 0 {
		logger.Error("下载文件出错！", code, " upload file "+fileName, url)
		return "", "", ""
	}
	b := bytes.NewReader(bt)
	size := util.ConvertFileSize(b.Len())
	name, oldFileType := getNameAndType(fileName, url)
	newFileType := util.GetFileType(bt)
	//禁止上传
	if filterTypeReg.MatchString(newFileType) {
		fileName = name + "." + newFileType
		logger.Error("上传文件出错，该文件类型禁止上传！", code, " upload file "+fileName, url)
		return "", fileName, size
	}
	if oldFileType != "" {
		fileName = name + "." + oldFileType
	} else if newFileType != "" {
		fileName = name + "." + newFileType
	} else {
		fileName = name
	}
	fid, publicurl, err := weedclClient.UploadFromReader(fileName, io.MultiReader(b))
	if err != nil {
		logger.Error("上传文件出错！", code, " upload file "+fileName, url, err)
		return "", fileName, size
	}
	logger.Debug("上传文件成功！", code, url, "http://"+publicurl+"/"+fid, fileName, size)
	return "http://" + publicurl + "/" + fid, fileName, size
}

//返回下载链接，文件名称，文件大小，文件类型，文件fid
func UploadFile(code, fileName, url string, bt []byte) (string, string, string, string, string) {
	if bt == nil || len(bt) < 1024*5 {
		logger.Error("下载文件出错！", code, " upload file "+fileName, url)
		return "", "", "", "", ""
	}
	bs := bytes.NewReader(bt)
	size := util.ConvertFileSize(bs.Len())
	name, oldFileType := getNameAndType(fileName, url)
	var newFileType string
	if path.Ext(fileName) == ".xls" {
		newFileType = "xls"
	} else {
		newFileType = util.GetFileType(bt)
	}
	fileName = name
	if !commUsedReg.MatchString(newFileType) { //
		if !commUsedReg.MatchString(oldFileType) { //都不是常见的附件类型不上传
			logger.Error("上传文件出错，该文件类型禁止上传！", code, " upload file "+fileName, url)
			return "", fileName, size, newFileType, ""
		} else {
			newFileType = oldFileType
			fileName = name + "." + newFileType
		}
	} else {
		fileName = name + "." + newFileType
		if commUsedReg.MatchString(oldFileType) && newFileType != oldFileType {
			logger.Info("附件类型不同：", newFileType, oldFileType, code, url)
		}
	}
	//newFileType字节解析出来的文件类型不是常见类型时，替换成oldFileType
	// if newFileType != oldFileType && !commUsedReg.MatchString(newFileType) && oldFileType != "" {
	// 	newFileType = oldFileType
	// }
	//禁止上传
	// if filterTypeReg.MatchString(newFileType) {
	// 	if oldFileType != "doc" && oldFileType != "docx" {
	// 		fileName = name + "." + newFileType
	// 		logger.Error("上传文件出错，该文件类型禁止上传！", code, " upload file "+fileName, url)
	// 		return "", fileName, size, newFileType, ""
	// 	} else {
	// 		newFileType = oldFileType
	// 	}
	// }
	// if oldFileType != "" {
	// 	fileName = name + "." + oldFileType
	// } else if newFileType != "" {
	// 	fileName = name + "." + newFileType
	// } else {
	// 	fileName = name
	// }
	key := getHashKey(bt) + TypeByExt(fileName)
	b, _ := OssPutObject(key, io.MultiReader(bs))
	if b {
		logger.Debug("上传文件成功！", code, "	", url, "	", key, "	", fileName, size)
		return "oss", fileName, size, newFileType, key
	}
	return "", "", "", "", ""
}

//返回下载链接，文件名称，文件大小，文件类型，文件fid
func UploadFile_back(code, fileName, url string, bt []byte) (string, string, string, string, string) {
	defer util.Catch()
	if bt == nil || len(bt) == 0 {
		logger.Error("下载文件出错！", code, " upload file "+fileName, url)
		return "", "", "", "", ""
	}
	b := bytes.NewReader(bt)
	size := util.ConvertFileSize(b.Len())
	name, oldFileType := getNameAndType(fileName, url)
	var newFileType string
	if path.Ext(fileName) == ".xls" {
		newFileType = "xls"
	} else {
		newFileType = util.GetFileType(bt)
	}
	//禁止上传
	if filterTypeReg.MatchString(newFileType) {
		fileName = name + "." + newFileType
		logger.Error("上传文件出错，该文件类型禁止上传！", code, " upload file "+fileName, url)
		return "", fileName, size, newFileType, ""
	}
	if oldFileType != "" {
		fileName = name + "." + oldFileType
	} else if newFileType != "" {
		fileName = name + "." + newFileType
	} else {
		fileName = name
	}
	fid, publicurl, err := weedclClient.UploadFromReader(fileName, io.MultiReader(b))
	if err != nil {
		logger.Error("上传文件出错！", code, " upload file "+fileName, url, err)
		return "", fileName, size, newFileType, ""
	}
	logger.Debug("上传文件成功！", code, url, "http://"+publicurl+"/"+fid, fileName, size)
	return "http://" + publicurl + "/" + fid, fileName, size, newFileType, fid
}

//获取文件名和文件类型，如果文件名中没有文件类型的话，
//再去url里面找,都没有话的就返回空
func getNameAndType(name, url string) (string, string) {
	//文件名中取
	fArray := strings.Split(name, ".")
	fName, fType := "", ""
	if len(fArray) > 1 {
		fType = fArray[len(fArray)-1]
		fName = strings.TrimRight(name, "."+fType)
	} else {
		fName = name
	}
	if fName != "" && fType != "" {
		return fName, fType
	}
	//url中取
	urlArray := strings.Split(url, "/")
	if len(urlArray) == 0 {
		return fName, fType
	}
	urlFileName := urlArray[len(urlArray)-1]
	if !strings.Contains(urlFileName, ".") {
		return fName, fType
	}
	uArray := strings.Split(urlFileName, ".")
	uName, uType := "", ""
	if len(uArray) > 1 {
		uType = uArray[len(uArray)-1]
		if fileTypeReg.MatchString(uType) {
			uName = strings.TrimRight(urlFileName, "."+uType)
		} else {
			uType = ""
		}
	}
	//
	if fName == "" && uName != "" {
		fName = uName
	} else if fName == "" && uName == "" {
		fName = fmt.Sprint(time.Now().Unix())
	}
	if fType == "" && uType != "" {
		fType = uType
	}
	return fName, fType
}