myself_util/spiderutil/oss.go

// oss
package spiderutil

import (
	"bytes"
	"crypto/sha256"
	"fmt"
	"github.com/aliyun/aliyun-oss-go-sdk/oss"
	"io"
	"io/ioutil"
	cu "jygit.jydev.jianyu360.cn/data_capture/myself_util/commonutil"
	"log"
	"net/http"
)

var (
	ossclient *oss.Client
	//ha        = sha256.New()

	ossEndpoint, ossAccessKeyId, ossAccessKeySecret, ossBucketName string
)

func OssInit(endpoint, accessKeyId, accessKeySecret, bucketName string) {
	ossEndpoint = endpoint
	ossAccessKeyId = accessKeyId
	ossAccessKeySecret = accessKeySecret
	ossBucketName = bucketName
	client, err := oss.New(ossEndpoint, ossAccessKeyId, ossAccessKeySecret)
	if err != nil {
		log.Fatal("Error:", err, ossEndpoint, ossAccessKeyId, ossAccessKeySecret)
	}
	ossclient = client
}

func OssGetObject(objectName string) []byte {
	cu.Catch()
	// 获取存储空间。
	bucket, err := ossclient.Bucket(ossBucketName)
	if err != nil {
		fmt.Println("Error:", err)
		return nil
	}

	// 下载文件到流。
	body, err := bucket.GetObject(objectName)
	if err != nil {
		fmt.Println("Error:", err)
		return nil
	}
	defer body.Close()
	data, err := ioutil.ReadAll(body)
	if err != nil {
		fmt.Println("Error:", err)
		return nil
	}

	return data
}

func OssPutObject(objectName string, fd io.Reader) (bool, error) {
	cu.Catch()
	// 获取存储空间。
	bucket, err := ossclient.Bucket(ossBucketName)
	if err != nil {
		log.Println("Error:", err)
		return false, err
	}
	// 上传文件流。
	err = bucket.PutObject(objectName, fd)
	if err != nil {
		log.Println("Error:", err)
		return false, err
	}
	return true, nil
}

func OssDeleteObject(objectName string) (bool, error) {

	// 获取存储空间。
	bucket, err := ossclient.Bucket(ossBucketName)
	if err != nil {
		log.Println("Error:", err)
		return false, err
	}
	err = bucket.DeleteObject(objectName)
	if err != nil {
		log.Println(err)
		return false, err
	}
	return true, nil
}

func demoDownFile(filename string, objectName string) {
	bs := OssGetObject(objectName)
	if bs != nil {
		ioutil.WriteFile(filename, bs, 0777)
	}
}

func GetHashKey(bs []byte) string {
	cu.Catch()
	ha := sha256.New()
	ha.Write(bs)
	hbs := ha.Sum(nil)
	key := fmt.Sprintf("%x", hbs)
	return key
}

func getbytes(resp *http.Response) []byte {
	cu.Catch()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		return []byte{}
	}
	return body
}

func getbytesBigFile(resp *http.Response) []byte {
	cu.Catch()
	buffer := bytes.NewBuffer(make([]byte, 0, 32*1024))
	_, err := io.Copy(buffer, resp.Body)
	if err != nil {
		log.Println(err)
		return []byte{}
	}
	temp := buffer.Bytes()
	length := len(temp)
	var body []byte
	//are we wasting more than 10% space?
	if cap(temp) > (length + length/10) {
		body = make([]byte, length)
		copy(body, temp)
	} else {
		body = temp
	}
	return body
}
func TypeByExt(path string) string {
	for i := len(path) - 1; i >= 0 && path[i] != '/'; i-- {
		if path[i] == '.' {
			return path[i:]
		}
	}
	return ""
}