crawlab_feader/FworkSpider/feapder/utils/aliyun.py

# -*- coding: utf-8 -*-
"""
Created on 2021/3/18 12:39 上午
---------
@summary:  阿里云附件上传
---------
@author: Boris
@email: boris_liu@foxmail.com
"""
import hashlib
import os
import traceback
import oss2
import requests
from feapder import setting
import time

class UploadOSS:
    """阿里云 oss"""

    def __init__(self):
        oss_conf = setting.oss_
        self.file_path: str = ""
        self.file_stream: bytes = b''
        self.__acc_key_id = oss_conf['key_id']
        self.__acc_key_secret = oss_conf['key_secret']
        self.__endpoint = oss_conf['endpoint']
        self.__bucket_name = oss_conf['bucket_name']

    @property
    def fid(self):
        """
        文本摘要值

        @return: 十六进制摘要值
        """
        sha1 = hashlib.sha1()
        sha1.update(str(self.file_stream).encode("utf-8"))
        return sha1.hexdigest()

    @property
    def file_size(self):
        """
        文件的大小,将字节(bytes)转化(kb/M/G单位)

        @return: 文件大小
        """
        try:
            size = os.path.getsize(self.file_path)
        except Exception:
            traceback.print_exc()
        else:
            try:
                _kb = float(size) / 1024
            except:
                return "Error"
            else:
                if _kb >= 1024:
                    _M = _kb / 1024
                    if _M >= 1024:
                        _G = _M / 1024
                        return "{:.1f} G".format(_G)
                    else:
                        return "{:.1f} M".format(_M)
                else:
                    return "{:.1f} kb".format(_kb)

    def get_state(self, attachment,count=0, **kwargs):
        """
        下载附件并上传阿里oss

        @param attachment: 附件
        @return: 附件处理结果
        """
        request_params = {
            'headers': setting.headers,
            'timeout': 20,
            'stream': True,
            **kwargs
        }
        with requests.get(attachment["org_url"], **request_params) as req:
            if req.status_code == 200:
                self.file_stream = req.content
                # img_dir = "file"
                img_dir = f"file/{attachment['channel']}"
                # 文件夹不存在则创建文件夹
                if not os.path.exists(img_dir):
                    os.makedirs(img_dir, mode=0o777, exist_ok=True)
                # 打开目录,放入下载的附件
                filname = hashlib.md5(attachment["filename"].encode("utf-8"))
                filname = filname.hexdigest() #加密1次
                types = attachment["ftype"]
                self.file_path = "{}/{}".format(img_dir, filname+'.'+types)
                with open(self.file_path, 'wb') as f:
                    f.write(self.file_stream)
                # 上传附件
                self.put_oss_from_local()
                file_state = self.file_state(attachment)
                # 删除附件
                os.remove(self.file_path)
                # 返回附件上传处理信息
                return file_state
            else:
                if count<3:
                    self.post_state(attachment,count=count+1, **kwargs)
                else:
                    # attachment["ftype"] = str(attachment["filename"]).split(".")[1]
                    attachment["url"] = 'oss'
                    attachment["fid"] = self.fid + "." + attachment["ftype"]
                    attachment["size"] = '0kb'
                    attachment["false"] = True
                    return attachment
    def post_state(self, attachment,count=0, **kwargs):
        """
        下载附件并上传阿里oss

        @param attachment: 附件
        @return: 附件处理结果
        """
        request_params = {
            'headers': setting.headers,
            'timeout': 20,
            'stream': True,
            **kwargs
        }
        with requests.post(attachment["org_url"], **request_params) as req:
            if req.status_code == 200:
                self.file_stream = req.content
                img_dir = f"file/{attachment['channel']}"
                # 文件夹不存在则创建文件夹
                if not os.path.exists(img_dir):
                    os.makedirs(img_dir, mode=0o777, exist_ok=True)
                # 打开目录,放入下载的附件
                filname = hashlib.md5(attachment["filename"].encode("utf-8"))
                filname = filname.hexdigest()  # 加密1次
                types = attachment["ftype"]
                self.file_path = "{}/{}".format(img_dir, filname + '.' + types)

                with open(self.file_path, 'wb') as f:
                    f.write(self.file_stream)
                # 上传附件
                self.put_oss_from_local()
                file_state = self.file_state(attachment)
                # 删除附件
                # os.remove(self.file_path)
                # 返回附件上传处理信息
                return file_state
            else:
                if count<3:
                    self.post_state(attachment,count=count+1, **kwargs)
                else:
                    attachment["url"] = 'oss'
                    attachment["fid"] = self.fid + "." + attachment["ftype"]
                    attachment["size"] = '0kb'
                    attachment["false"] = True
                    return attachment

    def put_oss_from_local(self):
        """上传一个本地文件到阿里OSS的普通文件"""
        auth = oss2.Auth(self.__acc_key_id, self.__acc_key_secret)
        bucket = oss2.Bucket(auth, self.__endpoint, self.__bucket_name)
        bucket.put_object_from_file(self.fid, self.file_path)

    def file_state(self, attachment):
        """
        文件信息

        @param attachment: 附件
        @return: 附件上传处理信息
        """
        # attachment["ftype"] = str(attachment["filename"]).split(".")[1]
        attachment["url"] = 'oss'
        attachment["fid"] = self.fid + "." + attachment["ftype"]
        attachment["size"] = self.file_size
        return attachment