data_quality_server/util/fs_client.py

# -*- coding: utf-8 -*-
'''
文件存储服务客户端
author:honbbo
date:2020-2-29
'''
from itertools import islice
import oss2

class FileServeClient(object):
    def __init__(self):
        '''
        抽象出来的文件存储客户端
        目前使用阿里云OSS对象存储服务
        如果更换SeaWeedFs，需要重写此文件
        注意：文件读写，都是以object-name为索引，请保存object-name
        '''
        self.auth = None
        self.bucket = None
        self.do_auth()

    def do_auth(self):
        '''
        身份验证,
        注意：如果auth有时间限制，请在每一批次调用do_auth方法。
        为了提高效率，不必要再每次读写文件时调用。
        '''
        auth = oss2.Auth('LTAI4G5x9aoZx8dDamQ7vfZi', 'Bk98FsbPYXcJe72n1bG3Ssf73acuNh')
        bucket = oss2.Bucket(auth, 'http://oss-cn-beijing.aliyuncs.com', 'topjy')
        self.auth = auth
        self.bucket = bucket

    def list(self):
        '''
        列举文件，仅测试用
        '''
        for b in islice(oss2.ObjectIterator(self.bucket), 10):
            print(b.key)

    def upload_text_file(self, object_name: str, file_content: str) -> (any, any):
        '''
        文件上传
        '''
        result = self.bucket.put_object(object_name, bytes(file_content, encoding='utf-8'))
        status, request_id = result.status, result.request_id
        return status, request_id

    def download_text_content(self, object_name) -> (bool, str):
        '''
        下载文本内容
        '''
        object_stream = self.bucket.get_object(object_name)
        content = object_stream.read()
        if object_stream.client_crc == object_stream.server_crc:
            return True, str(content, encoding='utf-8')
        else:
            return False, ''

    def delete_object(self, object_name: str) -> (any, any):
        '''
        删除内容
        '''
        result = self.bucket.delete_object(object_name)
        status, request_id = result.status, result.request_id
        return status, request_id

    def upload_bytes_file(self, object_name: str, file_content: bytes):
        result = self.bucket.put_object(object_name, file_content)
        status, request_id = result.status, result.request_id
        return status, request_id

    def download_file(self, object_name, save_path):
        object_stream = self.bucket.get_object_to_file(object_name, save_path)
        if object_stream.status == 200:
            return True, save_path

if __name__ == '__main__':
    fsc = FileServeClient()
    #fsc.list()
    #st, id = fsc.upload_text_file('my-object-2', '这是测试的文件内容')
    #print(st, id)
    st, content = fsc.download_text_content('af4477ae-705e-11ed-9389-0242ac120002')
    print(content)
    # with open("text01.txt","w") as f:
    #     f.write(content)
    #st, id = fsc.delete_object('my-object-2')
    #print(st, id)