current position:Home>django -- minio_ S3 file storage service

django -- minio_ S3 file storage service

2022-01-30 01:32:04 Jin Fengjun

Little knowledge , Great challenge ! This article is participating in 「 A programmer must have a little knowledge 」 Creative activities

This article also participates in  「 Digging force Star Program 」    , Win a creative gift bag , Challenge creation incentive fund

minio_s3 Official documents :docs.minio.org.cn/docs/master…

  • Install related configuration packages
pip install  minio
 Copy code 
  • Add basic configuration to yaml file (filename:minio.yaml)
base:
    POINT_URL: xxx.xxx.xxx.xxx
    ACCESS_KEY:  Your own key
    SECRET_KEY:  Your own SECRET_KEY
    REGION:  Your own REGION
    SCHEME: http
 Copy code 
  • Get configuration code ( Only part of the main code is given )
from utils.security.aes import aes_decrypt

_stream = open('minio.yaml', 'r')

class BaseMinioConfig(DBConfig):
    point_url = _data['base']['POINT_URL']
    access_key = aes_decrypt(_data['base']['ACCESS_KEY'])
    secret_key = aes_decrypt(_data['base']['SECRET_KEY'])
    region = _data['base']['REGION']
    scheme = _data['base']['SCHEME']
    
minio_conf = {
    EnvEnum.base.value: MiniobaseConfig,
}
 Copy code 
  • To configure settings file
from config.minio import minio_conf


# MinioS3 To configure 
MINIO_CONF = minio_conf['base']
PUBLIC_BUCKET_NAME = f"demo_base_public_bucket".replace('_', '-')
BUCKET_NAME = f"demo_base_bucket"
MINIO_ACCESS_KEY_ID = MINIO_CONF.access_key
MINIO_SECRET_ACCESS_KEY = MINIO_CONF.secret_key
MINIO_ENDPOINT_URL = MINIO_CONF.point_url
MINIO_REGION = MINIO_CONF.region
MINIO_SCHEME = MINIO_CONF.scheme

 Copy code 
  • minio_s3 Call the relevant code

base.py

import uuid

def make_s3_key_with_part(prefix: str, suffix: str, dir: str = '/') -> str:
    """
     structure s3 key
    :param prefix:
    :param suffix:
    :param dir:
    :return:
    """
    if not isinstance(dir, str):
        raise ValueError('Please enter a valid dir')

    if dir and not dir.endswith('/'):
        dir = f'{dir}/'

    return '{}{}-{}.{}'.format(dir, prefix, uuid.uuid4().hex, suffix)


def make_s3_key(file_name: str, dir: str = '/') -> str:
    """
     structure s3 key( complete )
    :param file_name:
    :param dir:
    :return:
    """
    from utils.basic.file import split_file_path
    prefix, suffix = split_file_path(file_name)
    if not all([prefix, suffix]):
        raise ValueError('Please enter a valid file name')
    key = make_s3_key_with_part(prefix, suffix, dir)
    return key


def is_false(value):
    return value == False


def is_false_f(value):
    return value[0] == False


def is_none(value):
    return not value


class BaseOssclient(object):
    def __init__(self,
                 bucket_name='',
                 access_key_id='',
                 secret_access_key='',
                 endpoint_url='',
                 region=''
                 ):
        self.bucket_name = bucket_name.replace("_", "-")
        self.access_key_id = access_key_id
        self.secret_access_key = secret_access_key
        self.endpoint_url = endpoint_url
        self.region = region

    @property
    def client(self):
        raise NotImplementedError()

    #  Transfer files to s3 The specified bucket in 
    def upload_file(self, file_path, upload_path, add_suffix=True, **kwargs):
        """
        :param file_path:  Corresponding s3 Storage location 
        :param upload_path:  File upload location 
        :return:
        """
        raise NotImplementedError()

    def upload_file_obj(self, file, upload_path, **kwargs):
        """
        :param file_path:
        :param upload_path:  File upload location 
        :return:
        """
        raise NotImplementedError()

    def get_signed_url(self, key):
        """
         Signature 
        :param key:
        :return:
        """
        raise NotImplementedError()

    def download_file(self, download_path, key):
        """
         Download files to local 
        :param download_path:
        :param key:
        :return:
        """
        raise NotImplementedError()

    def download_file_obj(self, key):
        """
         Download the file to buffer
        :param key:
        :return:
        """
        raise NotImplementedError()

    def delete_file(self, key):
        """
         Delete file 
        :param key:
        :return:
        """
        raise NotImplementedError()

    def check_key_is_exist(self, key):
        """
         Judge whether the file exists 
        :param key:
        :return:
        """
        raise NotImplementedError()

    def get_pages(self, delimiter='/', prefix=None):
        """
         Pagination 
        :param delimiter:
        :param prefix:
        :return:
        """
        raise NotImplementedError()

    def get_object_list(self, delimiter='/', prefix=None):
        """
         Get all the files 
        :param delimiter:
        :param prefix:
        :return:
        """
        raise NotImplementedError()

    def download_dir(self, delimiter='/', prefix=None, local='/tmp'):
        """
         Download the entire folder 
        :param delimiter:
        :param prefix:
        :param local:
        :return:
        """
        raise NotImplementedError()

    def creat_bucket(self):
        """
         Creating buckets 
        :return:
        """
        raise NotImplementedError()
 Copy code 

minio_s3.py

import os
from datetime import timedelta

from django.conf import settings
from minio import Minio
from tenacity import stop_after_delay, stop_after_attempt, wait_random, retry_if_result, retry_if_exception_type, retry

from base import make_s3_key ,BaseOssclient, is_false, is_false_f

#  obtain S3 Of url
def get_private_url(key: str) -> str:
    """
    :param key: minios3 Stored name
    :return: url
    """
    client = MinioS3()
    return client.get_signed_url(key=key)


def get_public_url(key: str) -> str:
    url = f"{settings.MINIO_SCHEME}://{settings.MINIO_ENDPOINT_URL}/" \
          f"{settings.PUBLIC_BUCKET_NAME.replace('_', '-')}/{key}"
    return url


class MinioS3(BaseOssclient):
    def __init__(self, bucket_name='',
                 access_key_id='',
                 secret_access_key='',
                 endpoint_url='',
                 region='',
                 is_secure: bool = None
                 ):
        self.bucket_name = bucket_name.replace("_", "-") or settings.BUCKET_NAME.replace("_", "-")
        self.access_key_id = access_key_id or settings.MINIO_ACCESS_KEY_ID
        self.secret_access_key = secret_access_key or settings.MINIO_SECRET_ACCESS_KEY
        self.endpoint_url = endpoint_url or settings.MINIO_ENDPOINT_URL
        self.region = region or settings.MINIO_REGION
        if is_secure is None:
            self.is_secure = False if settings.MINIO_SCHEME == "http" else True
        else:
            self.is_secure = is_secure
        super().__init__(bucket_name=self.bucket_name,
                         access_key_id=self.access_key_id,
                         secret_access_key=self.secret_access_key,
                         endpoint_url=self.endpoint_url,
                         region=self.region)

    @property
    def client(self):
        client = Minio(
            self.endpoint_url,
            access_key=self.access_key_id,
            secret_key=self.secret_access_key,
            secure=self.is_secure,
            region=self.region
        )
        return client

    def creat_bucket(self):
        """
         Creating buckets 
        :return:
        """
        if not self.client.bucket_exists(self.bucket_name):
            self.client.make_bucket(self.bucket_name)
            return True
        return False

    #  Transfer files to s3 The specified bucket in 
    @retry(reraise=True,
           stop=(stop_after_delay(10) | stop_after_attempt(5)),
           wait=wait_random(min=0.01, max=0.03),
           retry=(retry_if_result(is_false) | retry_if_exception_type(Exception)))
           
    def upload_file(self, file_path, upload_path, add_suffix=True, **kwargs):
        """
        :param file_path:  Local 
        :param upload_path:  Distal 
        :return:
        """
        if os.path.exists(file_path):
            self.client.fput_object(bucket_name=self.bucket_name,
                                    object_name=make_s3_key(upload_path) if add_suffix else upload_path,
                                    file_path=file_path,
                                    num_parallel_uploads=5
                                    )
            return True
        else:
            return False

    @retry(reraise=True, stop=(stop_after_delay(10) | stop_after_attempt(5)), wait=wait_random(min=0.01, max=0.03),
           retry=(retry_if_result(is_false) | retry_if_exception_type(Exception)))
    def upload_file_obj(self, file, upload_path, **kwargs):
        """
        :param file:
        :param upload_path:  File upload location 
        :return:
        """
        self.client.put_object(bucket_name=self.bucket_name,
                               object_name=upload_path,
                               data=file,
                               length=file.size,
                               content_type=file.content_type,
                               num_parallel_uploads=5)
        return True

    @retry(reraise=True,
           stop=(stop_after_delay(10) | stop_after_attempt(5)),
           wait=wait_random(min=0.01, max=0.03),
           retry=(retry_if_result(is_false_f) | retry_if_exception_type(Exception)))
    def download_file(self, download_path, key):
        self.client.fget_object(self.bucket_name, key, download_path)
        return True, download_path

    @retry(reraise=True, stop=(stop_after_delay(10) | stop_after_attempt(5)), wait=wait_random(min=0.01, max=0.03),
           retry=(retry_if_result(is_false_f) | retry_if_exception_type(Exception)))
    def download_file_obj(self, key):
        data = self.client.get_object(self.bucket_name, key)
        from utils.basic.file import content_to_file
        buf = content_to_file(data.read())
        return True, buf

    @retry(reraise=True, stop=(stop_after_delay(10) | stop_after_attempt(5)), wait=wait_random(min=0.01, max=0.03),
           retry=(retry_if_result(is_false) | retry_if_exception_type(Exception)))
    def delete_file(self, key):
        """
         Delete file 
        :param key:
        :return:
        """
        self.client.remove_object(self.bucket_name, key)
        return True

    @retry(reraise=True, stop=(stop_after_delay(10) | stop_after_attempt(5)), wait=wait_random(min=0.01, max=0.03),
           retry=(retry_if_result(is_false) | retry_if_exception_type(Exception)))
    def get_signed_url(self, key):
        if not key:
            return ''
        url = self.client.presigned_get_object(self.bucket_name, key, expires=timedelta(hours=24))
        return url

    def public_policy(self, bucket_name):
        policy = '{"Version":"","Statement":[{"Effect":"Allow","Principal":' \
                 '{"AWS": ["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":' \
                 '["arn:aws:s3:::%s"]},{"Effect":"Allow","Principal":{"AWS":["*"]},' \
                 '"Action": ["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}' % (
                     bucket_name, bucket_name)
        return policy
 Copy code 

copyright notice
author[Jin Fengjun],Please bring the original link to reprint, thank you.
https://en.pythonmana.com/2022/01/202201300132023681.html

Random recommended