基于aws-s3安裝

安裝

安裝環(huán)境依賴：

Debian/Ubuntu-based systems : sudo apt-get install libcurl4-openssl-dev
libssl-dev uuid-dev zlib1g-dev libpulse-dev cmake
Redhat/Fedora-based systems : sudo dnf install libcurl-devel openssl-devel
libuuid-devel pulseaudio-devel cmake

獲取SDK源碼并安裝：

git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git
cd aws-sdk-cpp
mkdir build
cd build
cmake ..
make -j8
sudo make install

項目中CMakeLists.txt配置：

cmake_minimum_required(VERSION 3.19)
project(project_name)
set(CMAKE_CXX_STANDARD 11)
 
# Find the AWS SDK for C++ package.
find_package(AWSSDK CONFIG COMPONENTS core s3 REQUIRED)
 
include_directories(${AWSSDK_INCLUDE_DIRS})
add_executable(project_name mian.cpp)
target_link_libraries(project_name PRIVATE  aws-cpp-sdk-s3)

初始化S3連接

S3Client InitS3Client(const Aws::String IPPort, const Aws::String& accessKeyId, const Aws::String& secretKey, std::string securityToken){
    // 初始化 S3 Client
    Aws::Client::ClientConfiguration cfg;
    cfg.endpointOverride = IPPort;
    cfg.scheme = Aws::Http::Scheme::HTTP;
    cfg.verifySSL = false;

    Aws::Auth::AWSCredentials cred(accessKeyId, secretKey, securityToken);
    S3Client client(cred, cfg,
                    Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false);
    return client;
}

上傳文件

一般文件上傳

/// 判斷待上傳的文件是否存在
inline bool file_exists(const string& name)
{
    struct stat buffer;
    return (stat(name.c_str(), &buffer) == 0);
}

/// 上傳文件
bool doPutObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const string& file_name, const Aws::String &region)
{
    // 判斷文件是否存在
    if (!file_exists(file_name)) {
        cout << "ERROR: 找不到這個文件，這個文件不存在"
             << endl;
        return false;
    }

    /// 初始化上傳請求
    Aws::S3::Model::PutObjectRequest object_request;
    object_request.SetBucket(fromBucket);
    object_request.SetKey(objectKey);

    /// 建立文件輸入串流，可為任何串流
    const shared_ptr<Aws::IOStream> input_data =
            Aws::MakeShared<Aws::FStream>("SampleAllocationTag",
                                          file_name.c_str(),
                                          ios_base::in | ios_base::binary);
    object_request.SetBody(input_data);

    /// 開始上傳至S3
    Aws::S3::Model::PutObjectOutcome put_object_outcome = client.PutObject(object_request);

    if (!put_object_outcome.IsSuccess()) {
        auto error = put_object_outcome.GetError();
        cout << "ERROR: " << error.GetExceptionName() << ": "
             << error.GetMessage() << endl;
        return false;
    }else {
        cout << "success" << endl;
        return true;
    }
}

/// 啟動上傳程序
bool StartUpLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){
    // 初始化
    Aws::SDKOptions options;
    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
    Aws::InitAPI(options);

    /// 設置ip、acccessKey、secretKey
    Aws::String IP = Ips;
    Aws::String accessKeyId = acccessKey;
    Aws::String secretKey = secretKeyS;

    S3Client client;
    client = InitS3Client(IP, accessKeyId, secretKey, securityToken);

    Aws::InitAPI(options);
    {
        /// 設置上傳的目標桶和文件路徑
        const Aws::String bucket_name = bucket_names;
        const Aws::String object_name = object_names;
        if (!doPutObject(object_name, bucket_name, client, imagename, " ")) {
            std::string errorInfo =  object_name + " 上傳失敗";
            writeLog(L_ERROR, errorInfo.c_str());
            return false;
        }
    }
    writeLog(L_INFO, "====> 文件上傳成功");
    return true;
}

分塊上傳

/// MINIO 切片上傳
bool mainCutUpload(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){


    Aws::SDKOptions options;
    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
    Aws::InitAPI(options);

    Aws::String IP = Ips;
    Aws::String accessKeyId = acccessKey;
    Aws::String secretKey = secretKeyS;

    /// 初始化S3服務
    S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken);

    Aws::InitAPI(options);
    {
        /// 設置上傳的目標桶和文件路徑
        const Aws::String bucket_name = bucket_names;
        const Aws::String object_name = object_names;
        std::string bucket = bucket_name;
        std::string key = object_name;

        /// 初始化分塊上傳任務
        Aws::S3::Model::CreateMultipartUploadRequest create_request;
        create_request.SetBucket(bucket.c_str());
        create_request.SetKey(key.c_str());
        Aws::S3::Model::CreateMultipartUploadOutcome outcome = client.CreateMultipartUpload(create_request);

        /// 獲取上傳ID
        std::string upload_id = outcome.GetResult().GetUploadId();
        std::cout << "upload id is:" << upload_id << std::endl;

        /// 創(chuàng)建分段上傳輸出
        if (!outcome.IsSuccess()) {
            auto err = outcome.GetError();
            std::cout << "Error: CreateMultipartUpload: " <<
                      err.GetExceptionName() << ": " << err.GetMessage() << std::endl;
            return false;
        }
        else
            std::cout << "Success: CreateMultipartUpload: " << bucket << std::endl;

        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetBucket() << std::endl;
        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetKey() << std::endl;
        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetUploadId() << std::endl;

        Aws::String UploadId = upload_id;

        /// 啟動分塊上傳
        Aws::S3::Model::UploadPartRequest request1;
        request1.SetBucket(bucket);
        request1.SetKey(key);

        /// 每個分片大小
        long partSize = 5 * 1024 * 1024;

        /// 讀取文件
        std::fstream file(imagename.c_str(),std::ios::in | std::ios::binary);
        file.seekg(0,std::ios::end);
        long fileSize = file.tellg();   /// 獲取文件大小
        file.seekg(0, std::ios::beg);
        char* buffer = new char[partSize];  // 定義緩存

        /// 初始化上傳變量
        long filePosition = 0;
        Aws::Vector<Aws::S3::Model::CompletedPart> completeParts;
        int partNumber = 1;

        /// 開始上傳
        while(filePosition < fileSize){
            partSize = std::min(partSize,(fileSize - filePosition));
            file.read(buffer,partSize);
            Aws::S3::Model::UploadPartRequest uploadPartRequest;
            uploadPartRequest.WithBucket(bucket).WithKey(key).WithUploadId(UploadId).WithPartNumber(partNumber).WithContentLength(partSize);
            Aws::String str(buffer,partSize);
            auto input_data = Aws::MakeShared<Aws::StringStream>("UploadPartStream",str);
            uploadPartRequest.SetBody(input_data);
            filePosition += partSize;
            auto uploadPartResult = client.UploadPart(uploadPartRequest);
            completeParts.push_back(Aws::S3::Model::CompletedPart().WithETag(uploadPartResult.GetResult().GetETag()).WithPartNumber(partNumber));
            memset(buffer, 0, partSize);
            ++partNumber;
        }

        // 完成分塊上傳
        Aws::S3::Model::CompleteMultipartUploadRequest request2;
        request2.SetBucket(bucket);
        request2.SetKey(key);
        request2.SetUploadId(UploadId);
        Aws::S3::Model::CompletedMultipartUpload completed_multipart_upload;

        completed_multipart_upload.SetParts(completeParts);
        request2.SetMultipartUpload(completed_multipart_upload);
        Aws::S3::Model::CompleteMultipartUploadOutcome outcome3 = client.CompleteMultipartUpload(request2);

        if (outcome3.IsSuccess()) {
            std::cout << "Success: CompleteMultipartUpload " << std::endl;
            return true;
        }
        else
        {
            std::cout << "Error: CompleteMultipartUpload: " <<
                      outcome.GetError().GetMessage() << std::endl;
            return false;
        }

    }
}

下載文件

/// 從MINIO中下載文件或數據
bool doGetObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const Aws::String savePath,const Aws::String &region)
{
    /// 初始化下載請求
    Aws::S3::Model::GetObjectRequest object_request;
    object_request.SetBucket(fromBucket);
    object_request.SetKey(objectKey);

    /// 從S3服務器中下載數據
    Aws::S3::Model::GetObjectOutcome get_object_outcome = client.GetObject(object_request);

    if (get_object_outcome.IsSuccess()) {
        /// 保存文件
        Aws::OFStream local_file;
        local_file.open(savePath, std::ios::out | std::ios::binary);
        local_file << get_object_outcome.GetResultWithOwnership().GetBody().rdbuf();
        std::cout << "Done!" << std::endl;
        return true;
    } else {
        auto err = get_object_outcome.GetError();
        std::cout << "Error: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl;
        return false;
    }
}


/// 啟動下載程序
bool StartDownLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_name, std::string downPath, std::string imagePath, std::string securityToken,
    bool &flags){
    // 初始化Aws API
    Aws::SDKOptions options;
    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
    Aws::InitAPI(options);

    /// 設置ip、acccessKey、secretKey
    Aws::String IP = Ips;
    Aws::String accessKeyId = acccessKey;
    Aws::String secretKey = secretKeyS;

    /// 初始化S3服務
    S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken);

    /// 根據get接口的桶名稱設置文件所在的桶
    Aws::S3::Model::Bucket temps{};
    std::string stringK = bucket_name;
    temps.SetName(stringK);

    /// 查找MINIO相關數據
    Aws::S3::Model::ListObjectsRequest objects_request;
    objects_request.WithBucket(temps.GetName());   /// 設置桶路徑
    objects_request.SetPrefix(downPath);   /// 設置文件夾路徑

    auto list_objects_outcome = client.ListObjects(objects_request);  /// 判斷存儲桶及相關連接是否有效

    if (flags){
        writeLog(L_INFO, "====> 文件下載終止");
        return false;
    }
    if (list_objects_outcome.IsSuccess())
    {
        /// 獲取該目錄下的前一千個文件
        Aws::Vector<Aws::S3::Model::Object> object_list =
                list_objects_outcome.GetResult().GetContents();

        if (object_list.size() == 1000){

            /// 當下載數量大于一千時 循環(huán)下載文件
            while(object_list.size() == 1000){
                auto finally_object = object_list[0];
                for (auto const &s3_object : object_list)
                {
                    if (flags){
                        writeLog(L_INFO, "====> 文件下載終止");
                        return false;
                    }
                    /// 設置桶名稱
                    const Aws::String bucket_name = stringK;
                    /// 設置待下載的文件名稱
                    const Aws::String object_name = s3_object.GetKey();
                    /// 設置保存文件路徑及文件名稱
                    std::string NewPath;
                    size_t indexNew = object_name.find_last_of('/');
                    NewPath = object_name.substr(indexNew + 1, object_name.length());
                    const Aws::String SavePath = imagePath + NewPath;
                    size_t indexK = downPath.find_last_of('/');
                    std::string newPathEs = downPath.substr(0, indexK);
                    std::string strK;
                    strK = imagePath + "/" + NewPath;
                    const Aws::String SavePaths = strK;

                    //篩選數據
                    std::string fileName;
                    size_t index = object_name.find_last_of('.');
                    fileName = object_name.substr(index + 1, object_name.length());
                    if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){
                        /// 啟動下載
                        if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {
                            std::string errorInfo =  object_name + " 下載失敗";
                            writeLog(L_ERROR, errorInfo.c_str());
                            continue;
                        }
                    }
                }
                finally_object = object_list[999];
                /// 重新配置相關路徑
                objects_request.WithBucket(temps.GetName());   /// 設置桶路徑
                objects_request.SetPrefix(finally_object.GetKey());   /// 設置新文件夾路徑

                list_objects_outcome = client.ListObjects(objects_request);  /// 判斷存儲桶及相關連接是否有效
                if (list_objects_outcome.IsSuccess()){
                    object_list = list_objects_outcome.GetResult().GetContents();
                }else{
                    writeLog(L_ERROR, "====> 再次連接失敗");
                }
            }
            for (auto const &s3_object : object_list)
            {
                if (flags){
                    writeLog(L_INFO, "====> 文件下載終止");
                    return false;
                }
                /// 設置桶名稱
                const Aws::String bucket_name = stringK;
                /// 設置待下載的文件名稱
                const Aws::String object_name = s3_object.GetKey();
                /// 設置保存文件路徑及文件名稱
                std::string NewPath;
                size_t indexNew = object_name.find_last_of('/');
                NewPath = object_name.substr(indexNew + 1, object_name.length());
                const Aws::String SavePath = imagePath + NewPath;
                size_t indexK = downPath.find_last_of('/');
                std::string newPathEs = downPath.substr(0, indexK);

                std::string strK;
                strK = imagePath + "/" + NewPath;
                const Aws::String SavePaths = strK;

                //篩選數據
                std::string fileName;
                size_t index = object_name.find_last_of('.');
                fileName = object_name.substr(index + 1, object_name.length());
                if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){
                    /// 啟動下載
                    if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {
                        std::string errorInfo =  object_name + " 下載失敗";
                        writeLog(L_ERROR, errorInfo.c_str());
                        continue;
                    }
                }
            }
        }else{
            for (auto const &s3_object : object_list)
            {
                if (flags){
                    writeLog(L_INFO, "====> 文件下載終止");
                    return false;
                }
                /// 設置桶名稱
                const Aws::String bucket_name = stringK;
                /// 設置待下載的文件名稱
                const Aws::String object_name = s3_object.GetKey();
                /// 設置保存文件路徑及文件名稱
                std::string NewPath;
                size_t indexNew = object_name.find_last_of('/');
                NewPath = object_name.substr(indexNew + 1, object_name.length());
                const Aws::String SavePath = imagePath +  NewPath;
                size_t indexK = downPath.find_last_of('/');
                std::string newPathEs = downPath.substr(0, indexK);
                std::string strK;
                strK = imagePath + "/" + NewPath;
                const Aws::String SavePaths = strK;

                //篩選數據
                std::string fileName;
                size_t index = object_name.find_last_of('.');
                fileName = object_name.substr(index + 1, object_name.length());
                if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){
                    /// 啟動下載
                    if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {
                        std::string errorInfo =  object_name + " 下載失敗";
                        writeLog(L_ERROR, errorInfo.c_str());
                        continue;
                    }
                }

            }
        }

    }
    writeLog(L_INFO, "====> 文件下載成功");
    return true;
}

參考資料

SDK文檔資料
C++_SDK.pdf

下載地址

實現下載的整體類代碼下載
具體內容如下
文章來源地址http://www.zghlxwxcb.cn/news/detail-846259.html

到了這里，關于MINIO服務器基于AWS S3 SDK 文件分片上傳及下載（C++實現）的文章就介紹完了。如果您還想了解更多內容，請在右上角搜索TOY模板網以前的文章或繼續(xù)瀏覽下面的相關文章，希望大家以后多多支持TOY模板網！