How to debug and fix slow upload performance in MinIO?

# /etc/systemd/system/minio.service optimizations

[Service]
# Increase file descriptor limits
LimitNOFILE=1048576

# Increase process limits
LimitNPROC=1048576

# CPU scheduling
CPUSchedulingPolicy=2
CPUSchedulingPriority=99

# Memory settings
MemoryLimit=16G
MemorySwappiness=1

# I/O settings
IOSchedulingClass=1
IOSchedulingPriority=4

package main

import (
    "context"
    "fmt"
    "io"
    "log"
    "sync"

    "github.com/minio/minio-go/v7"
    "github.com/minio/minio-go/v7/pkg/credentials"
)

func optimizedUpload(client *minio.Client, bucketName, objectName string, reader io.Reader, objectSize int64) error {
    options := minio.PutObjectOptions{
        PartSize:     64 * 1024 * 1024, // 64MB parts
        NumThreads:   4,                // Parallel uploads
        ContentType:  "application/octet-stream",
        SendContentMd5: true,
    }

    _, err := client.PutObject(context.Background(), bucketName, objectName, reader, objectSize, options)
    return err
}

func concurrentUploads(client *minio.Client, files []string) {
    var wg sync.WaitGroup
    semaphore := make(chan struct{}, 10) // Limit concurrent uploads

    for _, file := range files {
        wg.Add(1)
        go func(filename string) {
            defer wg.Done()
            semaphore <- struct{}{}
            defer func() { <-semaphore }()

            // Upload logic here
            fmt.Printf("Uploading %s\n", filename)
        }(file)
    }

    wg.Wait()
}

from minio import Minio
from concurrent.futures import ThreadPoolExecutor, as_completed
import os

def upload_file_optimized(client, bucket, file_path):
    """Upload file with optimized settings"""
    file_size = os.path.getsize(file_path)

    # Use larger part size for better performance
    part_size = 64 * 1024 * 1024  # 64MB

    with open(file_path, 'rb') as file_data:
        client.put_object(
            bucket,
            os.path.basename(file_path),
            file_data,
            file_size,
            part_size=part_size
        )

def concurrent_upload(client, bucket, file_list, max_workers=10):
    """Upload multiple files concurrently"""
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        futures = {
            executor.submit(upload_file_optimized, client, bucket, file_path): file_path
            for file_path in file_list
        }

        for future in as_completed(futures):
            file_path = futures[future]
            try:
                future.result()
                print(f"Successfully uploaded {file_path}")
            except Exception as e:
                print(f"Failed to upload {file_path}: {e}")

# Optimized HAProxy configuration for MinIO
backend minio_servers
    balance roundrobin
    option httpchk GET /minio/health/live

    # Connection optimization
    option tcp-check
    option log-health-checks

    # Timeout settings
    timeout connect 5s
    timeout server 60s

    # Buffer sizes
    tune.bufsize 32768
    tune.maxrewrite 8192

    server minio1 minio-node-1:9000 check maxconn 1000
    server minio2 minio-node-2:9000 check maxconn 1000
    server minio3 minio-node-3:9000 check maxconn 1000
    server minio4 minio-node-4:9000 check maxconn 1000

upstream minio_servers {
    least_conn;
    server minio-node-1:9000 max_fails=2 fail_timeout=30s;
    server minio-node-2:9000 max_fails=2 fail_timeout=30s;
    server minio-node-3:9000 max_fails=2 fail_timeout=30s;
    server minio-node-4:9000 max_fails=2 fail_timeout=30s;
}

server {
    # Buffer optimization
    client_body_buffer_size 1M;
    client_max_body_size 1G;
    client_body_timeout 300s;

    # Proxy settings
    proxy_buffering off;
    proxy_request_buffering off;
    proxy_http_version 1.1;
    proxy_set_header Connection "";

    location / {
        proxy_pass http://minio_servers;
    }
}