diff --git a/app/config.py b/app/config.py index 37ff2d8..217db10 100644 --- a/app/config.py +++ b/app/config.py @@ -1,7 +1,10 @@ from pydantic_settings import BaseSettings class Settings(BaseSettings): - # S3 settings + # COSI settings (preferred) + cosi_bucket_info_path: str = "" + + # S3 settings (fallback) s3_endpoint: str = "http://localhost:9000" s3_access_key_id: str = "minioadmin" s3_secret_key: str = "minioadmin" diff --git a/app/s3.py b/app/s3.py index 3f42bd3..b2d7f2d 100644 --- a/app/s3.py +++ b/app/s3.py @@ -1,6 +1,7 @@ import boto3 -import tempfile +import json import os +import tempfile from botocore.client import Config from fastapi import UploadFile from app.config import settings @@ -8,7 +9,36 @@ from app.logger import get_logger logger = get_logger(__name__) + +def read_bucket_info() -> dict: + if not settings.cosi_bucket_info_path: + raise ValueError("COSI_BUCKET_INFO_PATH not set") + with open(settings.cosi_bucket_info_path, "r") as f: + return json.load(f) + + +def get_cosi_s3_config() -> dict: + bucket_info = read_bucket_info() + s3_conf = bucket_info["spec"]["secretS3"] + return { + "endpoint": s3_conf["endpoint"], + "access_key": s3_conf["accessKeyID"], + "secret_key": s3_conf["accessSecretKey"], + "bucket": bucket_info["spec"]["bucketName"] + } + + def get_client(): + if settings.cosi_bucket_info_path: + cosi_config = get_cosi_s3_config() + return boto3.client( + "s3", + endpoint_url=cosi_config["endpoint"], + aws_access_key_id=cosi_config["access_key"], + aws_secret_access_key=cosi_config["secret_key"], + config=Config(signature_version="s3v4"), + region_name="us-east-1" + ) return boto3.client( "s3", endpoint_url=settings.s3_endpoint, @@ -18,51 +48,54 @@ def get_client(): region_name=settings.s3_region ) + +def get_bucket_name() -> str: + if settings.cosi_bucket_info_path: + return get_cosi_s3_config()["bucket"] + return settings.s3_bucket + + def ensure_bucket_exists() -> None: - """Ensure the S3 bucket exists, create it if it doesn't exist. - - Raises: - Exception: If bucket creation fails (service will fail to start) - """ + bucket_name = get_bucket_name() client = get_client() try: - client.head_bucket(Bucket=settings.s3_bucket) - logger.info(f"Bucket '{settings.s3_bucket}' already exists") + client.head_bucket(Bucket=bucket_name) + logger.info(f"Bucket '{bucket_name}' already exists") except client.exceptions.ClientError as e: error_code = e.response['Error']['Code'] if error_code == '404': try: client.create_bucket( - Bucket=settings.s3_bucket, + Bucket=bucket_name, CreateBucketConfiguration={ - 'LocationConstraint': settings.s3_region + 'LocationConstraint': 'us-east-1' } ) - logger.info(f"Created bucket '{settings.s3_bucket}'") + logger.info(f"Created bucket '{bucket_name}'") except Exception as create_error: - logger.error(f"Failed to create bucket '{settings.s3_bucket}': {create_error}") + logger.error(f"Failed to create bucket '{bucket_name}': {create_error}") raise else: logger.error(f"Error checking bucket: {e}") raise + def upload_file(file: UploadFile, s3_key: str, content_type: str, metadata: dict = None) -> str: - """Upload file to S3 with metadata""" + bucket_name = get_bucket_name() client = get_client() - - # Read file content + file.file.seek(0, os.SEEK_END) file_size = file.file.tell() file.file.seek(0) file_content = file.file.read() file.file.seek(0) - + extra_args = {"ContentType": content_type} if metadata: extra_args["Metadata"] = metadata - + client.put_object( - Bucket=settings.s3_bucket, + Bucket=bucket_name, Key=s3_key, Body=file_content, ContentLength=file_size, @@ -71,40 +104,45 @@ def upload_file(file: UploadFile, s3_key: str, content_type: str, metadata: dict ) return s3_key + def delete_file(s3_key: str) -> None: - """Delete file from S3""" + bucket_name = get_bucket_name() client = get_client() - client.delete_object(Bucket=settings.s3_bucket, Key=s3_key) + client.delete_object(Bucket=bucket_name, Key=s3_key) + def file_exists(s3_key: str) -> bool: - """Check if file exists in S3""" + bucket_name = get_bucket_name() client = get_client() try: - client.head_object(Bucket=settings.s3_bucket, Key=s3_key) + client.head_object(Bucket=bucket_name, Key=s3_key) return True except client.exceptions.ClientError: return False + def get_file_metadata(s3_key: str) -> dict: - """Get file metadata from S3""" + bucket_name = get_bucket_name() client = get_client() - response = client.head_object(Bucket=settings.s3_bucket, Key=s3_key) + response = client.head_object(Bucket=bucket_name, Key=s3_key) return response.get("Metadata", {}) + def download_to_temp(s3_key: str) -> str: - """Download file from S3 to temp file""" + bucket_name = get_bucket_name() client = get_client() suffix = os.path.splitext(s3_key)[-1] or ".tmp" tmp = tempfile.NamedTemporaryFile(delete=False, suffix=suffix) - client.download_fileobj(settings.s3_bucket, s3_key, tmp) + client.download_fileobj(bucket_name, s3_key, tmp) tmp.close() return tmp.name + def presigned_download_url(s3_key: str, expires_in: int = 3600) -> str: - """Generate presigned download URL""" + bucket_name = get_bucket_name() client = get_client() return client.generate_presigned_url( "get_object", - Params={"Bucket": settings.s3_bucket, "Key": s3_key}, + Params={"Bucket": bucket_name, "Key": s3_key}, ExpiresIn=expires_in - ) + ) \ No newline at end of file diff --git a/ops/chart/values.yaml b/ops/chart/values.yaml index e66f352..7b0f2de 100644 --- a/ops/chart/values.yaml +++ b/ops/chart/values.yaml @@ -11,22 +11,8 @@ controllers: env: LOG_LEVEL: info PORT: "8082" - S3_ENDPOINT: - value: "https://dev.s3.corredorconect.com/" - S3_ACCESS_KEY_ID: - valueFrom: - secretKeyRef: - name: 'document-service-s3-credentials' - key: rootAccessKeyId - S3_SECRET_KEY: - valueFrom: - secretKeyRef: - name: 'document-service-s3-credentials' - key: rootSecretAccessKey - S3_BUCKET: - value: "document-bucket" - S3_REGION: - value: "us-east-1" + COSI_BUCKET_INFO_PATH: + value: "/var/run/secrets/cosi/BucketInfo" probes: liveness: enabled: true @@ -56,6 +42,15 @@ service: port: 8082 protocol: HTTP +persistence: + cosi-bucket-info: + enabled: true + type: secret + name: document-service-s3-credentials + globalMounts: + - path: /var/run/secrets/cosi + readOnly: true + rawResources: bucket: enabled: true