🚚(back) serve legacy deposited files from Scaleway S3

wilbrdt · wilbrdt · commit 11cc20c8993e · 2025-07-15T10:45:07.000+02:00
Deposited files are now served from the aws/ directory in Scaleway S3.
They are served using the django-storage already in place.
As content headers cannot be set to Scaleway Edge services URLs, legacy
deposited files need to be renamed to their filenames. Adding a management
command to do that.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,7 +8,11 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
-## [5.10.] - 2025-07-09
+### Changed
+
+- Serve legacy deposited files from Scaleway S3 after AWS migration
+
+## [5.10.0] - 2025-07-09
 
 ### Changed
 
diff --git a/src/backend/marsha/deposit/management/commands/rename_deposited_files.py b/src/backend/marsha/deposit/management/commands/rename_deposited_files.py
@@ -0,0 +1,88 @@
+"""Rename deposited files in Scaleway S3 for serving them with the right name."""
+
+import logging
+from os.path import splitext
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+import boto3
+
+from marsha.core.defaults import AWS_S3, AWS_STORAGE_BASE_DIRECTORY, READY
+from marsha.core.storage.storage_class import file_storage
+from marsha.core.utils import time_utils
+from marsha.deposit.models import DepositedFile
+
+
+logger = logging.getLogger(__name__)
+
+scw_credentials = {
+    "aws_access_key_id": settings.STORAGE_S3_ACCESS_KEY,
+    "aws_secret_access_key": settings.STORAGE_S3_SECRET_KEY,
+    "region_name": settings.STORAGE_S3_REGION_NAME,
+    "endpoint_url": settings.STORAGE_S3_ENDPOINT_URL,
+}
+
+# Configure medialive client
+s3_client = boto3.client("s3", **scw_credentials)
+
+
+class Command(BaseCommand):
+    """Rename deposited files in Scaleway S3 to their filename."""
+
+    help = "Rename deposited files in Scaleway S3 to their filename."
+
+    def validate_filename(self, value):
+        """Transform filename to make it valid."""
+
+        value = value.replace("/", "_")
+        value = value.replace("\\", "_")
+        value = value.lstrip(".")
+
+        return value
+
+    def handle(self, *args, **options):
+        """Execute management command."""
+
+        files = DepositedFile.objects.filter(
+            storage_location=AWS_S3, upload_state=READY
+        )
+
+        for file in files:
+            # Get the file stored on Scaleway S3 under `aws/`
+            stamp = time_utils.to_timestamp(file.uploaded_on)
+            extension = ""
+            if "." in file.filename:
+                extension = splitext(file.filename)[1]
+
+            file_key_src = file.get_storage_key(
+                filename=f"{stamp}{extension}", base_dir=AWS_STORAGE_BASE_DIRECTORY
+            )
+            copy_source = {
+                "Bucket": settings.STORAGE_S3_BUCKET_NAME,
+                "Key": file_key_src,
+            }
+
+            filename = self.validate_filename(file.filename)
+
+            # Override file filename with the validated S3-compatible filename
+            if filename != file.filename:
+                file.filename = filename
+                file.save()
+
+            # Compute file key destination which should be the document filename
+            file_key_dest = file.get_storage_key(
+                filename, base_dir=AWS_STORAGE_BASE_DIRECTORY
+            )
+            if file_storage.exists(file_key_dest):
+                logger.info("Object %s already exists", file_key_dest)
+                continue
+
+            logger.info("Copying %s to %s", file_key_src, file_key_dest)
+            s3_client.copy_object(
+                Bucket=settings.STORAGE_S3_BUCKET_NAME,
+                CopySource=copy_source,
+                Key=file_key_dest,
+            )
+
+        logger.info("Finished copying!")
diff --git a/src/backend/marsha/deposit/serializers.py b/src/backend/marsha/deposit/serializers.py
@@ -2,23 +2,20 @@
 
 import mimetypes
 from os.path import splitext
-from urllib.parse import quote, quote_plus
 
 from django.conf import settings
 from django.urls import reverse
 
 from rest_framework import serializers
 
-from marsha.core.defaults import SCW_S3
+from marsha.core.defaults import AWS_STORAGE_BASE_DIRECTORY, SCW_S3
 from marsha.core.models import User
 from marsha.core.serializers import (
     BaseInitiateUploadSerializer,
     UploadableFileWithExtensionSerializerMixin,
-    get_resource_cloudfront_url_params,
 )
 from marsha.core.serializers.playlist import PlaylistLiteSerializer
 from marsha.core.storage.storage_class import file_storage
-from marsha.core.utils import cloudfront_utils, time_utils
 from marsha.deposit.models import DepositedFile, FileDepository
 
 
@@ -145,26 +142,10 @@ def get_url(self, obj):
 
             return file_storage.url(file_key)
 
-        base = (
-            f"{settings.AWS_S3_URL_PROTOCOL}://{settings.CLOUDFRONT_DOMAIN}/"
-            f"{obj.file_depository.pk}/depositedfile/{obj.pk}/"
-            f"{time_utils.to_timestamp(obj.uploaded_on)}"
+        file_key = obj.get_storage_key(
+            obj.filename, base_dir=AWS_STORAGE_BASE_DIRECTORY
         )
-
-        response_content_disposition = quote_plus(
-            "attachment; filename=" + quote(obj.filename)
-        )
-        url = (
-            f"{base:s}{self._get_extension_string(obj)}?"
-            f"response-content-disposition={response_content_disposition}"
-        )
-
-        if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
-            params = get_resource_cloudfront_url_params(
-                "depositedfile", obj.file_depository_id
-            )
-            url = cloudfront_utils.build_signed_url(url, params)
-        return url
+        return file_storage.url(file_key)
 
 
 class DepositedFileInitiateUploadSerializer(BaseInitiateUploadSerializer):
diff --git a/src/backend/marsha/deposit/tests/api/depositedfiles/test_list.py b/src/backend/marsha/deposit/tests/api/depositedfiles/test_list.py
@@ -24,7 +24,8 @@
 
 
 # We don't enforce arguments documentation in tests
-# pylint: disable=unused-argument
+# flake8: noqa: E501
+# pylint: disable=unused-argument,line-too-long
 
 
 @override_settings(DEPOSIT_ENABLED=True)
@@ -292,11 +293,8 @@ def test_api_file_depository_list_deposited_files_instructor_filtered(self):
             },
         )
 
-    @override_settings(
-        CLOUDFRONT_SIGNED_URLS_ACTIVE=True,
-        CLOUDFRONT_SIGNED_PUBLIC_KEY_ID="cloudfront-access-key-id",
-    )
-    def test_api_file_depository_list_deposited_files_instructor_signed_urls_aws(self):
+    @override_settings(MEDIA_URL="https://abc.svc.edge.scw.cloud/")
+    def test_api_file_depository_list_deposited_files_instructor_urls_aws(self):
         """All deposited files should have the same signature."""
         file_depository = FileDepositoryFactory(
             id="4e126eac-9ca8-47b1-8dcd-157686b43c60"
@@ -329,17 +327,6 @@ def test_api_file_depository_list_deposited_files_instructor_signed_urls_aws(sel
             )
         self.assertEqual(response.status_code, 200)
 
-        expected_cloudfront_signature = (
-            "Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9hYmMuY2xvdWRmcm9udC5uZXQvNGUxM"
-            "jZlYWMtOWNhOC00N2IxLThkY2QtMTU3Njg2YjQzYzYwLyoiLCJDb25kaXRpb24iOnsiRGF0ZUxlc3NUaGFuI"
-            "jp7IkFXUzpFcG9jaFRpbWUiOjE2MzgyMzc2MDB9fX1dfQ__&Signature=BGMvqnlKwJW~PwkL1Om4Pp7Pk5"
-            "ZLlJGgS~q5c02NIL5--QBssu6C-gbhBgfGVQOY8~YwEqkJVSFfsqX54jOvzjVi-0t4mDANocv0hD5CQAy103"
-            "79gj14UQ5-4i2lcPoDdEcpsTekrtC9W1oRzZlyKSygNnL5NJKSjLy7St3TN8AK7sHbOMYTiFEpnxvuz8CaIh"
-            "DLf0xG~IbILgw83w9D1xlmAFu9Mxe5KXXQZa6Z60dXcXf67AS9vO1YRTK4CxtfF5EkDI31DeOm-Fm78VZzFE"
-            "j4MtdzMRQV1ag~4SruE7RMS10nIgHLN7CxpdHpybqAK4V-OWXlMsx8vSxC1bLHcQ__"
-            "&Key-Pair-Id=cloudfront-access-key-id"
-        )
-
         self.assertEqual(
             response.json(),
             {
@@ -357,10 +344,8 @@ def test_api_file_depository_list_deposited_files_instructor_signed_urls_aws(sel
                         "upload_state": "pending",
                         "uploaded_on": "2018-08-08T00:00:00Z",
                         "url": (
-                            f"https://abc.cloudfront.net/{file_depository.id}/depositedfile/"
-                            f"{unicode_deposited_file.id}/1533686400?response-content-disposition"
-                            "=attachment%3B+filename%3Dtest%25E2%2580%2599.pdf"
-                            f"&{expected_cloudfront_signature}"
+                            f"https://abc.svc.edge.scw.cloud/aws/{file_depository.id}/depositedfile/"
+                            f"{unicode_deposited_file.id}/test%E2%80%99.pdf"
                         ),
                     },
                     {
@@ -373,10 +358,8 @@ def test_api_file_depository_list_deposited_files_instructor_signed_urls_aws(sel
                         "upload_state": "pending",
                         "uploaded_on": "2018-08-08T00:00:00Z",
                         "url": (
-                            f"https://abc.cloudfront.net/{file_depository.id}/depositedfile/"
-                            f"{deposited_files[2].id}/1533686400?response-content-disposition"
-                            f"=attachment%3B+filename%3D{deposited_files[2].filename}"
-                            f"&{expected_cloudfront_signature}"
+                            f"https://abc.svc.edge.scw.cloud/aws/{file_depository.id}/depositedfile/"
+                            f"{deposited_files[2].id}/{deposited_files[2].filename}"
                         ),
                     },
                     {
@@ -389,10 +372,8 @@ def test_api_file_depository_list_deposited_files_instructor_signed_urls_aws(sel
                         "upload_state": "pending",
                         "uploaded_on": "2018-08-08T00:00:00Z",
                         "url": (
-                            f"https://abc.cloudfront.net/{file_depository.id}/depositedfile/"
-                            f"{deposited_files[1].id}/1533686400?response-content-disposition"
-                            f"=attachment%3B+filename%3D{deposited_files[1].filename}"
-                            f"&{expected_cloudfront_signature}"
+                            f"https://abc.svc.edge.scw.cloud/aws/{file_depository.id}/depositedfile/"
+                            f"{deposited_files[1].id}/{deposited_files[1].filename}"
                         ),
                     },
                     {
@@ -405,19 +386,15 @@ def test_api_file_depository_list_deposited_files_instructor_signed_urls_aws(sel
                         "upload_state": "pending",
                         "uploaded_on": "2018-08-08T00:00:00Z",
                         "url": (
-                            f"https://abc.cloudfront.net/{file_depository.id}/depositedfile/"
-                            f"{deposited_files[0].id}/1533686400?response-content-disposition"
-                            f"=attachment%3B+filename%3D{deposited_files[0].filename}"
-                            f"&{expected_cloudfront_signature}"
+                            f"https://abc.svc.edge.scw.cloud/aws/{file_depository.id}/depositedfile/"
+                            f"{deposited_files[0].id}/{deposited_files[0].filename}"
                         ),
                     },
                 ],
             },
         )
 
-    @override_settings(
-        MEDIA_URL="https://abc.svc.edge.scw.cloud/",
-    )
+    @override_settings(MEDIA_URL="https://abc.svc.edge.scw.cloud/")
     def test_api_file_depository_list_deposited_files_instructor_urls_scw(self):
         """Deposited files on Scaleway should not be signed."""
         file_depository = FileDepositoryFactory(
diff --git a/src/backend/marsha/deposit/tests/test_command_rename_deposited_files.py b/src/backend/marsha/deposit/tests/test_command_rename_deposited_files.py
@@ -0,0 +1,119 @@
+"""Test the ``rename_deposited_files`` management command."""
+
+from datetime import datetime, timezone
+from os.path import splitext
+from unittest import mock
+
+from django.core.management import call_command
+from django.test import TestCase
+
+from botocore.stub import Stubber
+
+from marsha import settings
+from marsha.core.defaults import AWS_S3, PENDING, READY, SCW_S3
+from marsha.core.utils import time_utils
+from marsha.deposit.factories import DepositedFileFactory
+from marsha.deposit.management.commands import rename_deposited_files
+from marsha.deposit.models import DepositedFile
+
+
+class RenameDepositedFilesTestCase(TestCase):
+    """
+    Test the ``rename_deposited_files`` command.
+    """
+
+    @mock.patch("marsha.core.storage.storage_class.file_storage.exists")
+    def test_rename_deposited_files(self, mock_exists):
+        """Command should rename S3 objects to their filename."""
+
+        mock_exists.return_value = False
+
+        now = datetime(2018, 8, 8, tzinfo=timezone.utc)
+
+        with Stubber(rename_deposited_files.s3_client) as s3_client_stubber:
+            # Generate some classroom files
+            # (<original filename>, <expected and cleaned>)
+            filenames = [
+                ("normal_filename.pdf", "normal_filename.pdf"),
+                ("weird\\file/name.pdf", "weird_file_name.pdf"),
+                (".hidden_file", "hidden_file"),
+            ]
+
+            files = []
+            for filename_src, _ in filenames:
+                file = DepositedFileFactory(
+                    filename=filename_src,
+                    uploaded_on=now,
+                    upload_state=READY,
+                    storage_location=AWS_S3,
+                )
+                files.append(file)
+
+            # Create mocks for copy_objects with Stubber
+            # Note: Stubber requires that its mocks are called in the exact order they
+            # were created, so we must iterate over objects.all() in the same sequence
+            for file in DepositedFile.objects.all():
+                stamp = time_utils.to_timestamp(file.uploaded_on)
+                extension = ""
+                if "." in file.filename:
+                    extension = splitext(file.filename)[1]
+
+                file_key_src = (
+                    f"aws/{file.file_depository.id}/depositedfile/"
+                    f"{file.id}/{stamp}{extension}"
+                )
+
+                sanitized_filename = rename_deposited_files.Command().validate_filename(
+                    file.filename
+                )
+                file_key_dest = (
+                    f"aws/{file.file_depository.id}/depositedfile/"
+                    f"{file.id}/{sanitized_filename}"
+                )
+
+                expected_params = {
+                    "Bucket": settings.STORAGE_S3_BUCKET_NAME,
+                    "CopySource": {
+                        "Bucket": settings.STORAGE_S3_BUCKET_NAME,
+                        "Key": file_key_src,
+                    },
+                    "Key": file_key_dest,
+                }
+                s3_client_stubber.add_response("copy_object", {}, expected_params)
+
+            # Create some classroom files that should not be concerned
+            DepositedFileFactory(
+                upload_state=READY,
+                storage_location=SCW_S3,
+            )
+            DepositedFileFactory(
+                upload_state=PENDING,
+                storage_location=AWS_S3,
+            )
+
+            call_command("rename_deposited_files")
+
+            s3_client_stubber.assert_no_pending_responses()
+
+            # Check that each file.filename has been updated with the clean
+            # S3-compatible filename
+            for file, (_, expected_filename) in zip(files, filenames):
+                file.refresh_from_db()
+                assert file.filename == expected_filename
+
+    @mock.patch("marsha.core.storage.storage_class.file_storage.exists")
+    def test_rename_deposited_files_file_exists(self, mock_exists):
+        """Command should not copy file if file already exists."""
+
+        mock_exists.return_value = True
+
+        now = datetime(2018, 8, 8, tzinfo=timezone.utc)
+
+        DepositedFileFactory(
+            filename="filename.pdf",
+            uploaded_on=now,
+        )
+
+        with Stubber(rename_deposited_files.s3_client) as s3_client_stubber:
+            call_command("rename_deposited_files")
+            s3_client_stubber.assert_no_pending_responses()
